From b5f56b92b45150ab92c7879e5e2cd6733dfab059 Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Fri, 29 May 2026 12:06:00 +0800 Subject: [PATCH 01/29] fix: classify InsertInto as write request (#8200) Signed-off-by: Ruihang Xia --- src/auth/src/permission.rs | 21 +++++++-- tests-integration/src/grpc.rs | 84 ++++++++++++++++++++++++++++++++++- 2 files changed, 100 insertions(+), 5 deletions(-) diff --git a/src/auth/src/permission.rs b/src/auth/src/permission.rs index 88adfda633..8914635290 100644 --- a/src/auth/src/permission.rs +++ b/src/auth/src/permission.rs @@ -16,6 +16,7 @@ use std::fmt::Debug; use std::sync::Arc; use api::v1::greptime_request::Request; +use api::v1::query_request::Query; use common_telemetry::debug; use sql::statements::statement::Statement; @@ -42,10 +43,12 @@ impl<'a> PermissionReq<'a> { /// Returns true if the permission request is for read operations. pub fn is_readonly(&self) -> bool { match self { - PermissionReq::GrpcRequest(Request::Query(_)) - | PermissionReq::PromQuery - | PermissionReq::LogQuery - | PermissionReq::PromStoreRead => true, + PermissionReq::GrpcRequest(Request::Query(query_request)) => { + !matches!(query_request.query, Some(Query::InsertIntoPlan(_))) + } + PermissionReq::PromQuery | PermissionReq::LogQuery | PermissionReq::PromStoreRead => { + true + } PermissionReq::SqlStatement(stmt) => stmt.is_readonly(), PermissionReq::GrpcRequest(_) @@ -196,4 +199,14 @@ mod tests { assert!(matches!(read_result, PermissionResp::Reject)); assert!(matches!(write_result, PermissionResp::Allow)); } + + #[test] + fn test_grpc_insert_into_plan_is_write_request() { + let request = Request::Query(api::v1::QueryRequest { + query: Some(Query::InsertIntoPlan(api::v1::InsertIntoPlan::default())), + }); + let req = PermissionReq::GrpcRequest(&request); + + assert!(req.is_write()); + } } diff --git a/tests-integration/src/grpc.rs b/tests-integration/src/grpc.rs index 181e698e87..7c2148e130 100644 --- a/tests-integration/src/grpc.rs +++ b/tests-integration/src/grpc.rs @@ -54,13 +54,19 @@ mod test { use api::v1::{ AddColumn, AddColumns, AlterTableExpr, Column, ColumnDataType, ColumnDataTypeExtension, ColumnDef, CreateDatabaseExpr, CreateTableExpr, DdlRequest, DeleteRequest, DeleteRequests, - DropTableExpr, InsertRequest, InsertRequests, QueryRequest, SemanticType, + DropTableExpr, InsertIntoPlan, InsertRequest, InsertRequests, QueryRequest, SemanticType, VectorTypeExtension, alter_table_expr, }; + use auth::{ + DefaultPermissionChecker, Identity, Password, PermissionCheckerRef, UserProvider, + static_user_provider_from_option, + }; use client::OutputData; + use common_base::Plugins; use common_catalog::consts::MITO_ENGINE; use common_meta::rpc::router::region_distribution; use common_query::Output; + use common_query::logical_plan::breakup_insert_plan; use common_recordbatch::RecordBatches; use frontend::instance::Instance; use query::parser::QueryLanguageParser; @@ -129,6 +135,82 @@ mod test { .unwrap() } + #[tokio::test(flavor = "multi_thread")] + async fn test_grpc_insert_into_plan_rejects_readonly_user() { + let plugins = Plugins::new(); + plugins.insert::(DefaultPermissionChecker::arc()); + + let standalone = + GreptimeDbStandaloneBuilder::new("test_grpc_insert_into_plan_rejects_readonly_user") + .with_plugin(plugins) + .build() + .await; + let instance = standalone.fe_instance(); + let table_name = "grpc_insert_into_plan_auth"; + + create_table( + instance, + format!("CREATE TABLE {table_name} (host STRING, val DOUBLE, ts TIMESTAMP TIME INDEX)"), + ) + .await; + + let stmt = QueryLanguageParser::parse_sql( + &format!("INSERT INTO {table_name} VALUES ('readonly-bypass', 42.0, 1000)"), + &QueryContext::arc(), + ) + .unwrap(); + let plan = instance + .statement_executor() + .plan(&stmt, QueryContext::arc()) + .await + .unwrap(); + let (table_name, insert_plan) = breakup_insert_plan(&plan, "greptime", "public").unwrap(); + let logical_plan = DFLogicalSubstraitConvertor + .encode(&insert_plan, DefaultSerializer) + .unwrap() + .to_vec(); + + let request = Request::Query(QueryRequest { + query: Some(Query::InsertIntoPlan(InsertIntoPlan { + table_name: Some(table_name), + logical_plan, + })), + }); + let ctx = QueryContext::arc(); + let provider = + static_user_provider_from_option("static_user_provider:cmd:readonly:ro=readonly_pwd") + .unwrap(); + let readonly_user = provider + .authenticate( + Identity::UserId("readonly", None), + Password::PlainText("readonly_pwd".to_string().into()), + ) + .await + .unwrap(); + ctx.set_current_user(readonly_user); + + let err = GrpcQueryHandler::do_query(instance.as_ref(), request, ctx) + .await + .unwrap_err(); + let err_msg = format!("{err:?}"); + assert!( + err_msg.contains("not authorized"), + "unexpected error: {err_msg}" + ); + + query_and_expect( + instance, + "SELECT count(*) FROM grpc_insert_into_plan_auth", + "\ ++----------+ +| count(*) | ++----------+ +| 0 | ++----------+", + ) + .await; + } + async fn test_handle_multi_ddl_request(instance: &Instance) { let request = Request::Ddl(DdlRequest { expr: Some(DdlExpr::CreateDatabase(CreateDatabaseExpr { From ba15a9c056668e22357860376a7502b1928a15c3 Mon Sep 17 00:00:00 2001 From: discord9 Date: Fri, 29 May 2026 14:59:21 +0800 Subject: [PATCH 02/29] feat: support pending flow metadata with defer_on_missing_source (#8124) * feat: support defer_on_missing_source for pending flow creation Add `defer_on_missing_source` flow option that allows creating flows even when source tables do not yet exist. The flow enters a pending state and is automatically activated when source tables become available. Key changes: - New `FlowStatus::PendingSources` and fields in `FlowInfoValue` for unresolved source table names and last activation error - `defer_on_missing_source` create-time-only option: stripped from runtime/flownode `CreateRequest` but preserved in metadata for SQL round-trip (`SHOW CREATE FLOW`, `information_schema.flows`) - `CreateFlowProcedure` creates pending metadata when sources are missing and `defer_on_missing_source=true`; falls back to `FlowType::Batching` for missing-source flows - `PendingFlowReconcileManager` in meta-srv periodically checks pending flows and activates them when source tables resolve - `ActivatePendingFlowProcedure` handles activation: allocates peers, creates flows on flownodes, updates metadata, invalidates cache - `OR REPLACE` properly handles pending<->active transitions, including peer allocation and flownode flow teardown - `FlowMetadataAllocator::alloc_peers` for peer allocation at activation time - Validated flow options: only `defer_on_missing_source` allowed; unknown options rejected - Known issue: standalone mode does not support flownodes, so pending flow flush/sink behavior covered only in distributed sqlness; operator and meta unit tests cover activation logic Tests: - operator `determine_flow_type_for_source_state` (3 passed) - common-meta `create_flow` (19 passed) including replacement - common-meta `activate_flow` (4 passed) - meta-srv `flow` (11 passed) - sqlness: `flow_pending` covers create/replace/round-trip Signed-off-by: discord9 * chore: simplify pending flow PR scope Reduce PR #8124 to the metadata-only MVP after complexity review. Changes: - Remove automatic activation procedure and meta-srv reconcile wiring - Remove activation tests and activation-only metadata fields - Reject cross-state pending<->active `OR REPLACE` transitions for now - Keep pending metadata creation and SQL round-trip behavior - Allow `DROP FLOW` for pending flows without routes - Reduce flow_pending sqlness to metadata/round-trip/drop coverage only Deferred follow-ups are documented locally in `.tmp/tasks/pending-defer-semantics/deferred-followups.md` and intentionally not committed. Tests: - `cargo test -p operator determine_flow_type_for_source_state` - `cargo test -p common-meta create_flow` - `cargo test -p common-meta drop_flow` - `cargo sqlness bare --test-filter flow_pending --bins-dir /mnt/nvme_rust/rust-targets/pending_defer/debug` Signed-off-by: discord9 * test: cover pending flow metadata edge cases Signed-off-by: discord9 * test: fix pending flow metadata test lint Signed-off-by: discord9 * docs: document pending flow metadata fields Signed-off-by: discord9 * chore: more sleep when test Signed-off-by: discord9 --------- Signed-off-by: discord9 --- .../meta/src/cache/flow/table_flownode.rs | 5 +- src/common/meta/src/ddl/create_flow.rs | 132 ++++- .../meta/src/ddl/create_flow/metadata.rs | 29 +- src/common/meta/src/ddl/drop_flow/metadata.rs | 2 +- src/common/meta/src/ddl/tests/create_flow.rs | 487 +++++++++++++++++- src/common/meta/src/ddl/tests/drop_flow.rs | 44 +- src/common/meta/src/key/flow.rs | 10 + src/common/meta/src/key/flow/flow_info.rs | 69 ++- src/operator/src/statement/ddl.rs | 95 +++- .../common/flow/flow_pending.result | 52 ++ .../standalone/common/flow/flow_pending.sql | 24 + .../common/flow/flow_rebuild.result | 1 + .../standalone/common/flow/flow_rebuild.sql | 1 + 13 files changed, 910 insertions(+), 41 deletions(-) create mode 100644 tests/cases/standalone/common/flow/flow_pending.result create mode 100644 tests/cases/standalone/common/flow/flow_pending.sql diff --git a/src/common/meta/src/cache/flow/table_flownode.rs b/src/common/meta/src/cache/flow/table_flownode.rs index ebe3664202..4d3513a21d 100644 --- a/src/common/meta/src/cache/flow/table_flownode.rs +++ b/src/common/meta/src/cache/flow/table_flownode.rs @@ -210,7 +210,7 @@ mod tests { use crate::cache::flow::table_flownode::{FlowIdent, new_table_flownode_set_cache}; use crate::instruction::{CacheIdent, CreateFlow, DropFlow}; use crate::key::flow::FlowMetadataManager; - use crate::key::flow::flow_info::FlowInfoValue; + use crate::key::flow::flow_info::{FlowInfoValue, FlowStatus}; use crate::key::flow::flow_route::FlowRouteValue; use crate::kv_backend::memory::MemoryKvBackend; use crate::peer::Peer; @@ -242,11 +242,14 @@ mod tests { catalog_name: DEFAULT_CATALOG_NAME.to_string(), query_context: None, flow_name: "my_flow".to_string(), + all_source_table_names: vec![], + unresolved_source_table_names: vec![], raw_sql: "sql".to_string(), expire_after: Some(300), eval_interval_secs: None, comment: "comment".to_string(), options: Default::default(), + status: FlowStatus::Active, created_time: chrono::Utc::now(), updated_time: chrono::Utc::now(), }, diff --git a/src/common/meta/src/ddl/create_flow.rs b/src/common/meta/src/ddl/create_flow.rs index 7120e50425..ddfb0c0759 100644 --- a/src/common/meta/src/ddl/create_flow.rs +++ b/src/common/meta/src/ddl/create_flow.rs @@ -14,7 +14,7 @@ mod metadata; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::fmt; use api::v1::ExpireAfter; @@ -34,13 +34,14 @@ use serde::{Deserialize, Serialize}; use snafu::{ResultExt, ensure}; use strum::AsRefStr; use table::metadata::TableId; +use table::table_name::TableName; use crate::cache_invalidator::Context; use crate::ddl::DdlContext; use crate::ddl::utils::{add_peer_context_if_needed, map_to_procedure_error}; use crate::error::{self, Result, UnexpectedSnafu}; use crate::instruction::{CacheIdent, CreateFlow, DropFlow}; -use crate::key::flow::flow_info::FlowInfoValue; +use crate::key::flow::flow_info::{FlowInfoValue, FlowStatus}; use crate::key::flow::flow_route::FlowRouteValue; use crate::key::table_name::TableNameKey; use crate::key::{DeserializedValueWithBytes, FlowId, FlowPartitionId}; @@ -67,6 +68,7 @@ impl CreateFlowProcedure { flow_id: None, peers: vec![], source_table_ids: vec![], + unresolved_source_table_names: vec![], flow_context: query_context.into(), // Convert to FlowQueryContext state: CreateFlowState::Prepare, prev_flow_info_value: None, @@ -89,6 +91,8 @@ impl CreateFlowProcedure { let create_if_not_exists = self.data.task.create_if_not_exists; let or_replace = self.data.task.or_replace; + validate_flow_options(&self.data.task)?; + let flow_name_value = self .context .flow_metadata_manager @@ -167,6 +171,21 @@ impl CreateFlowProcedure { } self.collect_source_tables().await?; + ensure!( + self.data.unresolved_source_table_names.is_empty() + || defer_on_missing_source(&self.data.task)?, + error::UnsupportedSnafu { + operation: format!( + "Create flow with missing source tables requires WITH ('{DEFER_ON_MISSING_SOURCE_KEY}'='true'): {}", + self.data + .unresolved_source_table_names + .iter() + .map(ToString::to_string) + .join(", ") + ) + } + ); + self.ensure_supported_replace_transition()?; // Validate that source and sink tables are not the same let sink_table_name = &self.data.task.sink_table_name; @@ -189,13 +208,38 @@ impl CreateFlowProcedure { if self.data.flow_id.is_none() { self.allocate_flow_id().await?; } - self.data.state = CreateFlowState::CreateFlows; - // determine flow type self.data.flow_type = Some(get_flow_type_from_options(&self.data.task)?); + self.data.state = if self.data.is_pending() { + self.data.peers.clear(); + CreateFlowState::CreateMetadata + } else { + CreateFlowState::CreateFlows + }; + Ok(Status::executing(true)) } + fn ensure_supported_replace_transition(&self) -> Result<()> { + if !self.data.task.or_replace { + return Ok(()); + } + + let Some(prev_flow_info) = self.data.prev_flow_info_value.as_ref() else { + return Ok(()); + }; + let prev_pending = prev_flow_info.get_inner_ref().is_pending(); + let new_pending = self.data.is_pending(); + ensure!( + prev_pending == new_pending, + error::UnsupportedSnafu { + operation: "Replacing between pending and active flow states is not supported yet" + } + ); + + Ok(()) + } + async fn on_flownode_create_flows(&mut self) -> Result { // Safety: must be allocated. let mut create_flow = Vec::with_capacity(self.data.peers.len()); @@ -365,6 +409,61 @@ pub fn get_flow_type_from_options(flow_task: &CreateFlowTask) -> Result Result { + flow_task + .flow_options + .get(DEFER_ON_MISSING_SOURCE_KEY) + .map(|value| { + value + .trim() + .to_ascii_lowercase() + .parse::() + .map_err(|_| { + error::UnexpectedSnafu { + err_msg: format!( + "Invalid flow option '{DEFER_ON_MISSING_SOURCE_KEY}': {value}" + ), + } + .build() + }) + }) + .transpose() + .map(|value| value.unwrap_or(false)) +} + +pub fn validate_flow_options(flow_task: &CreateFlowTask) -> Result<()> { + for key in flow_task.flow_options.keys() { + match key.as_str() { + DEFER_ON_MISSING_SOURCE_KEY | FlowType::FLOW_TYPE_KEY => {} + unknown => { + return UnexpectedSnafu { + err_msg: format!( + "Unknown flow option '{unknown}', supported user options: {DEFER_ON_MISSING_SOURCE_KEY}" + ), + } + .fail(); + } + } + } + + defer_on_missing_source(flow_task)?; + get_flow_type_from_options(flow_task)?; + Ok(()) +} + +fn user_runtime_flow_options(options: &HashMap) -> HashMap { + let mut options = options.clone(); + options.remove(DEFER_ON_MISSING_SOURCE_KEY); + options +} + +fn metadata_flow_options(options: &HashMap) -> HashMap { + options.clone() +} + /// The state of [CreateFlowProcedure]. #[derive(Debug, Clone, Serialize, Deserialize, AsRefStr, PartialEq)] pub enum CreateFlowState { @@ -411,6 +510,8 @@ pub struct CreateFlowData { pub(crate) flow_id: Option, pub(crate) peers: Vec, pub(crate) source_table_ids: Vec, + #[serde(default)] + pub(crate) unresolved_source_table_names: Vec, /// Use alias for backward compatibility with QueryContext serialized data #[serde(alias = "query_context")] pub(crate) flow_context: FlowQueryContext, @@ -424,6 +525,16 @@ pub struct CreateFlowData { pub(crate) flow_type: Option, } +impl CreateFlowData { + pub(crate) fn is_pending(&self) -> bool { + !self.unresolved_source_table_names.is_empty() + } + + pub(crate) fn is_active(&self) -> bool { + !self.is_pending() + } +} + impl From<&CreateFlowData> for CreateRequest { fn from(value: &CreateFlowData) -> Self { let flow_id = value.flow_id.unwrap(); @@ -446,7 +557,7 @@ impl From<&CreateFlowData> for CreateRequest { .map(|seconds| api::v1::EvalInterval { seconds }), comment: value.task.comment.clone(), sql: value.task.sql.clone(), - flow_options: value.task.flow_options.clone(), + flow_options: user_runtime_flow_options(&value.task.flow_options), }; let flow_type = value.flow_type.unwrap_or_default().to_string(); @@ -466,9 +577,9 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa eval_interval_secs: eval_interval, comment, sql, - flow_options: mut options, .. } = value.task.clone(); + let mut options = metadata_flow_options(&value.task.flow_options); let flownode_ids = value .peers @@ -484,7 +595,7 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa .collect::>(); let flow_type = value.flow_type.unwrap_or_default().to_string(); - options.insert("flow_type".to_string(), flow_type); + options.insert(FlowType::FLOW_TYPE_KEY.to_string(), flow_type); let mut create_time = chrono::Utc::now(); if let Some(prev_flow_value) = value.prev_flow_info_value.as_ref() @@ -495,6 +606,8 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa let flow_info: FlowInfoValue = FlowInfoValue { source_table_ids: value.source_table_ids.clone(), + all_source_table_names: value.task.source_table_names.clone(), + unresolved_source_table_names: value.unresolved_source_table_names.clone(), sink_table_name, flownode_ids, catalog_name, @@ -506,6 +619,11 @@ impl From<&CreateFlowData> for (FlowInfoValue, Vec<(FlowPartitionId, FlowRouteVa eval_interval_secs: eval_interval, comment, options, + status: if value.is_active() { + FlowStatus::Active + } else { + FlowStatus::PendingSources + }, created_time: create_time, updated_time: chrono::Utc::now(), }; diff --git a/src/common/meta/src/ddl/create_flow/metadata.rs b/src/common/meta/src/ddl/create_flow/metadata.rs index 27b85b7946..f97ecfdf4a 100644 --- a/src/common/meta/src/ddl/create_flow/metadata.rs +++ b/src/common/meta/src/ddl/create_flow/metadata.rs @@ -12,10 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use snafu::OptionExt; - use crate::ddl::create_flow::CreateFlowProcedure; -use crate::error::{self, Result}; +use crate::error::Result; use crate::key::table_name::TableNameKey; impl CreateFlowProcedure { @@ -34,9 +32,8 @@ impl CreateFlowProcedure { Ok(()) } - /// Ensures all source tables exist and collects source table ids + /// Collects source table ids and keeps track of missing tables. pub(crate) async fn collect_source_tables(&mut self) -> Result<()> { - // Ensures all source tables exist. let keys = self .data .task @@ -52,22 +49,24 @@ impl CreateFlowProcedure { .batch_get(keys) .await?; - let source_table_ids = self + let mut resolved = Vec::with_capacity(self.data.task.source_table_names.len()); + let mut unresolved = Vec::new(); + + for (name, table_id) in self .data .task .source_table_names .iter() .zip(source_table_ids) - .map(|(name, table_id)| { - Ok(table_id - .with_context(|| error::TableNotFoundSnafu { - table_name: name.to_string(), - })? - .table_id()) - }) - .collect::>>()?; + { + match table_id { + Some(table_id) => resolved.push(table_id.table_id()), + None => unresolved.push(name.clone()), + } + } - self.data.source_table_ids = source_table_ids; + self.data.source_table_ids = resolved; + self.data.unresolved_source_table_names = unresolved; Ok(()) } } diff --git a/src/common/meta/src/ddl/drop_flow/metadata.rs b/src/common/meta/src/ddl/drop_flow/metadata.rs index 0437098be3..7afd00f9d5 100644 --- a/src/common/meta/src/ddl/drop_flow/metadata.rs +++ b/src/common/meta/src/ddl/drop_flow/metadata.rs @@ -43,7 +43,7 @@ impl DropFlowProcedure { .map(|(_, value)| value) .collect::>(); ensure!( - !flow_route_values.is_empty(), + flow_info_value.is_pending() || !flow_route_values.is_empty(), error::FlowRouteNotFoundSnafu { flow_name: format_full_flow_name(catalog_name, flow_name), } diff --git a/src/common/meta/src/ddl/tests/create_flow.rs b/src/common/meta/src/ddl/tests/create_flow.rs index 344fc05024..a1a6c040f1 100644 --- a/src/common/meta/src/ddl/tests/create_flow.rs +++ b/src/common/meta/src/ddl/tests/create_flow.rs @@ -16,12 +16,17 @@ use std::assert_matches; use std::collections::HashMap; use std::sync::Arc; +use api::v1::flow::CreateRequest; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; +use common_procedure::Status; use common_procedure_test::execute_procedure_until_done; use table::table_name::TableName; use crate::ddl::DdlContext; -use crate::ddl::create_flow::{CreateFlowData, CreateFlowProcedure, CreateFlowState, FlowType}; +use crate::ddl::create_flow::{ + CreateFlowData, CreateFlowProcedure, CreateFlowState, DEFER_ON_MISSING_SOURCE_KEY, FlowType, + defer_on_missing_source, +}; use crate::ddl::test_util::create_table::test_create_table_task; use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler; use crate::error; @@ -63,6 +68,11 @@ pub(crate) fn test_create_flow_task( } } +fn enable_defer_on_missing_source(task: &mut CreateFlowTask) { + task.flow_options + .insert(DEFER_ON_MISSING_SOURCE_KEY.to_string(), "true".to_string()); +} + #[tokio::test] async fn test_create_flow_source_table_not_found() { let source_table_names = vec![TableName::new( @@ -78,7 +88,261 @@ async fn test_create_flow_source_table_not_found() { let query_ctx = test_query_context(); let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context); let err = procedure.on_prepare().await.unwrap_err(); - assert_matches!(err, error::Error::TableNotFound { .. }); + assert_matches!(err, error::Error::Unsupported { .. }); + assert!( + err.to_string() + .contains("requires WITH ('defer_on_missing_source'='true')") + ); +} + +#[tokio::test] +async fn test_create_pending_flow_source_table_not_found_with_defer() { + let source_table_names = vec![TableName::new( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + "my_table", + )]; + let sink_table_name = + TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"); + let mut task = test_create_flow_task("my_flow", source_table_names, sink_table_name, false); + enable_defer_on_missing_source(&mut task); + let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); + let ddl_context = new_ddl_context(node_manager); + let query_ctx = test_query_context(); + let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone()); + let status = procedure.on_prepare().await.unwrap(); + assert_matches!(status, Status::Executing { persist: true, .. }); + assert_eq!(procedure.data.unresolved_source_table_names.len(), 1); + assert_eq!(procedure.data.source_table_ids, Vec::::new()); + + let output = execute_procedure_until_done(&mut procedure).await.unwrap(); + let flow_id = *output.downcast_ref::().unwrap(); + let flow_info = ddl_context + .flow_metadata_manager + .flow_info_manager() + .get(flow_id) + .await + .unwrap() + .unwrap(); + assert_eq!(flow_info.source_table_ids(), Vec::::new()); + assert_eq!( + flow_info + .options() + .get(DEFER_ON_MISSING_SOURCE_KEY) + .map(String::as_str), + Some("true") + ); +} + +#[tokio::test] +async fn test_create_pending_flow_source_table_not_found_with_defer_false() { + let source_table_names = vec![TableName::new( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + "my_table", + )]; + let sink_table_name = + TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"); + let mut task = test_create_flow_task("my_flow", source_table_names, sink_table_name, false); + task.flow_options + .insert(DEFER_ON_MISSING_SOURCE_KEY.to_string(), "false".to_string()); + let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); + let ddl_context = new_ddl_context(node_manager); + let query_ctx = test_query_context(); + let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context); + let err = procedure.on_prepare().await.unwrap_err(); + assert_matches!(err, error::Error::Unsupported { .. }); + assert!( + err.to_string() + .contains("requires WITH ('defer_on_missing_source'='true')") + ); +} + +#[tokio::test] +async fn test_create_pending_flow_records_partial_source_resolution() { + let existing_source = TableName::new( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + "partial_existing_source_table", + ); + let missing_source = TableName::new( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + "partial_missing_source_table", + ); + let sink_table_name = TableName::new( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + "partial_pending_sink_table", + ); + let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); + let ddl_context = new_ddl_context(node_manager); + + let existing_table_id = 3026; + let create_table_task = + test_create_table_task("partial_existing_source_table", existing_table_id); + ddl_context + .table_metadata_manager + .create_table_metadata( + create_table_task.table_info.clone(), + TableRouteValue::physical(vec![]), + HashMap::new(), + ) + .await + .unwrap(); + + let mut task = test_create_flow_task( + "partial_pending_flow", + vec![existing_source.clone(), missing_source.clone()], + sink_table_name, + false, + ); + enable_defer_on_missing_source(&mut task); + let query_ctx = test_query_context(); + let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone()); + let status = procedure.on_prepare().await.unwrap(); + assert_matches!(status, Status::Executing { persist: true, .. }); + assert_eq!(procedure.data.source_table_ids, vec![existing_table_id]); + assert_eq!( + procedure.data.unresolved_source_table_names, + vec![missing_source.clone()] + ); + + let output = execute_procedure_until_done(&mut procedure).await.unwrap(); + let flow_id = *output.downcast_ref::().unwrap(); + let flow_info = ddl_context + .flow_metadata_manager + .flow_info_manager() + .get(flow_id) + .await + .unwrap() + .unwrap(); + + assert!(flow_info.is_pending()); + assert_eq!(flow_info.source_table_ids(), &[existing_table_id]); + let expected_all_sources = vec![existing_source, missing_source.clone()]; + assert_eq!( + flow_info.all_source_table_names(), + expected_all_sources.as_slice() + ); + assert_eq!(flow_info.unresolved_source_table_names(), &[missing_source]); + assert!(flow_info.flownode_ids().is_empty()); +} + +#[test] +fn test_defer_on_missing_source_defaults_false() { + let task = test_create_flow_task( + "my_flow", + vec![], + TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"), + false, + ); + + assert!(!defer_on_missing_source(&task).unwrap()); +} + +#[test] +fn test_defer_on_missing_source_true() { + let mut task = test_create_flow_task( + "my_flow", + vec![], + TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"), + false, + ); + task.flow_options + .insert(DEFER_ON_MISSING_SOURCE_KEY.to_string(), "true".to_string()); + + assert!(defer_on_missing_source(&task).unwrap()); +} + +#[test] +fn test_defer_on_missing_source_invalid_value() { + let mut task = test_create_flow_task( + "my_flow", + vec![], + TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"), + false, + ); + task.flow_options.insert( + DEFER_ON_MISSING_SOURCE_KEY.to_string(), + "invalid".to_string(), + ); + + let err = defer_on_missing_source(&task).unwrap_err(); + assert!( + err.to_string() + .contains("Invalid flow option 'defer_on_missing_source': invalid") + ); +} + +#[tokio::test] +async fn test_create_flow_rejects_unknown_option_in_meta_task() { + let mut task = test_create_flow_task( + "my_flow", + vec![], + TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"), + false, + ); + task.flow_options + .insert("unknown_option".to_string(), "value".to_string()); + let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); + let ddl_context = new_ddl_context(node_manager); + let query_ctx = test_query_context(); + let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context); + + let err = procedure.on_prepare().await.unwrap_err(); + assert_matches!(err, error::Error::Unexpected { .. }); + assert!( + err.to_string() + .contains("Unknown flow option 'unknown_option'") + ); +} + +#[test] +fn test_create_request_strips_defer_on_missing_source_runtime_option() { + let mut task = test_create_flow_task( + "my_flow", + vec![], + TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"), + false, + ); + enable_defer_on_missing_source(&mut task); + + let data = CreateFlowData { + state: CreateFlowState::CreateFlows, + task, + flow_id: Some(1024), + peers: vec![], + source_table_ids: vec![], + unresolved_source_table_names: vec![], + flow_context: FlowQueryContext { + catalog: DEFAULT_CATALOG_NAME.to_string(), + schema: DEFAULT_SCHEMA_NAME.to_string(), + timezone: "UTC".to_string(), + extensions: HashMap::new(), + channel: 0, + snapshot_seqs: HashMap::new(), + sst_min_sequences: HashMap::new(), + }, + prev_flow_info_value: None, + did_replace: false, + flow_type: Some(FlowType::Batching), + }; + + let request: CreateRequest = (&data).into(); + + assert!( + !request + .flow_options + .contains_key(DEFER_ON_MISSING_SOURCE_KEY) + ); + assert_eq!( + request + .flow_options + .get(FlowType::FLOW_TYPE_KEY) + .map(String::as_str), + Some(FlowType::BATCHING) + ); } pub(crate) async fn create_test_flow( @@ -101,6 +365,27 @@ pub(crate) async fn create_test_flow( *flow_id } +pub(crate) async fn create_test_pending_flow( + ddl_context: &DdlContext, + flow_name: &str, + source_table_names: Vec, + sink_table_name: TableName, +) -> FlowId { + let mut task = test_create_flow_task( + flow_name, + source_table_names.clone(), + sink_table_name.clone(), + false, + ); + enable_defer_on_missing_source(&mut task); + let query_ctx = test_query_context(); + let mut procedure = CreateFlowProcedure::new(task, query_ctx, ddl_context.clone()); + let output = execute_procedure_until_done(&mut procedure).await.unwrap(); + let flow_id = output.downcast_ref::().unwrap(); + + *flow_id +} + #[tokio::test] async fn test_create_flow() { let table_id = 1024; @@ -154,6 +439,201 @@ async fn test_create_flow() { assert_matches!(err, error::Error::FlowAlreadyExists { .. }); } +#[tokio::test] +async fn test_replace_pending_flow_with_active_flow_is_unsupported() { + let source_table_name = TableName::new( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + "replace_pending_source_table", + ); + let sink_table_name = TableName::new( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + "replace_pending_sink_table", + ); + let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); + let ddl_context = new_ddl_context(node_manager); + + let pending_flow_id = create_test_pending_flow( + &ddl_context, + "replace_pending_flow", + vec![source_table_name.clone()], + sink_table_name.clone(), + ) + .await; + + let pending_flow = ddl_context + .flow_metadata_manager + .flow_info_manager() + .get(pending_flow_id) + .await + .unwrap() + .unwrap(); + assert!(pending_flow.is_pending()); + assert!(pending_flow.flownode_ids().is_empty()); + + let create_table_task = test_create_table_task("replace_pending_source_table", 1026); + ddl_context + .table_metadata_manager + .create_table_metadata( + create_table_task.table_info.clone(), + TableRouteValue::physical(vec![]), + HashMap::new(), + ) + .await + .unwrap(); + + let mut replace_task = test_create_flow_task( + "replace_pending_flow", + vec![source_table_name], + sink_table_name, + false, + ); + replace_task.or_replace = true; + let query_ctx = test_query_context(); + let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone()); + let err = procedure.on_prepare().await.unwrap_err(); + assert_matches!(err, error::Error::Unsupported { .. }); + assert!( + err.to_string() + .contains("Replacing between pending and active flow states") + ); +} + +#[tokio::test] +async fn test_replace_active_flow_with_pending_flow_is_unsupported() { + let existing_source_table = TableName::new( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + "replace_active_source_table", + ); + let missing_source_table = TableName::new( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + "replace_missing_source_table", + ); + let sink_table_name = TableName::new( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + "replace_active_sink_table", + ); + + let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); + let ddl_context = new_ddl_context(node_manager); + + let create_table_task = test_create_table_task("replace_active_source_table", 2026); + ddl_context + .table_metadata_manager + .create_table_metadata( + create_table_task.table_info.clone(), + TableRouteValue::physical(vec![]), + HashMap::new(), + ) + .await + .unwrap(); + + let _flow_id = create_test_flow( + &ddl_context, + "replace_active_flow_to_pending", + vec![existing_source_table], + sink_table_name.clone(), + ) + .await; + + let mut replace_task = test_create_flow_task( + "replace_active_flow_to_pending", + vec![missing_source_table], + sink_table_name, + false, + ); + enable_defer_on_missing_source(&mut replace_task); + replace_task.or_replace = true; + let query_ctx = test_query_context(); + let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone()); + let err = procedure.on_prepare().await.unwrap_err(); + assert_matches!(err, error::Error::Unsupported { .. }); + assert!( + err.to_string() + .contains("Replacing between pending and active flow states") + ); +} + +#[tokio::test] +async fn test_replace_pending_flow_with_pending_flow_updates_metadata() { + let first_missing_source = TableName::new( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + "replace_pending_first_missing_source", + ); + let second_missing_source = TableName::new( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + "replace_pending_second_missing_source", + ); + let sink_table_name = TableName::new( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + "replace_pending_to_pending_sink_table", + ); + let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); + let ddl_context = new_ddl_context(node_manager); + + let original_flow_id = create_test_pending_flow( + &ddl_context, + "replace_pending_to_pending_flow", + vec![first_missing_source.clone()], + sink_table_name.clone(), + ) + .await; + + let original_flow = ddl_context + .flow_metadata_manager + .flow_info_manager() + .get(original_flow_id) + .await + .unwrap() + .unwrap(); + assert!(original_flow.is_pending()); + assert_eq!( + original_flow.unresolved_source_table_names(), + &[first_missing_source] + ); + assert!(original_flow.flownode_ids().is_empty()); + + let mut replace_task = test_create_flow_task( + "replace_pending_to_pending_flow", + vec![second_missing_source.clone()], + sink_table_name, + false, + ); + enable_defer_on_missing_source(&mut replace_task); + replace_task.or_replace = true; + let query_ctx = test_query_context(); + let mut procedure = CreateFlowProcedure::new(replace_task, query_ctx, ddl_context.clone()); + let output = execute_procedure_until_done(&mut procedure).await.unwrap(); + let replaced_flow_id = *output.downcast_ref::().unwrap(); + assert_eq!(replaced_flow_id, original_flow_id); + + let replaced_flow = ddl_context + .flow_metadata_manager + .flow_info_manager() + .get(replaced_flow_id) + .await + .unwrap() + .unwrap(); + assert!(replaced_flow.is_pending()); + assert_eq!(replaced_flow.source_table_ids(), Vec::::new()); + assert_eq!( + replaced_flow.unresolved_source_table_names(), + std::slice::from_ref(&second_missing_source) + ); + assert_eq!( + replaced_flow.all_source_table_names(), + &[second_missing_source] + ); + assert!(replaced_flow.flownode_ids().is_empty()); +} + #[tokio::test] async fn test_create_flow_same_source_and_sink_table() { let table_id = 1024; @@ -228,6 +708,7 @@ fn test_create_flow_data_serialization_backward_compatibility() { "flow_id": null, "peers": [], "source_table_ids": [], + "unresolved_source_table_names": [], "query_context": { "current_catalog": "old_catalog", "current_schema": "old_schema", @@ -265,6 +746,7 @@ fn test_create_flow_data_new_format_serialization() { flow_id: None, peers: vec![], source_table_ids: vec![], + unresolved_source_table_names: vec![], flow_context, prev_flow_info_value: None, did_replace: false, @@ -327,6 +809,7 @@ fn test_flow_info_conversion_with_flow_context() { flow_id: Some(123), peers: vec![], source_table_ids: vec![456, 789], + unresolved_source_table_names: vec![], flow_context, prev_flow_info_value: None, did_replace: false, diff --git a/src/common/meta/src/ddl/tests/drop_flow.rs b/src/common/meta/src/ddl/tests/drop_flow.rs index af34da4809..400fd2e118 100644 --- a/src/common/meta/src/ddl/tests/drop_flow.rs +++ b/src/common/meta/src/ddl/tests/drop_flow.rs @@ -23,7 +23,7 @@ use table::table_name::TableName; use crate::ddl::drop_flow::DropFlowProcedure; use crate::ddl::test_util::create_table::test_create_table_task; use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler; -use crate::ddl::tests::create_flow::create_test_flow; +use crate::ddl::tests::create_flow::{create_test_flow, create_test_pending_flow}; use crate::error; use crate::key::table_route::TableRouteValue; use crate::rpc::ddl::DropFlowTask; @@ -91,3 +91,45 @@ async fn test_drop_flow() { let err = procedure.on_prepare().await.unwrap_err(); assert_matches!(err, error::Error::FlowNotFound { .. }); } + +#[tokio::test] +async fn test_drop_pending_flow_without_routes() { + let source_table_name = TableName::new( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + "drop_pending_missing_source_table", + ); + let sink_table_name = TableName::new( + DEFAULT_CATALOG_NAME, + DEFAULT_SCHEMA_NAME, + "drop_pending_sink_table", + ); + let node_manager = Arc::new(MockFlownodeManager::new(NaiveFlownodeHandler)); + let ddl_context = new_ddl_context(node_manager); + + let flow_id = create_test_pending_flow( + &ddl_context, + "drop_pending_flow", + vec![source_table_name], + sink_table_name, + ) + .await; + let flow_info = ddl_context + .flow_metadata_manager + .flow_info_manager() + .get(flow_id) + .await + .unwrap() + .unwrap(); + assert!(flow_info.is_pending()); + assert!(flow_info.flownode_ids().is_empty()); + + let task = test_drop_flow_task("drop_pending_flow", flow_id, false); + let mut procedure = DropFlowProcedure::new(task, ddl_context.clone()); + execute_procedure_until_done(&mut procedure).await; + + let task = test_drop_flow_task("drop_pending_flow", flow_id, false); + let mut procedure = DropFlowProcedure::new(task, ddl_context); + let err = procedure.on_prepare().await.unwrap_err(); + assert_matches!(err, error::Error::FlowNotFound { .. }); +} diff --git a/src/common/meta/src/key/flow.rs b/src/common/meta/src/key/flow.rs index d581b92685..bc9aaaa6b3 100644 --- a/src/common/meta/src/key/flow.rs +++ b/src/common/meta/src/key/flow.rs @@ -459,6 +459,7 @@ mod tests { use super::*; use crate::FlownodeId; + use crate::key::flow::flow_info::FlowStatus; use crate::key::flow::table_flow::TableFlowKey; use crate::key::node_address::{NodeAddressKey, NodeAddressValue}; use crate::key::{FlowPartitionId, MetadataValue}; @@ -522,6 +523,8 @@ mod tests { query_context: None, flow_name: flow_name.to_string(), source_table_ids, + all_source_table_names: vec![], + unresolved_source_table_names: vec![], sink_table_name, flownode_ids, raw_sql: "raw".to_string(), @@ -529,6 +532,7 @@ mod tests { eval_interval_secs: None, comment: "hi".to_string(), options: Default::default(), + status: FlowStatus::Active, created_time: chrono::Utc::now(), updated_time: chrono::Utc::now(), } @@ -774,6 +778,8 @@ mod tests { query_context: None, flow_name: "flow".to_string(), source_table_ids: vec![1024, 1025, 1026], + all_source_table_names: vec![], + unresolved_source_table_names: vec![], sink_table_name: another_sink_table_name, flownode_ids: [(0, 1u64)].into(), raw_sql: "raw".to_string(), @@ -781,6 +787,7 @@ mod tests { eval_interval_secs: None, comment: "hi".to_string(), options: Default::default(), + status: FlowStatus::Active, created_time: chrono::Utc::now(), updated_time: chrono::Utc::now(), }; @@ -1151,6 +1158,8 @@ mod tests { query_context: None, flow_name: "flow".to_string(), source_table_ids: vec![1024, 1025, 1026], + all_source_table_names: vec![], + unresolved_source_table_names: vec![], sink_table_name: another_sink_table_name, flownode_ids: [(0, 1u64)].into(), raw_sql: "raw".to_string(), @@ -1158,6 +1167,7 @@ mod tests { eval_interval_secs: None, comment: "hi".to_string(), options: Default::default(), + status: FlowStatus::Active, created_time: chrono::Utc::now(), updated_time: chrono::Utc::now(), }; diff --git a/src/common/meta/src/key/flow/flow_info.rs b/src/common/meta/src/key/flow/flow_info.rs index d501822c3c..b1056902da 100644 --- a/src/common/meta/src/key/flow/flow_info.rs +++ b/src/common/meta/src/key/flow/flow_info.rs @@ -16,6 +16,8 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use chrono::{DateTime, Utc}; +use futures::TryStreamExt; +use futures::stream::BoxStream; use lazy_static::lazy_static; use regex::Regex; use serde::{Deserialize, Serialize}; @@ -27,12 +29,27 @@ use crate::FlownodeId; use crate::error::{self, Result}; use crate::key::flow::FlowScoped; use crate::key::txn_helper::TxnOpGetResponseSet; -use crate::key::{DeserializedValueWithBytes, FlowId, FlowPartitionId, MetadataKey, MetadataValue}; +use crate::key::{ + BytesAdapter, DeserializedValueWithBytes, FlowId, FlowPartitionId, MetadataKey, MetadataValue, +}; use crate::kv_backend::KvBackendRef; use crate::kv_backend::txn::{Compare, CompareOp, Txn, TxnOp}; +use crate::range_stream::{DEFAULT_PAGE_SIZE, PaginationStream}; +use crate::rpc::KeyValue; +use crate::rpc::store::RangeRequest; pub const FLOW_INFO_KEY_PREFIX: &str = "info"; +/// The lifecycle status of a flow stored in metadata. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)] +pub enum FlowStatus { + /// The flow metadata exists, but at least one source table did not exist at create time. + PendingSources, + /// The flow has resolved source tables and can be scheduled on flownodes. + #[default] + Active, +} + lazy_static! { static ref FLOW_INFO_KEY_PATTERN: Regex = Regex::new(&format!("^{FLOW_INFO_KEY_PREFIX}/([0-9]+)$")).unwrap(); @@ -114,7 +131,12 @@ impl<'a> MetadataKey<'a, FlowInfoKeyInner> for FlowInfoKeyInner { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct FlowInfoValue { /// The source tables used by the flow. + #[serde(default)] pub source_table_ids: Vec, + #[serde(default)] + pub all_source_table_names: Vec, + #[serde(default)] + pub unresolved_source_table_names: Vec, /// The sink table used by the flow. pub sink_table_name: TableName, /// Which flow nodes this flow is running on. @@ -145,6 +167,8 @@ pub struct FlowInfoValue { pub comment: String, /// The options. pub options: HashMap, + #[serde(default)] + pub status: FlowStatus, /// The created time #[serde(default)] pub created_time: DateTime, @@ -154,6 +178,14 @@ pub struct FlowInfoValue { } impl FlowInfoValue { + pub fn is_pending(&self) -> bool { + self.status == FlowStatus::PendingSources + } + + pub fn is_active(&self) -> bool { + self.status == FlowStatus::Active + } + /// Returns the `flownode_id`. pub fn flownode_ids(&self) -> &BTreeMap { &self.flownode_ids @@ -173,6 +205,14 @@ impl FlowInfoValue { &self.source_table_ids } + pub fn all_source_table_names(&self) -> &[TableName] { + &self.all_source_table_names + } + + pub fn unresolved_source_table_names(&self) -> &[TableName] { + &self.unresolved_source_table_names + } + pub fn catalog_name(&self) -> &String { &self.catalog_name } @@ -209,6 +249,10 @@ impl FlowInfoValue { &self.options } + pub fn status(&self) -> &FlowStatus { + &self.status + } + pub fn created_time(&self) -> &DateTime { &self.created_time } @@ -225,6 +269,12 @@ pub struct FlowInfoManager { kv_backend: KvBackendRef, } +pub fn flow_info_decoder(kv: KeyValue) -> Result<(FlowInfoKey, FlowInfoValue)> { + let key = FlowInfoKey::from_bytes(&kv.key)?; + let value = FlowInfoValue::try_from_raw_value(&kv.value)?; + Ok((key, value)) +} + impl FlowInfoManager { /// Returns a new [FlowInfoManager]. pub fn new(kv_backend: KvBackendRef) -> Self { @@ -254,6 +304,23 @@ impl FlowInfoManager { .transpose() } + pub fn flow_infos(&self) -> BoxStream<'static, Result<(FlowId, FlowInfoValue)>> { + let start_key = FlowScoped::new(BytesAdapter::from( + format!("{FLOW_INFO_KEY_PREFIX}/").into_bytes(), + )) + .to_bytes(); + let req = RangeRequest::new().with_prefix(start_key); + let stream = PaginationStream::new( + self.kv_backend.clone(), + req, + DEFAULT_PAGE_SIZE, + flow_info_decoder, + ) + .into_stream(); + + Box::pin(stream.map_ok(|(key, value)| (key.flow_id(), value))) + } + /// Builds a create flow transaction. /// It is expected that the `__flow/info/{flow_id}` wasn't occupied. /// Otherwise, the transaction will retrieve existing value. diff --git a/src/operator/src/statement/ddl.rs b/src/operator/src/statement/ddl.rs index 1fab624f46..aef164c6bb 100644 --- a/src/operator/src/statement/ddl.rs +++ b/src/operator/src/statement/ddl.rs @@ -35,7 +35,7 @@ use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, is_reado use common_catalog::{format_full_flow_name, format_full_table_name}; use common_error::ext::BoxedError; use common_meta::cache_invalidator::Context; -use common_meta::ddl::create_flow::FlowType; +use common_meta::ddl::create_flow::{DEFER_ON_MISSING_SOURCE_KEY, FlowType}; use common_meta::instruction::CacheIdent; use common_meta::key::schema_name::{SchemaName, SchemaNameKey}; use common_meta::procedure_executor::ExecutorContext; @@ -114,7 +114,6 @@ struct DdlSubmitOptions { timeout: Duration, } -const DEFER_ON_MISSING_SOURCE_KEY: &str = "defer_on_missing_source"; const ALLOWED_FLOW_OPTIONS: [&str; 1] = [DEFER_ON_MISSING_SOURCE_KEY]; fn build_procedure_id_output(procedure_id: Vec) -> Result { @@ -205,6 +204,39 @@ fn validate_and_normalize_flow_options( .collect() } +fn determine_flow_type_for_source_state( + flow_name: &str, + flow_options: &HashMap, + has_missing_source_table: bool, + has_instant_ttl_source_table: bool, +) -> Result> { + if has_missing_source_table { + let defer_on_missing_source = flow_options + .get(DEFER_ON_MISSING_SOURCE_KEY) + .is_some_and(|value| value == "true"); + ensure!( + defer_on_missing_source, + InvalidSqlSnafu { + err_msg: format!( + "missing source tables for flow '{}'; use WITH ({DEFER_ON_MISSING_SOURCE_KEY} = true) to create a pending flow", + flow_name + ) + } + ); + info!( + "Flow `{}` is created as a pending batching flow because source tables are missing and defer_on_missing_source=true", + flow_name + ); + return Ok(Some(FlowType::Batching)); + } + + if has_instant_ttl_source_table { + return Ok(Some(FlowType::Streaming)); + } + + Ok(None) +} + impl StatementExecutor { pub fn catalog_manager(&self) -> CatalogManagerRef { self.catalog_manager.clone() @@ -715,7 +747,9 @@ impl StatementExecutor { expr: &CreateFlowExpr, query_ctx: QueryContextRef, ) -> Result { - // first check if source table's ttl is instant, if it is, force streaming mode + let mut has_missing_source_table = false; + let mut has_instant_ttl_source_table = false; + for src_table_name in &expr.source_table_names { let table = self .catalog_manager() @@ -727,16 +761,13 @@ impl StatementExecutor { ) .await .map_err(BoxedError::new) - .context(ExternalSnafu)? - .with_context(|| TableNotFoundSnafu { - table_name: format_full_table_name( - &src_table_name.catalog_name, - &src_table_name.schema_name, - &src_table_name.table_name, - ), - })?; + .context(ExternalSnafu)?; + + let Some(table) = table else { + has_missing_source_table = true; + continue; + }; - // instant source table can only be handled by streaming mode if table.table_info().meta.options.ttl == Some(common_time::TimeToLive::Instant) { warn!( "Source table `{}` for flow `{}`'s ttl=instant, fallback to streaming mode", @@ -747,10 +778,19 @@ impl StatementExecutor { ), expr.flow_name ); - return Ok(FlowType::Streaming); + has_instant_ttl_source_table = true; } } + if let Some(flow_type) = determine_flow_type_for_source_state( + &expr.flow_name, + &expr.flow_options, + has_missing_source_table, + has_instant_ttl_source_table, + )? { + return Ok(flow_type); + } + let engine = &self.query_engine; let stmts = ParserContext::create_with_dialect( &expr.sql, @@ -2515,6 +2555,35 @@ SELECT max(c1), min(c2) FROM schema_2.table_2;"; )); } + #[test] + fn test_determine_flow_type_for_source_state_missing_sources_require_opt_in() { + let err = determine_flow_type_for_source_state("my_flow", &HashMap::new(), true, false) + .unwrap_err(); + + assert!(err.to_string().contains( + "missing source tables for flow 'my_flow'; use WITH (defer_on_missing_source = true) to create a pending flow" + )); + } + + #[test] + fn test_determine_flow_type_for_source_state_missing_sources_prefer_batching() { + let flow_options = + HashMap::from([(DEFER_ON_MISSING_SOURCE_KEY.to_string(), "true".to_string())]); + + assert_eq!( + determine_flow_type_for_source_state("my_flow", &flow_options, true, true).unwrap(), + Some(FlowType::Batching) + ); + } + + #[test] + fn test_determine_flow_type_for_source_state_instant_ttl_without_missing_sources() { + assert_eq!( + determine_flow_type_for_source_state("my_flow", &HashMap::new(), false, true).unwrap(), + Some(FlowType::Streaming) + ); + } + #[test] fn test_name_is_match() { assert!(!NAME_PATTERN_REG.is_match("/adaf")); diff --git a/tests/cases/standalone/common/flow/flow_pending.result b/tests/cases/standalone/common/flow/flow_pending.result new file mode 100644 index 0000000000..d6fe01b38a --- /dev/null +++ b/tests/cases/standalone/common/flow/flow_pending.result @@ -0,0 +1,52 @@ +CREATE FLOW pending_without_defer +SINK TO pending_sink +AS SELECT val FROM pending_source; + +Error: 1004(InvalidArguments), Invalid SQL, error: missing source tables for flow 'pending_without_defer'; use WITH (defer_on_missing_source = true) to create a pending flow + +CREATE FLOW pending_with_defer +SINK TO pending_sink +WITH (defer_on_missing_source = true) +AS SELECT val FROM pending_source WHERE val > 10; + +Affected Rows: 0 + +SHOW CREATE FLOW pending_with_defer; + ++--------------------+--------------------------------------------------+ +| Flow | Create Flow | ++--------------------+--------------------------------------------------+ +| pending_with_defer | CREATE FLOW IF NOT EXISTS pending_with_defer | +| | SINK TO public.pending_sink | +| | WITH (defer_on_missing_source = 'true') | +| | AS SELECT val FROM pending_source WHERE val > 10 | ++--------------------+--------------------------------------------------+ + +SELECT + flow_definition, + source_table_ids, + source_table_names, + flownode_ids, + options LIKE '%"defer_on_missing_source":"true"%' AS has_defer_option, + options LIKE '%"flow_type":"batching"%' AS has_flow_type_option +FROM INFORMATION_SCHEMA.FLOWS +WHERE flow_name = 'pending_with_defer'; + ++--------------------------------------------------+------------------+--------------------+--------------+------------------+----------------------+ +| flow_definition | source_table_ids | source_table_names | flownode_ids | has_defer_option | has_flow_type_option | ++--------------------------------------------------+------------------+--------------------+--------------+------------------+----------------------+ +| CREATE FLOW IF NOT EXISTS pending_with_defer | [] | | {} | true | true | +| SINK TO public.pending_sink | | | | | | +| WITH (defer_on_missing_source = 'true') | | | | | | +| AS SELECT val FROM pending_source WHERE val > 10 | | | | | | ++--------------------------------------------------+------------------+--------------------+--------------+------------------+----------------------+ + +DROP FLOW pending_with_defer; + +Affected Rows: 0 + +SELECT flow_name FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name = 'pending_with_defer'; + +++ +++ + diff --git a/tests/cases/standalone/common/flow/flow_pending.sql b/tests/cases/standalone/common/flow/flow_pending.sql new file mode 100644 index 0000000000..498f5b2782 --- /dev/null +++ b/tests/cases/standalone/common/flow/flow_pending.sql @@ -0,0 +1,24 @@ +CREATE FLOW pending_without_defer +SINK TO pending_sink +AS SELECT val FROM pending_source; + +CREATE FLOW pending_with_defer +SINK TO pending_sink +WITH (defer_on_missing_source = true) +AS SELECT val FROM pending_source WHERE val > 10; + +SHOW CREATE FLOW pending_with_defer; + +SELECT + flow_definition, + source_table_ids, + source_table_names, + flownode_ids, + options LIKE '%"defer_on_missing_source":"true"%' AS has_defer_option, + options LIKE '%"flow_type":"batching"%' AS has_flow_type_option +FROM INFORMATION_SCHEMA.FLOWS +WHERE flow_name = 'pending_with_defer'; + +DROP FLOW pending_with_defer; + +SELECT flow_name FROM INFORMATION_SCHEMA.FLOWS WHERE flow_name = 'pending_with_defer'; diff --git a/tests/cases/standalone/common/flow/flow_rebuild.result b/tests/cases/standalone/common/flow/flow_rebuild.result index db2f314b32..bd2bf9c892 100644 --- a/tests/cases/standalone/common/flow/flow_rebuild.result +++ b/tests/cases/standalone/common/flow/flow_rebuild.result @@ -273,6 +273,7 @@ ADMIN FLUSH_FLOW('test_wildcard_basic'); | FLOW_FLUSHED | +-----------------------------------------+ +-- SQLNESS SLEEP 3s SELECT wildcard FROM out_basic; +----------+ diff --git a/tests/cases/standalone/common/flow/flow_rebuild.sql b/tests/cases/standalone/common/flow/flow_rebuild.sql index c86c781d5d..4f30c80ea2 100644 --- a/tests/cases/standalone/common/flow/flow_rebuild.sql +++ b/tests/cases/standalone/common/flow/flow_rebuild.sql @@ -151,6 +151,7 @@ VALUES -- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | ADMIN FLUSH_FLOW('test_wildcard_basic'); +-- SQLNESS SLEEP 3s SELECT wildcard FROM out_basic; -- test again, this time with db restart From 869a584f8af7dd2d9f5f9ba5ff86f7a29857b6e7 Mon Sep 17 00:00:00 2001 From: LFC <990479+MichaelScofield@users.noreply.github.com> Date: Fri, 29 May 2026 15:07:49 +0800 Subject: [PATCH 03/29] ci: add nightly jsonbench test (#7750) Signed-off-by: luofucong --- .github/workflows/nightly-jsonbench.yaml | 162 ++++++++++++++++++ src/datatypes/src/json.rs | 85 +++++---- src/datatypes/src/json/value.rs | 108 +++++++----- src/datatypes/src/types/json_type.rs | 10 +- src/datatypes/src/vectors/json/builder.rs | 4 +- .../standalone/common/types/json/json2.result | 23 ++- .../standalone/common/types/json/json2.sql | 9 + 7 files changed, 315 insertions(+), 86 deletions(-) create mode 100644 .github/workflows/nightly-jsonbench.yaml diff --git a/.github/workflows/nightly-jsonbench.yaml b/.github/workflows/nightly-jsonbench.yaml new file mode 100644 index 0000000000..3667ee26a6 --- /dev/null +++ b/.github/workflows/nightly-jsonbench.yaml @@ -0,0 +1,162 @@ +name: Nightly JSONBench + +on: + schedule: + # Trigger at 00:00(Asia/Shanghai) on every weekday. + - cron: "0 16 * * 0-4" + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + allocate-runner: + name: Allocate runner + if: ${{ github.repository == 'GreptimeTeam/greptimedb' }} + runs-on: ubuntu-latest + outputs: + linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }} + + # The following EC2 resource id will be used for resource releasing. + linux-arm64-ec2-runner-label: ${{ steps.start-linux-arm64-runner.outputs.label }} + linux-arm64-ec2-runner-instance-id: ${{ steps.start-linux-arm64-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Allocate Linux ARM64 runner + uses: ./.github/actions/start-runner + id: start-linux-arm64-runner + with: + runner: ${{ vars.DEFAULT_ARM64_RUNNER }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} + image-id: ${{ vars.EC2_RUNNER_LINUX_ARM64_IMAGE_ID }} + security-group-id: ${{ vars.EC2_RUNNER_SECURITY_GROUP_ID }} + subnet-id: ${{ vars.EC2_RUNNER_SUBNET_ID }} + + jsonbench: + name: Run JSONBench + if: ${{ github.repository == 'GreptimeTeam/greptimedb' }} + needs: [ allocate-runner ] + runs-on: ${{ needs.allocate-runner.outputs.linux-arm64-runner }} + timeout-minutes: 120 + env: + JSONBENCH_DATA_DIR: /home/runner/data/bluesky + JSONBENCH_OUTPUT_PREFIX: _ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - uses: actions-rust-lang/setup-rust-toolchain@v1 + + - name: Rust Cache + uses: Swatinem/rust-cache@v2 + with: + shared-key: "nightly-jsonbench" + cache-all-crates: "true" + save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Build GreptimeDB + run: cargo build --profile nightly --bin greptime + + - name: Reclaim disk space + shell: bash + run: | + set -euo pipefail + + mkdir -p "${RUNNER_TEMP}/greptimedb-bin" + cp ./target/nightly/greptime "${RUNNER_TEMP}/greptimedb-bin/greptime" + chmod +x "${RUNNER_TEMP}/greptimedb-bin/greptime" + + rm -rf ./target + + - name: Run JSONBench + shell: bash + run: | + set -euo pipefail + + cd "${RUNNER_TEMP}" + cp "${RUNNER_TEMP}/greptimedb-bin/greptime" ./greptime + chmod +x ./greptime + + export GREPTIMEDB_STANDALONE__WAL__DIR=greptimedb_data/wal + export GREPTIMEDB_STANDALONE__STORAGE__DATA_HOME=greptimedb_data + export GREPTIMEDB_STANDALONE__LOGGING__DIR=greptimedb_data/logs + export GREPTIMEDB_STANDALONE__LOGGING__APPEND_STDOUT=false + export GREPTIMEDB_STANDALONE__HTTP__BODY_LIMIT=1GB + export GREPTIMEDB_STANDALONE__HTTP__TIMEOUT=500s + + ./greptime standalone start > greptimedb.log 2>&1 & + greptime_pid=$! + trap 'kill "${greptime_pid}" 2>/dev/null || true' EXIT + + until curl -s --fail -o /dev/null http://localhost:4000/health; do + if ! kill -0 "${greptime_pid}" 2>/dev/null; then + cat greptimedb.log + exit 1 + fi + sleep 1 + done + + git clone --branch greptimedb-new-json --depth 1 https://github.com/GreptimeTeam/JSONBench.git JSONBench + cp ./greptime JSONBench/greptimedb/greptime + + cd JSONBench/greptimedb + ./main.sh 3 "${JSONBENCH_DATA_DIR}" success.log error.log "${JSONBENCH_OUTPUT_PREFIX}" false + + - name: Upload JSONBench results + if: always() + uses: actions/upload-artifact@v4 + with: + name: jsonbench-results + path: | + ${{ runner.temp }}/greptimedb.log + ${{ runner.temp }}/JSONBench/greptimedb/*.log + ${{ runner.temp }}/JSONBench/greptimedb/*.total_size + ${{ runner.temp }}/JSONBench/greptimedb/*.data_size + ${{ runner.temp }}/JSONBench/greptimedb/*.index_size + ${{ runner.temp }}/JSONBench/greptimedb/*.count + ${{ runner.temp }}/JSONBench/greptimedb/*.results_runtime + ${{ runner.temp }}/JSONBench/greptimedb/*.query_results + if-no-files-found: ignore + retention-days: 7 + + stop-linux-arm64-runner: + name: Stop Linux ARM64 runner + # It's always run as the last job in the workflow to make sure that the runner is released. + if: ${{ always() }} + runs-on: ubuntu-latest + needs: [ + allocate-runner, + jsonbench, + ] + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Stop Linux ARM64 runner + uses: ./.github/actions/stop-runner + with: + label: ${{ needs.allocate-runner.outputs.linux-arm64-ec2-runner-label }} + ec2-instance-id: ${{ needs.allocate-runner.outputs.linux-arm64-ec2-runner-instance-id }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ vars.EC2_RUNNER_REGION }} + github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} diff --git a/src/datatypes/src/json.rs b/src/datatypes/src/json.rs index db657abbcb..33104084ad 100644 --- a/src/datatypes/src/json.rs +++ b/src/datatypes/src/json.rs @@ -26,12 +26,12 @@ use std::sync::Arc; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value as Json}; -use snafu::{OptionExt, ResultExt, ensure}; +use snafu::{OptionExt, ResultExt}; use crate::error::{self, InvalidJsonSnafu, Result, SerializeSnafu}; use crate::json::value::{JsonValue, JsonVariant}; use crate::types::json_type::{JsonNativeType, JsonNumberType, JsonObjectType}; -use crate::types::{StructField, StructType}; +use crate::types::{JsonType, StructField, StructType}; use crate::value::{ListValue, StructValue, Value}; /// The configuration of JSON encoding @@ -305,33 +305,47 @@ fn encode_json_array_with_context<'a>( ) -> Result { let json_array_len = json_array.len(); let mut items = Vec::with_capacity(json_array_len); - let mut element_type = item_type.cloned(); for (index, value) in json_array.into_iter().enumerate() { let array_context = context.with_key(&index.to_string()); - let item_value = - encode_json_value_with_context(value, element_type.as_ref(), &array_context)?; - let item_type = item_value.json_type().native_type().clone(); - items.push(item_value.into_variant()); - - // Determine the common type for the list - if let Some(current_type) = &element_type { - // It's valid for json array to have different types of items, for example, - // ["a string", 1]. However, the `JsonValue` will be converted to Arrow list array, - // which requires all items have exactly same type. So we forbid the different types - // case here. Besides, it's not common for items in a json array to differ. So I think - // we are good here. - ensure!( - item_type == *current_type, - error::InvalidJsonSnafu { - value: "all items in json array must have the same type" - } - ); - } else { - element_type = Some(item_type); - } + let item_value = encode_json_value_with_context(value, None, &array_context)?; + items.push(item_value); } + // In specification, it's valid for a JSON array to have different types of items, for example, + // ["a string", 1]. However, in implementation, the `JsonValue` will be converted to Arrow list + // array, which requires all items have exactly the same type. So we merge out the maybe + // different item types to a unified type, and align all the item values to it. + + let provided_item_type = item_type.map(|x| JsonType::new_json2(x.clone())); + let merged_item_type = if let Some((first, rests)) = items.split_first() { + let mut merged = first.json_type().clone(); + for rest in rests.iter().map(|x| x.json_type()) { + if matches!(merged.native_type(), JsonNativeType::Variant) { + break; + } + merged.merge(rest)?; + } + Some(merged) + } else { + None + }; + let unified_item_type = match (provided_item_type, merged_item_type) { + (Some(mut x), Some(y)) => { + x.merge(&y)?; + Some(x) + } + (x, y) => x.or(y), + }; + if let Some(unified_item_type) = unified_item_type { + for item in &mut items { + item.try_align(&unified_item_type)?; + } + } + let items = items + .into_iter() + .map(|x| x.into_variant()) + .collect::>(); Ok(JsonValue::new(JsonVariant::Array(items))) } @@ -1050,11 +1064,8 @@ mod tests { fn test_encode_json_array_mixed_types() { let json = json!([1, "hello", true, 3.15]); let settings = JsonStructureSettings::Structured(None); - let result = settings.encode_with_type(json, None); - assert_eq!( - result.unwrap_err().to_string(), - "Invalid JSON: all items in json array must have the same type" - ); + let value = settings.encode_with_type(json, None).unwrap(); + assert_eq!(value.data_type().to_string(), r#"Json2[""]"#); } #[test] @@ -1276,12 +1287,12 @@ mod tests { #[test] fn test_encode_json_array_with_item_type() { let json = json!([1, 2, 3]); - let item_type = Arc::new(ConcreteDataType::uint64_datatype()); + let item_type = Arc::new(ConcreteDataType::int64_datatype()); let settings = JsonStructureSettings::Structured(None); let result = settings .encode_with_type( json, - Some(&JsonNativeType::Array(Box::new(JsonNativeType::u64()))), + Some(&JsonNativeType::Array(Box::new(JsonNativeType::i64()))), ) .unwrap() .into_json_inner() @@ -1289,9 +1300,9 @@ mod tests { if let Value::List(list_value) = result { assert_eq!(list_value.items().len(), 3); - assert_eq!(list_value.items()[0], Value::UInt64(1)); - assert_eq!(list_value.items()[1], Value::UInt64(2)); - assert_eq!(list_value.items()[2], Value::UInt64(3)); + assert_eq!(list_value.items()[0], Value::Int64(1)); + assert_eq!(list_value.items()[1], Value::Int64(2)); + assert_eq!(list_value.items()[2], Value::Int64(3)); assert_eq!(list_value.datatype(), item_type); } else { panic!("Expected List value"); @@ -2249,10 +2260,10 @@ mod tests { )])), ); - let decoded_struct = settings.decode_struct(array_struct); + let decoded_struct = settings.decode_struct(array_struct).unwrap(); assert_eq!( - decoded_struct.unwrap_err().to_string(), - "Invalid JSON: all items in json array must have the same type" + format!("{decoded_struct:?}"), + r#"StructValue { items: [List(ListValue { items: [Binary(Bytes(b"1")), Binary(Bytes(b"\"hello\"")), Binary(Bytes(b"true")), Binary(Bytes(b"3.15"))], datatype: Binary(BinaryType { repr_type: Binary }) })], fields: StructType { fields: [StructField { name: "value", data_type: List(ListType { item_type: Binary(BinaryType { repr_type: Binary }) }), nullable: true, metadata: {} }] } }"# ); } diff --git a/src/datatypes/src/json/value.rs b/src/datatypes/src/json/value.rs index f3b652a549..4350630003 100644 --- a/src/datatypes/src/json/value.rs +++ b/src/datatypes/src/json/value.rs @@ -65,6 +65,14 @@ impl JsonNumber { JsonNumber::Float(n) => n.0, } } + + fn native_type(&self) -> JsonNativeType { + match self { + JsonNumber::PosInt(_) => JsonNativeType::u64(), + JsonNumber::NegInt(_) => JsonNativeType::i64(), + JsonNumber::Float(_) => JsonNativeType::f64(), + } + } } impl From for JsonNumber { @@ -147,26 +155,14 @@ impl JsonVariant { match self { JsonVariant::Null => JsonNativeType::Null, JsonVariant::Bool(_) => JsonNativeType::Bool, - JsonVariant::Number(n) => match n { - JsonNumber::PosInt(_) => JsonNativeType::u64(), - JsonNumber::NegInt(_) => JsonNativeType::i64(), - JsonNumber::Float(_) => JsonNativeType::f64(), - }, + JsonVariant::Number(n) => n.native_type(), JsonVariant::String(_) => JsonNativeType::String, JsonVariant::Array(array) => { - let item_type = if let Some(first) = array.first() { - first.native_type() - } else { - JsonNativeType::Null - }; - JsonNativeType::Array(Box::new(item_type)) + json_array_native_type(array.iter().map(JsonVariant::native_type)) + } + JsonVariant::Object(object) => { + json_object_native_type(object.iter().map(|(k, v)| (k, v.native_type()))) } - JsonVariant::Object(object) => JsonNativeType::Object( - object - .iter() - .map(|(k, v)| (k.clone(), v.native_type())) - .collect(), - ), JsonVariant::Variant(_) => JsonNativeType::Variant, } } @@ -469,6 +465,7 @@ impl JsonValue { .collect::>()?, ), + (JsonVariant::Object(kvs), _) if kvs.is_empty() => JsonVariant::Null, (JsonVariant::Object(mut kvs), JsonNativeType::Object(expected)) => { ensure!( expected.keys().len() >= kvs.keys().len() @@ -517,7 +514,7 @@ impl JsonValue { let x = std::mem::take(&mut self.json_variant); self.json_variant = helper(x, expected.native_type())?; - self.json_type = OnceLock::from(expected.clone()); + self.json_type = OnceLock::new(); Ok(()) } } @@ -623,35 +620,55 @@ pub enum JsonVariantRef<'a> { } impl JsonVariantRef<'_> { - fn json_type(&self) -> JsonType { - fn native_type(v: &JsonVariantRef<'_>) -> JsonNativeType { - match v { - JsonVariantRef::Null => JsonNativeType::Null, - JsonVariantRef::Bool(_) => JsonNativeType::Bool, - JsonVariantRef::Number(n) => match n { - JsonNumber::PosInt(_) => JsonNativeType::u64(), - JsonNumber::NegInt(_) => JsonNativeType::i64(), - JsonNumber::Float(_) => JsonNativeType::f64(), - }, - JsonVariantRef::String(_) => JsonNativeType::String, - JsonVariantRef::Array(array) => { - let item_type = if let Some(first) = array.first() { - native_type(first) - } else { - JsonNativeType::Null - }; - JsonNativeType::Array(Box::new(item_type)) - } - JsonVariantRef::Object(object) => JsonNativeType::Object( - object - .iter() - .map(|(k, v)| (k.to_string(), native_type(v))) - .collect(), - ), - JsonVariantRef::Variant(_) => JsonNativeType::Variant, + fn native_type(&self) -> JsonNativeType { + match self { + JsonVariantRef::Null => JsonNativeType::Null, + JsonVariantRef::Bool(_) => JsonNativeType::Bool, + JsonVariantRef::Number(n) => n.native_type(), + JsonVariantRef::String(_) => JsonNativeType::String, + JsonVariantRef::Array(array) => { + json_array_native_type(array.iter().map(JsonVariantRef::native_type)) } + JsonVariantRef::Object(object) => { + json_object_native_type(object.iter().map(|(k, v)| (*k, v.native_type()))) + } + JsonVariantRef::Variant(_) => JsonNativeType::Variant, } - JsonType::new_json2(native_type(self)) + } + + fn json_type(&self) -> JsonType { + JsonType::new_json2(self.native_type()) + } +} + +fn json_array_native_type(items: I) -> JsonNativeType +where + I: IntoIterator, +{ + let mut iter = items.into_iter(); + let mut item_type = match iter.next() { + Some(t) => t, + None => return JsonNativeType::Array(Box::new(JsonNativeType::Null)), + }; + for x in iter { + if matches!(item_type, JsonNativeType::Variant) { + break; + } + item_type.merge(&x); + } + JsonNativeType::Array(Box::new(item_type)) +} + +fn json_object_native_type(fields: I) -> JsonNativeType +where + I: IntoIterator, + K: Into, +{ + let mut fields = fields.into_iter().peekable(); + if fields.peek().is_none() { + JsonNativeType::Null + } else { + JsonNativeType::Object(fields.map(|(k, v)| (k.into(), v)).collect()) } } @@ -941,7 +958,6 @@ mod tests { ("name".to_string(), JsonVariant::Null), ]))) ); - assert_eq!(value.json_type(), &expected); // Object alignment should fail if the expected type misses any field from the value. let expected = JsonType::new_json2(JsonNativeType::Object(JsonObjectType::from([( diff --git a/src/datatypes/src/types/json_type.rs b/src/datatypes/src/types/json_type.rs index e8d06543ed..652847da43 100644 --- a/src/datatypes/src/types/json_type.rs +++ b/src/datatypes/src/types/json_type.rs @@ -115,6 +115,14 @@ impl JsonNativeType { (JsonNativeType::Null, that) => that.clone(), (this, JsonNativeType::Null) => this, (this, that) if this == *that => this, + + (JsonNativeType::Number(x), JsonNativeType::Number(y)) => { + JsonNativeType::Number(match (x, y) { + (x, y) if x == *y => x, + (JsonNumberType::F64, _) | (_, JsonNumberType::F64) => JsonNumberType::F64, + _ => JsonNumberType::I64, + }) + } _ => JsonNativeType::Variant, }; } @@ -822,7 +830,7 @@ mod tests { test( "1.5", &mut JsonType::new_json2(JsonNativeType::i64()), - Ok(r#""""#), + Ok(r#""""#), )?; // Object merge should preserve existing fields and append missing fields. diff --git a/src/datatypes/src/vectors/json/builder.rs b/src/datatypes/src/vectors/json/builder.rs index be79a921c7..7ca1ff2f6a 100644 --- a/src/datatypes/src/vectors/json/builder.rs +++ b/src/datatypes/src/vectors/json/builder.rs @@ -89,7 +89,9 @@ impl MutableVector for JsonVectorBuilder { .fail(); }; let json_type = value.json_type(); - self.merged_type.merge(json_type)?; + if !self.merged_type.is_include(json_type) { + self.merged_type.merge(json_type)?; + } let value = JsonValue::new(JsonVariant::from(value.variant().clone())); self.values.push(value); diff --git a/tests/cases/standalone/common/types/json/json2.result b/tests/cases/standalone/common/types/json/json2.result index 71e119307c..7de73f2a78 100644 --- a/tests/cases/standalone/common/types/json/json2.result +++ b/tests/cases/standalone/common/types/json/json2.result @@ -126,7 +126,7 @@ select j.a, j.a.x from json2_table order by ts; | {"b":-2} | | | {"b":3} | | | {"b":-4} | | -| {"b":null} | | +| | | | | | | {"b":"s7"} | | | {"b":8} | | @@ -151,6 +151,14 @@ select j.c, j.y from json2_table order by ts; | | false | +-----------------------------------+-----------------------------------+ +select j from json2_table order by ts; + +Error: 3001(EngineExecuteQuery), Failed to align JSON array, reason: Invalid argument error: use StructArray::try_new_with_length or StructArray::new_empty_fields to create a struct array with no fields so that the length can be set correctly + +select * from json2_table order by ts; + +Error: 3001(EngineExecuteQuery), Failed to align JSON array, reason: Invalid argument error: use StructArray::try_new_with_length or StructArray::new_empty_fields to create a struct array with no fields so that the length can be set correctly + select j.a.b + 1 from json2_table order by ts; +------------------------------------------------------------+ @@ -168,6 +176,19 @@ select j.a.b + 1 from json2_table order by ts; | 11 | +------------------------------------------------------------+ +select abs(j.a.b) from json2_table order by ts; + +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Function 'abs' expects NativeType::Numeric but received NativeType::String No function matches the given name and argument types 'abs(Utf8View)'. You might need to add explicit type casts. + Candidate functions: + abs(Numeric(1)) + +-- "j.c" is of type "String", "abs" is expected to be all "null"s. +select abs(j.c) from json2_table order by ts; + +Error: 3000(PlanQuery), Failed to plan SQL: Error during planning: Function 'abs' expects NativeType::Numeric but received NativeType::String No function matches the given name and argument types 'abs(Utf8View)'. You might need to add explicit type casts. + Candidate functions: + abs(Numeric(1)) + select j.d from json2_table order by ts; +-----------------------------------+ diff --git a/tests/cases/standalone/common/types/json/json2.sql b/tests/cases/standalone/common/types/json/json2.sql index 8dd6789bce..cb8df2f8b9 100644 --- a/tests/cases/standalone/common/types/json/json2.sql +++ b/tests/cases/standalone/common/types/json/json2.sql @@ -46,8 +46,17 @@ select j.a, j.a.x from json2_table order by ts; select j.c, j.y from json2_table order by ts; +select j from json2_table order by ts; + +select * from json2_table order by ts; + select j.a.b + 1 from json2_table order by ts; +select abs(j.a.b) from json2_table order by ts; + +-- "j.c" is of type "String", "abs" is expected to be all "null"s. +select abs(j.c) from json2_table order by ts; + select j.d from json2_table order by ts; drop table json2_table; From ed9312f8e35dc323b4c7052972c9b2e3aa1fa49d Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Mon, 1 Jun 2026 07:51:14 +0800 Subject: [PATCH 04/29] feat: global switch for creating tables automatically (#8203) * feat: global switch for creating table automatically Signed-off-by: Dennis Zhuang * chore: make auto_create_table as comment by default Signed-off-by: Dennis Zhuang * feat: respect gloabl switch for metric engine Signed-off-by: Dennis Zhuang --------- Signed-off-by: Dennis Zhuang --- config/config.md | 2 + config/frontend.example.toml | 4 ++ config/standalone.example.toml | 4 ++ src/cmd/tests/load_config_test.rs | 2 + src/flow/src/server.rs | 4 ++ src/frontend/src/frontend.rs | 6 +++ src/frontend/src/instance/builder.rs | 1 + src/operator/src/insert.rs | 59 +++++++++++++++------ src/standalone/src/options.rs | 6 +++ tests-integration/src/standalone.rs | 9 ++++ tests-integration/src/test_util.rs | 24 +++++++++ tests-integration/tests/grpc.rs | 79 +++++++++++++++++++++++++++- tests-integration/tests/http.rs | 1 + 13 files changed, 185 insertions(+), 16 deletions(-) diff --git a/config/config.md b/config/config.md index 0fae0caaa4..aa0c6701a0 100644 --- a/config/config.md +++ b/config/config.md @@ -14,6 +14,7 @@ | --- | -----| ------- | ----------- | | `default_timezone` | String | Unset | The default timezone of the server. | | `default_column_prefix` | String | Unset | The default column prefix for auto-created time index and value columns. | +| `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.
When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. | | `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd" | | `max_in_flight_write_bytes` | String | Unset | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
Set to 0 to disable the limit. Default: "0" (unlimited) | | `write_bytes_exhausted_policy` | String | Unset | Policy when write bytes quota is exhausted.
Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail" | @@ -230,6 +231,7 @@ | --- | -----| ------- | ----------- | | `default_timezone` | String | Unset | The default timezone of the server. | | `default_column_prefix` | String | Unset | The default column prefix for auto-created time index and value columns. | +| `auto_create_table` | Bool | `true` | Server-side global switch for auto table creation on write.
When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. | | `user_provider` | String | Unset | The user provider for authentication.
Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd" | | `max_in_flight_write_bytes` | String | Unset | Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight).
Set to 0 to disable the limit. Default: "0" (unlimited) | | `write_bytes_exhausted_policy` | String | Unset | Policy when write bytes quota is exhausted.
Options: "wait" (default, 10s timeout), "wait()" (e.g., "wait(30s)"), "fail" | diff --git a/config/frontend.example.toml b/config/frontend.example.toml index 39f38fbef9..a044aebda6 100644 --- a/config/frontend.example.toml +++ b/config/frontend.example.toml @@ -6,6 +6,10 @@ default_timezone = "UTC" ## @toml2docs:none-default default_column_prefix = "greptime" +## Server-side global switch for auto table creation on write. +## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. +#+ auto_create_table = true + ## The user provider for authentication. ## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd" ## @toml2docs:none-default diff --git a/config/standalone.example.toml b/config/standalone.example.toml index d5c42e744c..5740e0e1cf 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -6,6 +6,10 @@ default_timezone = "UTC" ## @toml2docs:none-default default_column_prefix = "greptime" +## Server-side global switch for auto table creation on write. +## When `false`, a missing table is never auto-created even if the request sets the `auto_create_table` hint to `true`. Default: `true`. +#+ auto_create_table = true + ## The user provider for authentication. ## Examples: "static_user_provider:file:/path/to/users", "static_user_provider:cmd:greptime_user=greptime_pwd" ## @toml2docs:none-default diff --git a/src/cmd/tests/load_config_test.rs b/src/cmd/tests/load_config_test.rs index 6cffcd67c2..cee29e4456 100644 --- a/src/cmd/tests/load_config_test.rs +++ b/src/cmd/tests/load_config_test.rs @@ -114,6 +114,7 @@ fn test_load_frontend_example_config() { component: FrontendOptions { default_timezone: Some("UTC".to_string()), default_column_prefix: Some("greptime".to_string()), + auto_create_table: true, meta_client: Some(MetaClientOptions { metasrv_addrs: vec!["127.0.0.1:3002".to_string()], timeout: Duration::from_secs(3), @@ -267,6 +268,7 @@ fn test_load_standalone_example_config() { component: StandaloneOptions { default_timezone: Some("UTC".to_string()), default_column_prefix: Some("greptime".to_string()), + auto_create_table: true, wal: DatanodeWalConfig::RaftEngine(RaftEngineConfig { dir: Some(format!("{}/{}", DEFAULT_DATA_HOME, WAL_DIR)), sync_period: Some(Duration::from_secs(10)), diff --git a/src/flow/src/server.rs b/src/flow/src/server.rs index 913128f386..c2d986f645 100644 --- a/src/flow/src/server.rs +++ b/src/flow/src/server.rs @@ -566,11 +566,15 @@ impl FrontendInvoker { name: TABLE_FLOWNODE_SET_CACHE_NAME, })?; + // TODO(auto_create_table): flow sink tables are created through a controlled + // `CREATE FLOW` path, not client writes, so they are intentionally exempt from + // the frontend's global auto-create switch. Revisit if flow should honor it. let inserter = Arc::new(Inserter::new( catalog_manager.clone(), partition_manager.clone(), node_manager.clone(), table_flownode_cache, + true, )); let deleter = Arc::new(Deleter::new( diff --git a/src/frontend/src/frontend.rs b/src/frontend/src/frontend.rs index fb3b096f06..918185cb8f 100644 --- a/src/frontend/src/frontend.rs +++ b/src/frontend/src/frontend.rs @@ -44,6 +44,11 @@ pub struct FrontendOptions { pub node_id: Option, pub default_timezone: Option, pub default_column_prefix: Option, + /// Server-side global switch for auto table creation on write. + /// Acts as an upper bound: when `false`, missing tables are never auto-created + /// even if a request sets the `auto_create_table` hint to `true`. When `true` + /// (default), the per-request hint still applies. Default: `true`. + pub auto_create_table: bool, /// Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight). /// Set to 0 to disable the limit. Default: "0" (unlimited) pub max_in_flight_write_bytes: ReadableSize, @@ -82,6 +87,7 @@ impl Default for FrontendOptions { node_id: None, default_timezone: None, default_column_prefix: None, + auto_create_table: true, max_in_flight_write_bytes: ReadableSize(0), write_bytes_exhausted_policy: OnExhaustedPolicy::default(), http: HttpOptions::default(), diff --git a/src/frontend/src/instance/builder.rs b/src/frontend/src/instance/builder.rs index ff857ed768..526d8aac73 100644 --- a/src/frontend/src/instance/builder.rs +++ b/src/frontend/src/instance/builder.rs @@ -185,6 +185,7 @@ impl FrontendBuilder { partition_manager.clone(), node_manager.clone(), table_flownode_cache, + self.options.auto_create_table, )); let deleter = Arc::new(Deleter::new( self.catalog_manager.clone(), diff --git a/src/operator/src/insert.rs b/src/operator/src/insert.rs index ff8ed2b78b..9ef4d2ff94 100644 --- a/src/operator/src/insert.rs +++ b/src/operator/src/insert.rs @@ -83,6 +83,10 @@ pub struct Inserter { pub(crate) partition_manager: PartitionRuleManagerRef, pub(crate) node_manager: NodeManagerRef, pub(crate) table_flownode_set_cache: TableFlownodeSetCacheRef, + /// Server-side upper bound for auto table creation on write. + /// When `false`, missing tables are never auto-created regardless of the + /// per-request `auto_create_table` hint. When `true`, the hint still applies. + auto_create_table: bool, } pub type InserterRef = Arc; @@ -135,12 +139,14 @@ impl Inserter { partition_manager: PartitionRuleManagerRef, node_manager: NodeManagerRef, table_flownode_set_cache: TableFlownodeSetCacheRef, + auto_create_table: bool, ) -> Self { Self { catalog_manager, partition_manager, node_manager, table_flownode_set_cache, + auto_create_table, } } @@ -469,6 +475,30 @@ impl Inserter { Ok(inserts) } + /// Returns `None` if auto table creation is allowed, or `Some(reason)` if + /// disabled by either the global config or the request hint. The reason tells + /// which one, for a clearer error. + fn auto_create_disabled_reason(&self, ctx: &QueryContextRef) -> Result> { + let auto_create_table_hint = ctx + .extension(AUTO_CREATE_TABLE_KEY) + .map(|v| v.parse::()) + .transpose() + .map_err(|_| { + InvalidInsertRequestSnafu { + reason: "`auto_create_table` hint must be a boolean", + } + .build() + })? + .unwrap_or(true); + Ok(if !self.auto_create_table { + Some("auto-create table is disabled by frontend config") + } else if !auto_create_table_hint { + Some("`auto_create_table` hint is disabled") + } else { + None + }) + } + /// Creates or alter tables on demand: /// - if table does not exist, create table by inferred CreateExpr /// - if table exist, check if schema matches. If any new column found, alter table by inferred `AlterExpr` @@ -498,19 +528,7 @@ impl Inserter { let schema = ctx.current_schema(); let mut table_infos = HashMap::new(); - // If `auto_create_table` hint is disabled, skip creating/altering tables. - let auto_create_table_hint = ctx - .extension(AUTO_CREATE_TABLE_KEY) - .map(|v| v.parse::()) - .transpose() - .map_err(|_| { - InvalidInsertRequestSnafu { - reason: "`auto_create_table` hint must be a boolean", - } - .build() - })? - .unwrap_or(true); - if !auto_create_table_hint { + if let Some(disabled_reason) = self.auto_create_disabled_reason(ctx)? { let mut instant_table_ids = HashSet::new(); for req in &requests.inserts { let table = self @@ -518,8 +536,8 @@ impl Inserter { .await? .context(InvalidInsertRequestSnafu { reason: format!( - "Table `{}` does not exist, and `auto_create_table` hint is disabled", - req.table_name + "Table `{}` does not exist, and {}", + req.table_name, disabled_reason ), })?; let table_info = table.table_info(); @@ -767,6 +785,16 @@ impl Inserter { return Ok(()); } + // Gate here too, otherwise a disabled switch would still leak the physical table. + if let Some(disabled_reason) = self.auto_create_disabled_reason(ctx)? { + return InvalidInsertRequestSnafu { + reason: format!( + "Physical table `{physical_table}` does not exist, and {disabled_reason}" + ), + } + .fail(); + } + let table_reference = TableReference::full(catalog_name, &schema_name, &physical_table); info!("Physical metric table `{table_reference}` does not exist, try creating table"); @@ -1333,6 +1361,7 @@ mod tests { Cache::new(100), kv_backend.clone(), )), + true, ); let alter_expr = inserter .get_alter_table_expr_on_demand(&mut req, &table, &ctx, true, true) diff --git a/src/standalone/src/options.rs b/src/standalone/src/options.rs index dece6389f0..bf1000fd58 100644 --- a/src/standalone/src/options.rs +++ b/src/standalone/src/options.rs @@ -38,6 +38,10 @@ pub struct StandaloneOptions { pub enable_telemetry: bool, pub default_timezone: Option, pub default_column_prefix: Option, + /// Server-side global switch for auto table creation on write. + /// Upper bound: when `false`, missing tables are never auto-created even if a + /// request sets the `auto_create_table` hint to `true`. Default: `true`. + pub auto_create_table: bool, /// Maximum total memory for all concurrent write request bodies and messages (HTTP, gRPC, Flight). /// Set to 0 to disable the limit. Default: "0" (unlimited) pub max_in_flight_write_bytes: ReadableSize, @@ -77,6 +81,7 @@ impl Default for StandaloneOptions { enable_telemetry: true, default_timezone: None, default_column_prefix: None, + auto_create_table: true, max_in_flight_write_bytes: ReadableSize(0), write_bytes_exhausted_policy: OnExhaustedPolicy::default(), http: HttpOptions::default(), @@ -130,6 +135,7 @@ impl StandaloneOptions { let cloned_opts = self.clone(); FrontendOptions { default_timezone: cloned_opts.default_timezone, + auto_create_table: cloned_opts.auto_create_table, max_in_flight_write_bytes: cloned_opts.max_in_flight_write_bytes, write_bytes_exhausted_policy: cloned_opts.write_bytes_exhausted_policy, http: cloned_opts.http, diff --git a/tests-integration/src/standalone.rs b/tests-integration/src/standalone.rs index b013b8b0d4..74a1501207 100644 --- a/tests-integration/src/standalone.rs +++ b/tests-integration/src/standalone.rs @@ -80,6 +80,7 @@ pub struct GreptimeDbStandaloneBuilder { default_store: Option, plugin: Option, slow_query_options: SlowQueryOptions, + auto_create_table: bool, } impl GreptimeDbStandaloneBuilder { @@ -97,9 +98,16 @@ impl GreptimeDbStandaloneBuilder { threshold: Duration::from_secs(1), ..Default::default() }, + auto_create_table: true, } } + #[must_use] + pub fn with_auto_create_table(mut self, auto_create_table: bool) -> Self { + self.auto_create_table = auto_create_table; + self + } + #[must_use] pub fn with_default_store_type(self, store_type: StorageType) -> Self { Self { @@ -347,6 +355,7 @@ impl GreptimeDbStandaloneBuilder { wal: self.metasrv_wal_config.clone().into(), grpc: GrpcOptions::default().with_server_addr("127.0.0.1:4001"), slow_query: self.slow_query_options.clone(), + auto_create_table: self.auto_create_table, ..StandaloneOptions::default() }; diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs index 15d65c34ea..60e0d9143e 100644 --- a/tests-integration/src/test_util.rs +++ b/tests-integration/src/test_util.rs @@ -649,6 +649,20 @@ pub async fn setup_grpc_server_with_user_provider( setup_grpc_server_with(store_type, name, user_provider, None, None).await } +/// Sets up a gRPC server backed by a standalone instance whose frontend has auto +/// table creation disabled, for testing the server-side global switch. +pub async fn setup_grpc_server_with_auto_create_table_disabled( + store_type: StorageType, + name: &str, +) -> (GreptimeDbStandalone, Arc) { + let instance = GreptimeDbStandaloneBuilder::new(name) + .with_default_store_type(store_type) + .with_auto_create_table(false) + .build() + .await; + setup_grpc_server_for_instance(instance, None, None, None).await +} + pub async fn setup_grpc_server_with( store_type: StorageType, name: &str, @@ -657,7 +671,17 @@ pub async fn setup_grpc_server_with( memory_limiter: Option, ) -> (GreptimeDbStandalone, Arc) { let instance = setup_standalone_instance(name, store_type).await; + setup_grpc_server_for_instance(instance, user_provider, grpc_config, memory_limiter).await +} +/// Builds and starts a gRPC server on top of an already-constructed standalone +/// instance. This is the shared core behind the `setup_grpc_server_*` helpers. +async fn setup_grpc_server_for_instance( + instance: GreptimeDbStandalone, + user_provider: Option, + grpc_config: Option, + memory_limiter: Option, +) -> (GreptimeDbStandalone, Arc) { let runtime: Runtime = RuntimeBuilder::default() .worker_threads(2) .thread_name("grpc-handlers") diff --git a/tests-integration/tests/grpc.rs b/tests-integration/tests/grpc.rs index 0f1112ff4a..d6b96484af 100644 --- a/tests-integration/tests/grpc.rs +++ b/tests-integration/tests/grpc.rs @@ -44,7 +44,8 @@ use servers::request_memory_limiter::ServerMemoryLimiter; use servers::server::Server; use servers::tls::{TlsMode, TlsOption}; use tests_integration::test_util::{ - StorageType, setup_grpc_server, setup_grpc_server_with, setup_grpc_server_with_user_provider, + StorageType, setup_grpc_server, setup_grpc_server_with, + setup_grpc_server_with_auto_create_table_disabled, setup_grpc_server_with_user_provider, }; use tonic::Request; use tonic::metadata::MetadataValue; @@ -82,6 +83,7 @@ macro_rules! grpc_tests { test_invalid_dbname, test_auto_create_table, test_auto_create_table_with_hints, + test_auto_create_table_disabled_by_config, test_otel_arrow_auth, test_insert_and_select, test_dbname, @@ -405,6 +407,81 @@ pub async fn test_auto_create_table_with_hints(store_type: StorageType) { let _ = fe_grpc_server.shutdown().await; } +/// When the frontend global switch disables auto table creation, a write to a +/// missing table must fail even if the request sets `auto_create_table=true`, +/// proving the global config is an upper bound that hints cannot bypass. +pub async fn test_auto_create_table_disabled_by_config(store_type: StorageType) { + let (_db, fe_grpc_server) = setup_grpc_server_with_auto_create_table_disabled( + store_type, + "test_auto_create_table_disabled_by_config", + ) + .await; + let addr = fe_grpc_server.bind_addr().unwrap().to_string(); + + let grpc_client = Client::with_urls(vec![addr]); + let db = Database::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, grpc_client); + + // Plain row insert to a missing table: must fail even with `auto_create_table=true`. + let (host, cpu, mem, ts) = expect_data(); + let request = InsertRequest { + table_name: "demo".to_string(), + columns: vec![host, cpu, mem, ts], + row_count: 4, + }; + let result = db + .insert_with_hints( + InsertRequests { + inserts: vec![request], + }, + &[("auto_create_table", "true")], + ) + .await; + let err = result.unwrap_err().to_string(); + assert!( + err.contains("does not exist") && err.contains("disabled by frontend config"), + "unexpected error: {err}" + ); + + // Metric path (via `physical_table` hint): must also fail without leaking the physical table. + let (host, cpu, mem, ts) = expect_data(); + let request = InsertRequest { + table_name: "demo_metric".to_string(), + columns: vec![host, cpu, mem, ts], + row_count: 4, + }; + let result = db + .insert_with_hints( + InsertRequests { + inserts: vec![request], + }, + &[ + ("auto_create_table", "true"), + ("physical_table", "greptime_physical_table"), + ], + ) + .await; + let err = result.unwrap_err().to_string(); + assert!( + err.contains("does not exist") && err.contains("disabled by frontend config"), + "unexpected error: {err}" + ); + + // The physical table must not have been created before the failure. + let output = db.sql("SHOW TABLES").await.unwrap(); + let record_batches = match output.data { + OutputData::RecordBatches(record_batches) => record_batches, + OutputData::Stream(stream) => RecordBatches::try_collect(stream).await.unwrap(), + OutputData::AffectedRows(_) => unreachable!(), + }; + let tables = record_batches.pretty_print().unwrap(); + assert!( + !tables.contains("greptime_physical_table"), + "physical table leaked despite disabled auto-create:\n{tables}" + ); + + let _ = fe_grpc_server.shutdown().await; +} + fn expect_data() -> (Column, Column, Column, Column) { // testing data: let expected_host_col = Column { diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 7f411cdec2..6e35ed7656 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -1491,6 +1491,7 @@ mem_threshold_on_create = "auto" let expected_toml_str = format!( r#" enable_telemetry = true +auto_create_table = true max_in_flight_write_bytes = "0KiB" write_bytes_exhausted_policy = "wait" init_regions_in_background = false From 28fd796f4eeeb056483d963cd01626600d89fb2a Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 1 Jun 2026 10:48:00 +0800 Subject: [PATCH 05/29] fix(flow): harden incremental read correctness (#8196) * fix(flow): harden incremental read correctness Signed-off-by: discord9 * fix(flow): propagate dirty window options Signed-off-by: discord9 * test: more Signed-off-by: discord9 * chore: test config api Signed-off-by: discord9 * refactor: split gen Signed-off-by: discord9 * chore: per review Signed-off-by: discord9 * fix: allowlist key Signed-off-by: discord9 --------- Signed-off-by: discord9 --- config/config.md | 1 + config/flownode.example.toml | 4 + src/common/meta/src/ddl/create_flow.rs | 9 +- src/common/meta/src/ddl/tests/create_flow.rs | 21 +- src/flow/src/batching_mode.rs | 6 +- src/flow/src/batching_mode/engine.rs | 133 +++++- .../src/batching_mode/incremental_filter.rs | 222 --------- src/flow/src/batching_mode/state.rs | 22 +- src/flow/src/batching_mode/task.rs | 435 ++++++++++++------ src/flow/src/batching_mode/task/inc.rs | 25 +- src/flow/src/batching_mode/task/test.rs | 316 ++++++++++++- src/flow/src/batching_mode/utils/test.rs | 7 +- src/operator/src/statement/ddl.rs | 32 +- tests-integration/tests/http.rs | 1 + .../common/flow/flow_incremental_aggr.result | 32 +- .../common/flow/flow_incremental_aggr.sql | 26 +- .../flow/flow_incremental_memtable.result | 4 +- .../common/flow/flow_incremental_memtable.sql | 4 +- .../flow/flow_incremental_partitioned.result | 4 +- .../flow/flow_incremental_partitioned.sql | 4 +- .../common/flow/show_create_flow.result | 2 +- 21 files changed, 873 insertions(+), 437 deletions(-) delete mode 100644 src/flow/src/batching_mode/incremental_filter.rs diff --git a/config/config.md b/config/config.md index aa0c6701a0..d9cffaf122 100644 --- a/config/config.md +++ b/config/config.md @@ -630,6 +630,7 @@ | `flow.batching_mode.experimental_frontend_scan_timeout` | String | `30s` | Flow wait for available frontend timeout,
if failed to find available frontend after frontend_scan_timeout elapsed, return error
which prevent flownode from starting | | `flow.batching_mode.experimental_max_filter_num_per_query` | Integer | `20` | Maximum number of filters allowed in a single query | | `flow.batching_mode.experimental_time_window_merge_threshold` | Integer | `3` | Time window merge distance | +| `flow.batching_mode.experimental_enable_incremental_read` | Bool | `false` | Whether to enable experimental flow incremental source reads.
When disabled, batching flows always execute full-snapshot queries.
Can be overridden per flow with WITH (experimental_enable_incremental_read = 'true'). | | `flow.batching_mode.read_preference` | String | `Leader` | Read preference of the Frontend client. | | `flow.batching_mode.frontend_tls` | -- | -- | -- | | `flow.batching_mode.frontend_tls.enabled` | Bool | `false` | Whether to enable TLS for client. | diff --git a/config/flownode.example.toml b/config/flownode.example.toml index 2c053e6e8c..ff8a9e4a50 100644 --- a/config/flownode.example.toml +++ b/config/flownode.example.toml @@ -31,6 +31,10 @@ node_id = 14 #+experimental_max_filter_num_per_query=20 ## Time window merge distance #+experimental_time_window_merge_threshold=3 +## Whether to enable experimental flow incremental source reads. +## When disabled, batching flows always execute full-snapshot queries. +## Can be overridden per flow with WITH (experimental_enable_incremental_read = 'true'). +#+experimental_enable_incremental_read=false ## Read preference of the Frontend client. #+read_preference="Leader" [flow.batching_mode.frontend_tls] diff --git a/src/common/meta/src/ddl/create_flow.rs b/src/common/meta/src/ddl/create_flow.rs index ddfb0c0759..8a419176c9 100644 --- a/src/common/meta/src/ddl/create_flow.rs +++ b/src/common/meta/src/ddl/create_flow.rs @@ -437,11 +437,13 @@ pub fn defer_on_missing_source(flow_task: &CreateFlowTask) -> Result { pub fn validate_flow_options(flow_task: &CreateFlowTask) -> Result<()> { for key in flow_task.flow_options.keys() { match key.as_str() { - DEFER_ON_MISSING_SOURCE_KEY | FlowType::FLOW_TYPE_KEY => {} + DEFER_ON_MISSING_SOURCE_KEY + | FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY + | FlowType::FLOW_TYPE_KEY => {} unknown => { return UnexpectedSnafu { err_msg: format!( - "Unknown flow option '{unknown}', supported user options: {DEFER_ON_MISSING_SOURCE_KEY}" + "Unknown flow option '{unknown}', supported user options: {DEFER_ON_MISSING_SOURCE_KEY}, {FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY}" ), } .fail(); @@ -487,6 +489,9 @@ pub enum FlowType { Streaming, } +pub const FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY: &str = + "experimental_enable_incremental_read"; + impl FlowType { pub const BATCHING: &str = "batching"; pub const STREAMING: &str = "streaming"; diff --git a/src/common/meta/src/ddl/tests/create_flow.rs b/src/common/meta/src/ddl/tests/create_flow.rs index a1a6c040f1..7150be39cb 100644 --- a/src/common/meta/src/ddl/tests/create_flow.rs +++ b/src/common/meta/src/ddl/tests/create_flow.rs @@ -24,8 +24,9 @@ use table::table_name::TableName; use crate::ddl::DdlContext; use crate::ddl::create_flow::{ - CreateFlowData, CreateFlowProcedure, CreateFlowState, DEFER_ON_MISSING_SOURCE_KEY, FlowType, - defer_on_missing_source, + CreateFlowData, CreateFlowProcedure, CreateFlowState, DEFER_ON_MISSING_SOURCE_KEY, + FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType, defer_on_missing_source, + validate_flow_options, }; use crate::ddl::test_util::create_table::test_create_table_task; use crate::ddl::test_util::flownode_handler::NaiveFlownodeHandler; @@ -275,6 +276,22 @@ fn test_defer_on_missing_source_invalid_value() { ); } +#[test] +fn test_validate_flow_options_allows_incremental_read_option() { + let mut task = test_create_flow_task( + "my_flow", + vec![], + TableName::new(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, "my_sink_table"), + false, + ); + task.flow_options.insert( + FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(), + "true".to_string(), + ); + + validate_flow_options(&task).unwrap(); +} + #[tokio::test] async fn test_create_flow_rejects_unknown_option_in_meta_task() { let mut task = test_create_flow_task( diff --git a/src/flow/src/batching_mode.rs b/src/flow/src/batching_mode.rs index 580762a142..a8bd139d98 100644 --- a/src/flow/src/batching_mode.rs +++ b/src/flow/src/batching_mode.rs @@ -23,7 +23,6 @@ use session::ReadPreference; mod checkpoint; pub(crate) mod engine; pub(crate) mod frontend_client; -mod incremental_filter; mod state; mod table_creator; mod task; @@ -55,6 +54,10 @@ pub struct BatchingModeOptions { pub experimental_max_filter_num_per_query: usize, /// Time window merge distance pub experimental_time_window_merge_threshold: usize, + /// Whether to enable experimental flow incremental source reads. + /// + /// When disabled, batching flows always execute full-snapshot queries. + pub experimental_enable_incremental_read: bool, /// Read preference of the Frontend client. pub read_preference: ReadPreference, /// TLS option for client connections to frontends. @@ -72,6 +75,7 @@ impl Default for BatchingModeOptions { experimental_frontend_scan_timeout: Duration::from_secs(30), experimental_max_filter_num_per_query: 20, experimental_time_window_merge_threshold: 3, + experimental_enable_incremental_read: false, read_preference: Default::default(), frontend_tls: None, } diff --git a/src/flow/src/batching_mode/engine.rs b/src/flow/src/batching_mode/engine.rs index f37e54d80b..68fb3793e4 100644 --- a/src/flow/src/batching_mode/engine.rs +++ b/src/flow/src/batching_mode/engine.rs @@ -21,7 +21,7 @@ use std::time::Duration; use api::v1::flow::DirtyWindowRequests; use catalog::CatalogManagerRef; use common_error::ext::BoxedError; -use common_meta::ddl::create_flow::FlowType; +use common_meta::ddl::create_flow::{FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType}; use common_meta::key::TableMetadataManagerRef; use common_meta::key::flow::FlowMetadataManagerRef; use common_meta::key::flow::flow_state::FlowStat; @@ -38,6 +38,7 @@ use session::context::QueryContext; use snafu::{OptionExt, ResultExt, ensure}; use sql::parsers::utils::is_tql; use store_api::metric_engine_consts::is_metric_engine_internal_column; +use store_api::mito_engine_options::APPEND_MODE_KEY; use store_api::storage::{RegionId, TableId}; use table::table_reference::TableReference; use tokio::sync::{RwLock, oneshot}; @@ -428,6 +429,55 @@ async fn get_table_info( } impl BatchingEngine { + fn batch_opts_for_flow_options( + &self, + flow_options: &HashMap, + ) -> Result, Error> { + let mut batch_opts = (*self.batch_opts).clone(); + if let Some(enable_incremental_read) = + flow_options.get(FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY) + { + batch_opts.experimental_enable_incremental_read = enable_incremental_read + .parse::() + .map_err(|_| { + InvalidQuerySnafu { + reason: format!( + "Invalid flow option {FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY}: {enable_incremental_read}" + ), + } + .build() + })?; + } + + Ok(Arc::new(batch_opts)) + } + + fn table_options_enable_append_mode(extra_options: &HashMap) -> bool { + extra_options + .get(APPEND_MODE_KEY) + .is_some_and(|value| value.eq_ignore_ascii_case("true")) + } + + fn ensure_incremental_source_append_only( + batch_opts: &BatchingModeOptions, + table_name: &[String; 3], + extra_options: &HashMap, + ) -> Result<(), Error> { + if batch_opts.experimental_enable_incremental_read { + ensure!( + Self::table_options_enable_append_mode(extra_options), + UnsupportedSnafu { + reason: format!( + "Flow incremental read requires append-only source table, but source table `{}` is not append-only. Consider setting append_mode='true' on the source table or disabling experimental_enable_incremental_read", + table_name.join(".") + ), + } + ); + } + + Ok(()) + } + pub async fn create_flow_inner(&self, args: CreateFlowArgs) -> Result, Error> { let CreateFlowArgs { flow_id, @@ -494,6 +544,8 @@ impl BatchingEngine { } ); + let batch_opts = self.batch_opts_for_flow_options(&flow_options)?; + let mut source_table_names = Vec::with_capacity(2); for src_id in source_table_ids { // also check table option to see if ttl!=instant @@ -509,6 +561,11 @@ impl BatchingEngine { ), } ); + Self::ensure_incremental_source_append_only( + &batch_opts, + &table_name, + &table_info.table_info.meta.options.extra_options, + )?; source_table_names.push(table_name); } @@ -563,7 +620,7 @@ impl BatchingEngine { query_ctx, catalog_manager: self.catalog_manager.clone(), shutdown_rx: rx, - batch_opts: self.batch_opts.clone(), + batch_opts, flow_eval_interval: eval_interval.map(|secs| Duration::from_secs(secs as u64)), }; @@ -808,7 +865,7 @@ impl BatchingEngine { }); let res = task - .gen_exec_once( + .execute_once_serialized( &self.query_engine, &self.frontend_client, cur_dirty_window_cnt, @@ -946,6 +1003,76 @@ mod tests { ) } + #[tokio::test] + async fn test_flow_option_overrides_incremental_read_switch() { + let engine = new_test_engine().await; + + let default_opts = engine.batch_opts_for_flow_options(&HashMap::new()).unwrap(); + assert!(!default_opts.experimental_enable_incremental_read); + + let enabled_opts = engine + .batch_opts_for_flow_options(&HashMap::from([( + FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(), + "true".to_string(), + )])) + .unwrap(); + assert!(enabled_opts.experimental_enable_incremental_read); + } + + #[test] + fn test_table_options_enable_append_mode() { + assert!(!BatchingEngine::table_options_enable_append_mode( + &HashMap::new() + )); + assert!(!BatchingEngine::table_options_enable_append_mode( + &HashMap::from([(APPEND_MODE_KEY.to_string(), "false".to_string())]) + )); + assert!(BatchingEngine::table_options_enable_append_mode( + &HashMap::from([(APPEND_MODE_KEY.to_string(), "TRUE".to_string())]) + )); + } + + #[test] + fn test_incremental_source_append_only_enforcement() { + let table_name = [ + "greptime".to_string(), + "public".to_string(), + "numbers".to_string(), + ]; + let disabled_opts = BatchingModeOptions::default(); + let enabled_opts = BatchingModeOptions { + experimental_enable_incremental_read: true, + ..Default::default() + }; + let non_append_options = HashMap::new(); + let append_options = HashMap::from([(APPEND_MODE_KEY.to_string(), "true".to_string())]); + + BatchingEngine::ensure_incremental_source_append_only( + &disabled_opts, + &table_name, + &non_append_options, + ) + .expect("disabled incremental read should not require append-only source"); + BatchingEngine::ensure_incremental_source_append_only( + &enabled_opts, + &table_name, + &append_options, + ) + .expect("append-only source should be accepted when incremental read is enabled"); + + let err = BatchingEngine::ensure_incremental_source_append_only( + &enabled_opts, + &table_name, + &non_append_options, + ) + .expect_err("non-append source should be rejected when incremental read is enabled"); + assert!( + err.to_string() + .contains("Flow incremental read requires append-only source table"), + "{err}" + ); + } + async fn new_test_task(flow_id: FlowId) -> (BatchingTask, oneshot::Sender<()>) { let query_engine = create_test_query_engine(); let ctx = QueryContext::arc(); diff --git a/src/flow/src/batching_mode/incremental_filter.rs b/src/flow/src/batching_mode/incremental_filter.rs deleted file mode 100644 index ddc58d0378..0000000000 --- a/src/flow/src/batching_mode/incremental_filter.rs +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use common_telemetry::tracing::debug; -use datafusion_expr::Expr; -use datatypes::schema::Schema; - -use crate::batching_mode::state::FilterExprInfo; -use crate::batching_mode::utils::IncrementalAggregateAnalysis; -use crate::{Error, FlowId}; - -pub(super) fn build_sink_dirty_time_window_filter_expr( - flow_id: FlowId, - analysis: &IncrementalAggregateAnalysis, - sink_schema: &Schema, - dirty_filter: Option<&FilterExprInfo>, -) -> Result, Error> { - let Some(dirty_filter) = dirty_filter else { - return Ok(None); - }; - - let Some(sink_filter_col) = - infer_sink_time_window_filter_col(flow_id, analysis, sink_schema, dirty_filter) - else { - return Ok(None); - }; - - dirty_filter.predicate_for_col(&sink_filter_col) -} - -fn infer_sink_time_window_filter_col( - flow_id: FlowId, - analysis: &IncrementalAggregateAnalysis, - sink_schema: &Schema, - dirty_filter: &FilterExprInfo, -) -> Option { - if analysis.group_key_names.is_empty() { - return None; - } - - let is_timestamp_group_key = |name: &str| { - analysis.group_key_names.iter().any(|key| key == name) - && sink_schema - .column_schema_by_name(name) - .is_some_and(|col| col.data_type.is_timestamp()) - }; - - if is_timestamp_group_key(&dirty_filter.col_name) { - return Some(dirty_filter.col_name.clone()); - } - - let candidates = analysis - .group_key_names - .iter() - .filter(|name| is_timestamp_group_key(name)) - .cloned() - .collect::>(); - - match candidates.as_slice() { - [name] => Some(name.clone()), - [] => { - debug!( - "Flow {} cannot infer sink dirty-window filter column: no timestamp group key in {:?}", - flow_id, analysis.group_key_names - ); - None - } - _ => { - debug!( - "Flow {} cannot infer sink dirty-window filter column: ambiguous timestamp group keys {:?}", - flow_id, candidates - ); - None - } - } -} - -#[cfg(test)] -mod test { - use datatypes::prelude::ConcreteDataType; - use datatypes::schema::ColumnSchema; - use pretty_assertions::assert_eq; - - use super::*; - use crate::adapter::AUTO_CREATED_UPDATE_AT_TS_COL; - use crate::batching_mode::state::FilterExprInfo; - use crate::batching_mode::utils::IncrementalAggregateAnalysis; - - fn test_analysis_with_group_keys(group_key_names: Vec<&str>) -> IncrementalAggregateAnalysis { - IncrementalAggregateAnalysis { - group_key_names: group_key_names - .into_iter() - .map(|name| name.to_string()) - .collect(), - merge_columns: vec![], - literal_columns: vec![], - output_field_names: vec![], - unsupported_exprs: vec![], - } - } - - fn test_dirty_filter(col_name: &str) -> FilterExprInfo { - FilterExprInfo { - expr: datafusion_expr::col(col_name), - col_name: col_name.to_string(), - time_ranges: vec![], - window_size: chrono::Duration::seconds(1), - } - } - - fn test_sink_schema(columns: Vec<(&str, ConcreteDataType)>) -> Schema { - Schema::new( - columns - .into_iter() - .map(|(name, data_type)| ColumnSchema::new(name, data_type, true)) - .collect(), - ) - } - - #[test] - fn test_infer_sink_time_window_filter_col_uses_matching_source_group_key() { - let analysis = test_analysis_with_group_keys(vec!["ts", "host"]); - let sink_schema = test_sink_schema(vec![ - ("ts", ConcreteDataType::timestamp_millisecond_datatype()), - ("host", ConcreteDataType::string_datatype()), - ]); - let dirty_filter = test_dirty_filter("ts"); - - assert_eq!( - Some("ts".to_string()), - infer_sink_time_window_filter_col(1, &analysis, &sink_schema, &dirty_filter) - ); - } - - #[test] - fn test_infer_sink_time_window_filter_col_uses_unique_timestamp_group_key() { - let analysis = test_analysis_with_group_keys(vec!["host", "time_window"]); - let sink_schema = test_sink_schema(vec![ - ("host", ConcreteDataType::string_datatype()), - ( - "time_window", - ConcreteDataType::timestamp_millisecond_datatype(), - ), - ( - AUTO_CREATED_UPDATE_AT_TS_COL, - ConcreteDataType::timestamp_millisecond_datatype(), - ), - ]); - let dirty_filter = test_dirty_filter("ts"); - - assert_eq!( - Some("time_window".to_string()), - infer_sink_time_window_filter_col(1, &analysis, &sink_schema, &dirty_filter) - ); - } - - #[test] - fn test_infer_sink_time_window_filter_col_skips_global_aggregate() { - let analysis = test_analysis_with_group_keys(vec![]); - let sink_schema = test_sink_schema(vec![ - ("number", ConcreteDataType::uint32_datatype()), - ( - "time_window", - ConcreteDataType::timestamp_millisecond_datatype(), - ), - ]); - let dirty_filter = test_dirty_filter("ts"); - - assert_eq!( - None, - infer_sink_time_window_filter_col(1, &analysis, &sink_schema, &dirty_filter) - ); - } - - #[test] - fn test_infer_sink_time_window_filter_col_skips_without_timestamp_group_key() { - let analysis = test_analysis_with_group_keys(vec!["host", "device"]); - let sink_schema = test_sink_schema(vec![ - ("host", ConcreteDataType::string_datatype()), - ("device", ConcreteDataType::string_datatype()), - ( - AUTO_CREATED_UPDATE_AT_TS_COL, - ConcreteDataType::timestamp_millisecond_datatype(), - ), - ]); - let dirty_filter = test_dirty_filter("ts"); - - assert_eq!( - None, - infer_sink_time_window_filter_col(1, &analysis, &sink_schema, &dirty_filter) - ); - } - - #[test] - fn test_infer_sink_time_window_filter_col_skips_ambiguous_timestamp_group_keys() { - let analysis = test_analysis_with_group_keys(vec!["ts", "time_window"]); - let sink_schema = test_sink_schema(vec![ - ("ts", ConcreteDataType::timestamp_millisecond_datatype()), - ( - "time_window", - ConcreteDataType::timestamp_millisecond_datatype(), - ), - ]); - let dirty_filter = test_dirty_filter("source_ts"); - - assert_eq!( - None, - infer_sink_time_window_filter_col(1, &analysis, &sink_schema, &dirty_filter) - ); - } -} diff --git a/src/flow/src/batching_mode/state.rs b/src/flow/src/batching_mode/state.rs index 42b71a4ec7..c5fcc74143 100644 --- a/src/flow/src/batching_mode/state.rs +++ b/src/flow/src/batching_mode/state.rs @@ -66,12 +66,20 @@ pub struct TaskState { } impl TaskState { pub fn new(query_ctx: QueryContextRef, shutdown_rx: oneshot::Receiver<()>) -> Self { + Self::with_dirty_time_windows(query_ctx, shutdown_rx, DirtyTimeWindows::default()) + } + + pub fn with_dirty_time_windows( + query_ctx: QueryContextRef, + shutdown_rx: oneshot::Receiver<()>, + dirty_time_windows: DirtyTimeWindows, + ) -> Self { Self { query_ctx, last_update_time: Instant::now(), last_query_duration: Duration::from_secs(0), last_exec_time_millis: None, - dirty_time_windows: Default::default(), + dirty_time_windows, checkpoint_mode: CheckpointMode::FullSnapshot, checkpoints: Default::default(), incremental_disabled: false, @@ -264,6 +272,16 @@ impl DirtyTimeWindows { time_window_merge_threshold, } } + + #[cfg(test)] + pub(crate) fn max_filter_num_per_query(&self) -> usize { + self.max_filter_num_per_query + } + + #[cfg(test)] + pub(crate) fn time_window_merge_threshold(&self) -> usize { + self.time_window_merge_threshold + } } impl Default for DirtyTimeWindows { @@ -681,7 +699,7 @@ impl DirtyTimeWindows { } } -fn to_df_literal(value: Timestamp) -> Result { +pub(crate) fn to_df_literal(value: Timestamp) -> Result { let value = Value::from(value); let value = value .try_to_scalar_value(&value.data_type()) diff --git a/src/flow/src/batching_mode/task.rs b/src/flow/src/batching_mode/task.rs index 3cdf7899a6..cbd6a05cc2 100644 --- a/src/flow/src/batching_mode/task.rs +++ b/src/flow/src/batching_mode/task.rs @@ -27,7 +27,7 @@ use datafusion::datasource::DefaultTableSource; use datafusion::sql::unparser::expr_to_sql; use datafusion_common::DFSchemaRef; use datafusion_common::tree_node::{Transformed, TreeNode}; -use datafusion_expr::{DmlStatement, LogicalPlan, WriteOp}; +use datafusion_expr::{DmlStatement, LogicalPlan, WriteOp, col, lit}; use datatypes::schema::Schema; use query::QueryEngineRef; use query::options::FLOW_INCREMENTAL_MODE; @@ -38,14 +38,16 @@ use sql::parsers::utils::is_tql; use store_api::mito_engine_options::MERGE_MODE_KEY; use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan}; use table::table::adapter::DfTableProviderAdapter; -use tokio::sync::oneshot; use tokio::sync::oneshot::error::TryRecvError; +use tokio::sync::{Mutex, oneshot}; use tokio::time::Instant; use crate::batching_mode::BatchingModeOptions; use crate::batching_mode::checkpoint::checkpoint_mode_label; use crate::batching_mode::frontend_client::{FrontendClient, PeerDesc}; -use crate::batching_mode::state::{CheckpointMode, DirtyTimeWindows, FilterExprInfo, TaskState}; +use crate::batching_mode::state::{ + CheckpointMode, DirtyTimeWindows, FilterExprInfo, TaskState, to_df_literal, +}; use crate::batching_mode::table_creator::{QueryType, create_table_with_expr}; use crate::batching_mode::time_window::TimeWindowExpr; use crate::batching_mode::utils::{ @@ -67,12 +69,6 @@ use crate::{Error, FlowId}; mod ckpt; mod inc; -/// Maximum number of dirty time-window predicates attached to one incremental -/// SQL query. This keeps generated OR filters bounded so Substrait encoding and -/// downstream planning remain predictable; if the backlog is larger, the flow -/// drains one capped batch and postpones checkpoint advancement to a later run. -const MAX_INCREMENTAL_DIRTY_WINDOW_FILTERS: usize = 4096; - /// The task's config, immutable once created #[derive(Clone)] pub struct TaskConfig { @@ -113,6 +109,10 @@ fn is_merge_mode_last_non_null(options: &HashMap) -> bool { pub struct BatchingTask { pub config: Arc, pub state: Arc>, + /// Serializes plan generation, execution, checkpoint advancement, and dirty + /// window restoration for this flow. Without this, a manual flush and the + /// background loop can process the same checkpoint range concurrently. + execution_lock: Arc>, } /// Arguments for creating batching task @@ -150,6 +150,16 @@ pub enum DirtyRestore { Unscoped(DirtyTimeWindows), } +struct ExecuteOnceOutcome { + new_query: Option, + /// Execution result of the generated insert plan. + /// + /// `Ok(Some((affected_rows, elapsed)))` means a query was executed. + /// `Ok(None)` means no query was generated because there was no dirty signal. + /// `Err(_)` means plan generation or execution failed. + result: Result, Error>, +} + impl BatchingTask { #[allow(clippy::too_many_arguments)] pub fn try_new( @@ -168,6 +178,18 @@ impl BatchingTask { flow_eval_interval, }: TaskArgs<'_>, ) -> Result { + let mut state = TaskState::with_dirty_time_windows( + query_ctx.clone(), + shutdown_rx, + DirtyTimeWindows::new( + batch_opts.experimental_max_filter_num_per_query, + batch_opts.experimental_time_window_merge_threshold, + ), + ); + if !batch_opts.experimental_enable_incremental_read { + state.disable_incremental(); + } + Ok(Self { config: Arc::new(TaskConfig { flow_id, @@ -182,7 +204,8 @@ impl BatchingTask { batch_opts, flow_eval_interval, }), - state: Arc::new(RwLock::new(TaskState::new(query_ctx, shutdown_rx))), + state: Arc::new(RwLock::new(state)), + execution_lock: Arc::new(Mutex::new(())), }) } @@ -251,40 +274,75 @@ impl BatchingTask { .context(ExternalSnafu) } - pub async fn gen_exec_once( + pub(crate) async fn execute_once_serialized( &self, engine: &QueryEngineRef, frontend_client: &Arc, max_window_cnt: Option, ) -> Result, Error> { - if let Some(new_query) = self.gen_insert_plan(engine, max_window_cnt).await? { + let outcome = self + .execute_once_serialized_with_outcome(engine, frontend_client, max_window_cnt) + .await; + outcome.result + } + + /// Executes one flow evaluation under `execution_lock` and keeps the + /// generated query context for the background loop's error logging/backoff. + async fn execute_once_serialized_with_outcome( + &self, + engine: &QueryEngineRef, + frontend_client: &Arc, + max_window_cnt: Option, + ) -> ExecuteOnceOutcome { + let _execution_guard = self.execution_lock.lock().await; + self.execute_once_unlocked(engine, frontend_client, max_window_cnt) + .await + } + + /// Executes one flow evaluation. Caller must hold `execution_lock`. + async fn execute_once_unlocked( + &self, + engine: &QueryEngineRef, + frontend_client: &Arc, + max_window_cnt: Option, + ) -> ExecuteOnceOutcome { + let new_query = match self.gen_insert_plan_unlocked(engine, max_window_cnt).await { + Ok(new_query) => new_query, + Err(err) => { + return ExecuteOnceOutcome { + new_query: None, + result: Err(err), + }; + } + }; + + if let Some(new_query) = new_query { debug!("Generate new query: {}", new_query.plan); - let dirty_filter = match &new_query.dirty_restore { - DirtyRestore::Scoped(f) => Some(f), - _ => None, - }; - match self - .execute_logical_plan( + let res = self + .execute_logical_plan_unlocked( frontend_client, &new_query.plan, - dirty_filter, new_query.can_advance_checkpoints, ) - .await - { - Ok(result) => Ok(result), - Err(err) => { - self.handle_executed_query_failure(Some(&new_query)); - Err(err) - } + .await; + if res.is_err() { + self.handle_executed_query_failure(Some(&new_query)); + } + ExecuteOnceOutcome { + new_query: Some(new_query), + result: res, } } else { debug!("Generate no query"); - Ok(None) + ExecuteOnceOutcome { + new_query: None, + result: Ok(None), + } } } - pub async fn gen_insert_plan( + /// Generates the insert plan. Caller must reach this through the serialized path. + async fn gen_insert_plan_unlocked( &self, engine: &QueryEngineRef, max_window_cnt: Option, @@ -388,11 +446,11 @@ impl BatchingTask { Ok(()) } - pub async fn execute_logical_plan( + /// Executes the insert plan. Caller must reach this through the serialized path. + async fn execute_logical_plan_unlocked( &self, frontend_client: &Arc, plan: &LogicalPlan, - dirty_filter: Option<&FilterExprInfo>, can_advance_checkpoints: bool, ) -> Result, Error> { let instant = Instant::now(); @@ -426,8 +484,7 @@ impl BatchingTask { // For incremental-mode SQL queries, attempt to rewrite the delta aggregate // plan into a safe delta-LEFT-JOIN-sink form before deciding on extensions. let incremental_plan = if can_advance_checkpoints { - self.prepare_plan_for_incremental(&plan, dirty_filter) - .await? + self.prepare_plan_for_incremental(&plan).await? } else { None }; @@ -580,6 +637,112 @@ impl BatchingTask { }) } + fn restore_unscoped_dirty_windows(&self, dirty_windows: &DirtyTimeWindows) { + self.state + .write() + .unwrap() + .dirty_time_windows + .add_dirty_windows(dirty_windows); + } + + fn restore_unscoped_dirty_windows_on_err( + &self, + dirty_windows: &DirtyTimeWindows, + result: Result, + ) -> Result { + result.inspect_err(|_| { + self.restore_unscoped_dirty_windows(dirty_windows); + }) + } + + fn drain_dirty_windows_signal(&self) -> (bool, DirtyTimeWindows) { + let mut state = self.state.write().unwrap(); + let dirty_windows_to_restore = state.dirty_time_windows.clone(); + let is_dirty = !dirty_windows_to_restore.is_empty(); + state.dirty_time_windows.clean(); + (is_dirty, dirty_windows_to_restore) + } + + #[allow(clippy::too_many_arguments)] + async fn gen_unfiltered_plan_info( + &self, + engine: QueryEngineRef, + query_ctx: QueryContextRef, + sink_table_schema: Arc, + primary_key_indices: &[usize], + allow_partial: bool, + dirty_windows_to_restore: DirtyTimeWindows, + retention_filter: Option<(&str, Timestamp, &'static str)>, + ) -> Result { + let mut plan = self.restore_unscoped_dirty_windows_on_err( + &dirty_windows_to_restore, + gen_plan_with_matching_schema( + &self.config.query, + query_ctx, + engine, + sink_table_schema, + primary_key_indices, + allow_partial, + ) + .await, + )?; + + if let Some((col_name, lower_bound, context)) = retention_filter { + let lower = self.restore_unscoped_dirty_windows_on_err( + &dirty_windows_to_restore, + to_df_literal(lower_bound), + )?; + let retention_filter = col(col_name).gt_eq(lit(lower)); + let mut add_filter = AddFilterRewriter::new(retention_filter); + plan = self.restore_unscoped_dirty_windows_on_err( + &dirty_windows_to_restore, + plan.clone() + .rewrite(&mut add_filter) + .with_context(|_| DatafusionSnafu { + context: format!( + "Failed to apply {context} expire_after filter to plan:\n {}\n", + plan + ), + }) + .map(|rewrite| rewrite.data), + )?; + } + + Ok(PlanInfo { + plan, + dirty_restore: DirtyRestore::Unscoped(dirty_windows_to_restore), + can_advance_checkpoints: true, + }) + } + + async fn gen_unfiltered_plan_info_if_dirty( + &self, + engine: QueryEngineRef, + query_ctx: QueryContextRef, + sink_table_schema: Arc, + primary_key_indices: &[usize], + allow_partial: bool, + retention_filter: Option<(&str, Timestamp, &'static str)>, + ) -> Result, Error> { + let (is_dirty, dirty_windows_to_restore) = self.drain_dirty_windows_signal(); + if !is_dirty { + debug!("Flow id={:?}, no new data, not update", self.config.flow_id); + return Ok(None); + } + + self.gen_unfiltered_plan_info( + engine, + query_ctx, + sink_table_schema, + primary_key_indices, + allow_partial, + dirty_windows_to_restore, + retention_filter, + ) + .await + .map(Some) + } + fn handle_executed_query_failure(&self, query: Option<&PlanInfo>) { if let Some(query) = query { self.restore_dirty_windows_after_failure(query); @@ -626,33 +789,11 @@ impl BatchingTask { let min_refresh = self.config.batch_opts.experimental_min_refresh_duration; - let new_query = match self.gen_insert_plan(&engine, max_window_cnt).await { - Ok(new_query) => new_query, - Err(err) => { - common_telemetry::error!(err; "Failed to generate query for flow={}", self.config.flow_id); - // also sleep for a little while before try again to prevent flooding logs - tokio::time::sleep(min_refresh).await; - continue; - } - }; + let outcome = self + .execute_once_serialized_with_outcome(&engine, &frontend_client, max_window_cnt) + .await; - let res = if let Some(new_query) = &new_query { - let dirty_filter = match &new_query.dirty_restore { - DirtyRestore::Scoped(f) => Some(f), - _ => None, - }; - self.execute_logical_plan( - &frontend_client, - &new_query.plan, - dirty_filter, - new_query.can_advance_checkpoints, - ) - .await - } else { - Ok(None) - }; - - match res { + match outcome.result { // normal execute, sleep for some time before doing next query Ok(Some(_)) => { // can increase max_window_cnt to query more windows next time @@ -703,11 +844,10 @@ impl BatchingTask { } // TODO(discord9): this error should have better place to go, but for now just print error, also more context is needed Err(err) => { - self.handle_executed_query_failure(new_query.as_ref()); METRIC_FLOW_BATCHING_ENGINE_ERROR_CNT .with_label_values(&[&flow_id_str]) .inc(); - match new_query { + match outcome.new_query { Some(query) => { common_telemetry::error!(err; "Failed to execute query for flow={} with query: {}", self.config.flow_id, query.plan); // TODO(discord9): add some backoff here? half the query time window or what @@ -743,6 +883,20 @@ impl BatchingTask { create_table_with_expr(&plan, &self.config.sink_table_name, &self.config.query_type) } + fn should_use_unfiltered_incremental_delta(&self) -> bool { + let state = self.state.read().unwrap(); + state.checkpoint_mode() == CheckpointMode::Incremental + && !state.is_incremental_disabled() + && matches!(self.config.query_type, QueryType::Sql) + } + + fn should_use_unfiltered_full_snapshot_seeding(&self) -> bool { + let state = self.state.read().unwrap(); + state.checkpoint_mode() == CheckpointMode::FullSnapshot + && !state.is_incremental_disabled() + && matches!(self.config.query_type, QueryType::Sql) + } + /// will merge and use the first ten time window in query async fn gen_query_with_time_window( &self, @@ -783,83 +937,35 @@ impl BatchingTask { self.config.flow_id ); // clean dirty time window too, this could be from create flow's check_execute - let (is_dirty, dirty_windows_to_restore) = { - let mut state = self.state.write().unwrap(); - let dirty_windows_to_restore = state.dirty_time_windows.clone(); - let is_dirty = !dirty_windows_to_restore.is_empty(); - state.dirty_time_windows.clean(); - (is_dirty, dirty_windows_to_restore) - }; - - if !is_dirty { - // no dirty data, hence no need to update - debug!("Flow id={:?}, no new data, not update", self.config.flow_id); - return Ok(None); - } - - let plan = match gen_plan_with_matching_schema( - &self.config.query, - query_ctx, - engine, - sink_table_schema.clone(), - primary_key_indices, - allow_partial, - ) - .await - { - Ok(plan) => plan, - Err(err) => { - self.state - .write() - .unwrap() - .dirty_time_windows - .add_dirty_windows(&dirty_windows_to_restore); - return Err(err); - } - }; - - return Ok(Some(PlanInfo { - plan, - dirty_restore: DirtyRestore::Unscoped(dirty_windows_to_restore), - can_advance_checkpoints: true, - })); + return self + .gen_unfiltered_plan_info_if_dirty( + engine, + query_ctx, + sink_table_schema.clone(), + primary_key_indices, + allow_partial, + None, + ) + .await; } _ => { // Clean dirty windows for full-query/non-scoped paths, // such as TQL, that cannot use a time-window filter. - let dirty_windows_to_restore = { - let mut state = self.state.write().unwrap(); - let dirty_windows_to_restore = state.dirty_time_windows.clone(); - state.dirty_time_windows.clean(); - dirty_windows_to_restore - }; + let (_, dirty_windows_to_restore) = self.drain_dirty_windows_signal(); - let plan = match gen_plan_with_matching_schema( - &self.config.query, - query_ctx, - engine, - sink_table_schema.clone(), - primary_key_indices, - allow_partial, - ) - .await - { - Ok(plan) => plan, - Err(err) => { - self.state - .write() - .unwrap() - .dirty_time_windows - .add_dirty_windows(&dirty_windows_to_restore); - return Err(err); - } - }; + let plan_info = self + .gen_unfiltered_plan_info( + engine, + query_ctx, + sink_table_schema.clone(), + primary_key_indices, + allow_partial, + dirty_windows_to_restore, + None, + ) + .await?; - return Ok(Some(PlanInfo { - plan, - dirty_restore: DirtyRestore::Unscoped(dirty_windows_to_restore), - can_advance_checkpoints: true, - })); + return Ok(Some(plan_info)); } }; @@ -889,22 +995,61 @@ impl BatchingTask { ), })?; + if self.should_use_unfiltered_full_snapshot_seeding() { + // A full-snapshot query that can seed/refresh incremental + // checkpoints must not use dirty-window predicates. Rows can be + // written after dirty windows are drained but before the source scan + // snapshot opens; a stale dirty-window filter could exclude those + // rows while the returned watermark includes them, causing the next + // incremental read to skip them forever. Execute an unfiltered full + // snapshot instead, and keep dirty windows only as the scheduling and + // failure-restoration signal. + let retention_filter = self + .config + .expire_after + .map(|_| (col_name.as_str(), expire_lower_bound, "full-snapshot")); + return self + .gen_unfiltered_plan_info_if_dirty( + engine, + query_ctx, + sink_table_schema.clone(), + primary_key_indices, + allow_partial, + retention_filter, + ) + .await; + } + + if self.should_use_unfiltered_incremental_delta() { + // In incremental mode, source correctness is defined by the + // per-region sequence range `(checkpoint, scan-open snapshot]`, not + // by dirty-window predicates. Dirty windows are only a scheduling + // signal here. Applying a stale dirty-window filter to the source can + // exclude rows that are inside the returned watermark and make a + // checkpoint advance skip them forever. The sink side is also left + // unfiltered by dirty windows; the incremental rewrite joins the + // delta groups with the full sink state for correctness. Future + // dynamic filters can prune sink reads as a pure optimization. + let retention_filter = self + .config + .expire_after + .map(|_| (col_name.as_str(), expire_lower_bound, "incremental")); + return self + .gen_unfiltered_plan_info_if_dirty( + engine, + query_ctx, + sink_table_schema.clone(), + primary_key_indices, + allow_partial, + retention_filter, + ) + .await; + } + let (expr, can_advance_checkpoints) = { let mut state = self.state.write().unwrap(); - let window_cnt = if state.checkpoint_mode() == CheckpointMode::Incremental - && !state.is_incremental_disabled() - && matches!(self.config.query_type, QueryType::Sql) - { - // Incremental scans are bounded by region sequence checkpoints, - // so the dirty-window filter only narrows sink-side/time-window - // work. Drain more windows than normal, but keep a hard cap to - // avoid building a huge OR filter after a long downtime. If - // windows remain, checkpoints won't advance this round. - MAX_INCREMENTAL_DIRTY_WINDOW_FILTERS - } else { - max_window_cnt - .unwrap_or(self.config.batch_opts.experimental_max_filter_num_per_query) - }; + let window_cnt = max_window_cnt + .unwrap_or(self.config.batch_opts.experimental_max_filter_num_per_query); let expr = state.dirty_time_windows.gen_filter_exprs( &col_name, Some(expire_lower_bound), diff --git a/src/flow/src/batching_mode/task/inc.rs b/src/flow/src/batching_mode/task/inc.rs index 4fb64a676e..9af54c1ba7 100644 --- a/src/flow/src/batching_mode/task/inc.rs +++ b/src/flow/src/batching_mode/task/inc.rs @@ -26,8 +26,7 @@ use snafu::ResultExt; use table::metadata::TableId; use crate::Error; -use crate::batching_mode::incremental_filter::build_sink_dirty_time_window_filter_expr; -use crate::batching_mode::state::{CheckpointMode, FilterExprInfo}; +use crate::batching_mode::state::CheckpointMode; use crate::batching_mode::table_creator::QueryType; use crate::batching_mode::task::BatchingTask; use crate::batching_mode::utils::{ @@ -74,7 +73,6 @@ impl BatchingTask { pub(super) async fn prepare_plan_for_incremental( &self, plan: &LogicalPlan, - dirty_filter: Option<&FilterExprInfo>, ) -> Result, Error> { let is_incremental_sql = { let state = self.state.read().unwrap(); @@ -152,31 +150,12 @@ impl BatchingTask { return Ok(None); } }; - let sink_schema = sink_table.table_info().meta.schema.clone(); - let sink_dirty_filter = match build_sink_dirty_time_window_filter_expr( - self.config.flow_id, - &analysis, - &sink_schema, - dirty_filter, - ) { - Ok(filter) => filter, - Err(err) => { - warn!( - "Flow {} failed to build sink dirty time window filter; \ - falling back to full snapshot for this round: {:?}", - self.config.flow_id, err - ); - self.state.write().unwrap().mark_full_snapshot(); - return Ok(None); - } - }; - let rewritten_inner = match rewrite_incremental_aggregate_with_sink_merge( &inner_plan, &analysis, sink_table, &self.config.sink_table_name, - sink_dirty_filter, + None, ) .await { diff --git a/src/flow/src/batching_mode/task/test.rs b/src/flow/src/batching_mode/task/test.rs index 959aeb00c9..d64b4ef1b9 100644 --- a/src/flow/src/batching_mode/task/test.rs +++ b/src/flow/src/batching_mode/task/test.rs @@ -25,7 +25,9 @@ use datatypes::data_type::ConcreteDataType as CDT; use datatypes::schema::ColumnSchema; use datatypes::vectors::{TimestampMillisecondVector, UInt32Vector, VectorRef}; use pretty_assertions::assert_eq; -use query::options::{FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY}; +use query::options::{ + FLOW_INCREMENTAL_AFTER_SEQS, FLOW_INCREMENTAL_MODE_MEMTABLE_ONLY, QueryOptions, +}; use session::context::QueryContext; use table::test_util::MemTable; @@ -38,6 +40,13 @@ use crate::batching_mode::state::CheckpointMode; use crate::batching_mode::time_window::find_time_window_expr; use crate::test_utils::create_test_query_engine; +fn incremental_batch_opts() -> Arc { + Arc::new(BatchingModeOptions { + experimental_enable_incremental_read: true, + ..Default::default() + }) +} + async fn new_test_task_and_plan_with_missing_sink() -> (BatchingTask, LogicalPlan) { new_test_task_engine_and_plan_with_query( "SELECT number, ts FROM numbers_with_ts", @@ -60,6 +69,15 @@ impl TestTaskParts { } async fn new_test_task_engine_and_plan_with_query(query: &str, sink_table: &str) -> TestTaskParts { + new_test_task_engine_and_plan_with_query_and_opts(query, sink_table, incremental_batch_opts()) + .await +} + +async fn new_test_task_engine_and_plan_with_query_and_opts( + query: &str, + sink_table: &str, + batch_opts: Arc, +) -> TestTaskParts { let query_engine = create_test_query_engine(); let ctx = QueryContext::arc(); let plan = sql_to_df_plan( @@ -91,7 +109,7 @@ async fn new_test_task_engine_and_plan_with_query(query: &str, sink_table: &str) query_ctx: ctx, catalog_manager: query_engine.engine_state().catalog_manager().clone(), shutdown_rx: rx, - batch_opts: Arc::new(BatchingModeOptions::default()), + batch_opts, flow_eval_interval: None, }) .unwrap(); @@ -103,6 +121,75 @@ async fn new_test_task_engine_and_plan_with_query(query: &str, sink_table: &str) } } +#[tokio::test] +async fn test_incremental_read_is_disabled_by_default() { + let task = new_test_task_engine_and_plan_with_query_and_opts( + "SELECT number, ts FROM numbers_with_ts", + "numbers_with_ts", + Arc::new(BatchingModeOptions::default()), + ) + .await + .task; + + assert!(task.state.read().unwrap().is_incremental_disabled()); +} + +#[tokio::test] +async fn test_dirty_time_windows_uses_batch_opts() { + let task = new_test_task_engine_and_plan_with_query_and_opts( + "SELECT number, ts FROM numbers_with_ts", + "numbers_with_ts", + Arc::new(BatchingModeOptions { + experimental_max_filter_num_per_query: 7, + experimental_time_window_merge_threshold: 11, + ..Default::default() + }), + ) + .await + .task; + + let state = task.state.read().unwrap(); + assert_eq!(7, state.dirty_time_windows.max_filter_num_per_query()); + assert_eq!(11, state.dirty_time_windows.time_window_merge_threshold()); +} + +#[tokio::test] +async fn test_execute_once_serialized_waits_for_execution_lock() { + let TestTaskParts { + task, query_engine, .. + } = new_test_task_engine_and_plan_with_query( + "SELECT number, ts FROM numbers_with_ts", + "missing_sink", + ) + .await; + let (frontend_client, _handler) = + FrontendClient::from_empty_grpc_handler(QueryOptions::default()); + let frontend_client = Arc::new(frontend_client); + + let guard = task.execution_lock.clone().lock_owned().await; + let task_to_run = task.clone(); + let query_engine_to_run = query_engine.clone(); + let frontend_client_to_run = frontend_client.clone(); + let exec = tokio::spawn(async move { + task_to_run + .execute_once_serialized(&query_engine_to_run, &frontend_client_to_run, None) + .await + }); + + tokio::time::sleep(Duration::from_millis(20)).await; + assert!( + !exec.is_finished(), + "execute_once_serialized should wait for execution_lock" + ); + + drop(guard); + tokio::time::timeout(Duration::from_secs(1), exec) + .await + .expect("execute_once_serialized should finish once execution_lock is released") + .expect("execute_once_serialized task should not panic") + .expect_err("missing sink should fail after acquiring execution_lock"); +} + async fn new_time_window_test_task_with_query(query: &str) -> TestTaskParts { let query_engine = create_test_query_engine(); let ctx = QueryContext::arc(); @@ -147,7 +234,7 @@ async fn new_time_window_test_task_with_query(query: &str) -> TestTaskParts { query_ctx: ctx, catalog_manager: query_engine.engine_state().catalog_manager().clone(), shutdown_rx: rx, - batch_opts: Arc::new(BatchingModeOptions::default()), + batch_opts: incremental_batch_opts(), flow_eval_interval: None, }) .unwrap(); @@ -226,6 +313,14 @@ fn dirty_range(start: i64, end: i64) -> DirtyTimeWindows { dirty } +fn expire_after_for_retention_filter_test() -> i64 { + let now_secs = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Time went backwards") + .as_secs(); + (now_secs - 10) as i64 +} + async fn assert_unscoped_failure_restore( consumed_dirty_windows: DirtyTimeWindows, current_dirty_windows: DirtyTimeWindows, @@ -626,6 +721,7 @@ async fn test_full_snapshot_scoped_plan_marks_checkpoint_advance_safe_only_after .await; { let mut state = task.state.write().unwrap(); + state.disable_incremental(); state .dirty_time_windows .add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5))); @@ -657,7 +753,7 @@ async fn test_full_snapshot_scoped_plan_marks_checkpoint_advance_safe_only_after } #[tokio::test] -async fn test_incremental_scoped_plan_consumes_all_dirty_windows_for_checkpoint_safety() { +async fn test_incremental_plan_consumes_dirty_signal_for_checkpoint_safety() { let TestTaskParts { task, query_engine, @@ -692,6 +788,192 @@ async fn test_incremental_scoped_plan_consumes_all_dirty_windows_for_checkpoint_ assert!(task.state.read().unwrap().dirty_time_windows.is_empty()); } +#[tokio::test] +async fn test_full_snapshot_seeding_for_incremental_does_not_add_dirty_window_filter() { + let TestTaskParts { + task, + query_engine, + .. + } = new_time_window_test_task_with_query( + "SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window", + ) + .await; + { + let mut state = task.state.write().unwrap(); + assert_eq!(state.checkpoint_mode(), CheckpointMode::FullSnapshot); + assert!(!state.is_incremental_disabled()); + state + .dirty_time_windows + .add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5))); + state + .dirty_time_windows + .add_window(Timestamp::new_second(30), Some(Timestamp::new_second(35))); + } + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", CDT::uint32_datatype(), false), + ColumnSchema::new("time_window", CDT::timestamp_millisecond_datatype(), false) + .with_time_index(true), + ])); + + let plan = task + .gen_query_with_time_window(query_engine, &sink_schema, &[], false, Some(1)) + .await + .unwrap() + .unwrap(); + + let plan_text = plan.plan.to_string(); + assert!(plan.can_advance_checkpoints); + assert!(task.state.read().unwrap().dirty_time_windows.is_empty()); + assert!(!plan_text.contains("Filter:"), "{plan_text}"); +} + +#[tokio::test] +async fn test_full_snapshot_seeding_applies_expire_after_retention_filter() { + let TestTaskParts { + mut task, + query_engine, + .. + } = new_time_window_test_task_with_query( + "SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window", + ) + .await; + { + let mut state = task.state.write().unwrap(); + assert_eq!(state.checkpoint_mode(), CheckpointMode::FullSnapshot); + assert!(!state.is_incremental_disabled()); + state + .dirty_time_windows + .add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5))); + } + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", CDT::uint32_datatype(), false), + ColumnSchema::new("time_window", CDT::timestamp_millisecond_datatype(), false) + .with_time_index(true), + ])); + + Arc::get_mut(&mut task.config) + .expect("test task config should be uniquely owned") + .expire_after = Some(expire_after_for_retention_filter_test()); + let plan = task + .gen_query_with_time_window(query_engine, &sink_schema, &[], false, Some(1)) + .await + .unwrap() + .unwrap(); + + assert!(plan.can_advance_checkpoints); + assert!(task.state.read().unwrap().dirty_time_windows.is_empty()); + let plan_text = plan.plan.to_string(); + assert!( + plan_text.contains("Filter: ts >= TimestampMillisecond("), + "{plan_text}" + ); +} + +#[tokio::test] +async fn test_incremental_plan_does_not_add_dirty_window_filter() { + let TestTaskParts { + task, + query_engine, + .. + } = new_time_window_test_task_with_query( + "SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window", + ) + .await; + { + let mut state = task.state.write().unwrap(); + state.advance_checkpoints(HashMap::from([(1_u64, 10_u64)])); + state + .dirty_time_windows + .add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5))); + } + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", CDT::uint32_datatype(), false), + ColumnSchema::new("time_window", CDT::timestamp_millisecond_datatype(), false) + .with_time_index(true), + ])); + + let plan = task + .gen_query_with_time_window(query_engine, &sink_schema, &[], false, Some(1)) + .await + .unwrap() + .unwrap(); + + let plan_text = plan.plan.to_string(); + assert!(plan.can_advance_checkpoints); + assert!(!plan_text.contains("Filter:"), "{plan_text}"); +} + +#[tokio::test] +async fn test_incremental_delta_applies_expire_after_retention_filter() { + let TestTaskParts { + mut task, + query_engine, + .. + } = new_time_window_test_task_with_query( + "SELECT max(number) AS number, date_bin(INTERVAL '5 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window", + ) + .await; + { + let mut state = task.state.write().unwrap(); + state.advance_checkpoints(HashMap::from([(1_u64, 10_u64)])); + state + .dirty_time_windows + .add_window(Timestamp::new_second(0), Some(Timestamp::new_second(5))); + } + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", CDT::uint32_datatype(), false), + ColumnSchema::new("time_window", CDT::timestamp_millisecond_datatype(), false) + .with_time_index(true), + ])); + + Arc::get_mut(&mut task.config) + .expect("test task config should be uniquely owned") + .expire_after = Some(expire_after_for_retention_filter_test()); + let plan = task + .gen_query_with_time_window(query_engine, &sink_schema, &[], false, Some(1)) + .await + .unwrap() + .unwrap(); + + assert!(plan.can_advance_checkpoints); + assert!(task.state.read().unwrap().dirty_time_windows.is_empty()); + let plan_text = plan.plan.to_string(); + assert!( + plan_text.contains("Filter: ts >= TimestampMillisecond("), + "{plan_text}" + ); +} + +#[tokio::test] +async fn test_non_scoped_path_generates_plan_with_empty_dirty_signal() { + let TestTaskParts { + mut task, + query_engine, + .. + } = new_test_task_engine_and_plan_with_query( + "SELECT number, ts FROM numbers_with_ts", + "missing_sink", + ) + .await; + Arc::get_mut(&mut task.config) + .expect("test task config should be uniquely owned") + .query_type = QueryType::Tql; + task.state.write().unwrap().dirty_time_windows.clean(); + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", CDT::uint32_datatype(), false), + ColumnSchema::new("ts", CDT::timestamp_millisecond_datatype(), false).with_time_index(true), + ])); + + let plan = task + .gen_query_with_time_window(query_engine, &sink_schema, &[], false, None) + .await + .unwrap() + .expect("non-scoped path should generate a plan even with an empty dirty signal"); + + assert!(plan.can_advance_checkpoints); + assert!(task.state.read().unwrap().dirty_time_windows.is_empty()); +} + #[tokio::test] async fn test_executed_query_failure_restores_scoped_dirty_windows_for_flush_path() { let (task, plan) = new_test_task_and_plan_with_missing_sink().await; @@ -773,7 +1055,7 @@ async fn test_prepare_plan_for_incremental_disables_on_non_aggregate() { query_ctx: ctx, catalog_manager: query_engine.engine_state().catalog_manager().clone(), shutdown_rx: rx, - batch_opts: Arc::new(BatchingModeOptions::default()), + batch_opts: incremental_batch_opts(), flow_eval_interval: None, }) .unwrap(); @@ -788,10 +1070,7 @@ async fn test_prepare_plan_for_incremental_disables_on_non_aggregate() { CheckpointMode::Incremental ); - let incremental_plan = task - .prepare_plan_for_incremental(&dml_plan, None) - .await - .unwrap(); + let incremental_plan = task.prepare_plan_for_incremental(&dml_plan).await.unwrap(); assert!(incremental_plan.is_none()); let state = task.state.read().unwrap(); assert!(state.is_incremental_disabled()); @@ -852,7 +1131,7 @@ async fn test_prepare_plan_for_incremental_falls_back_without_disable_on_rewrite query_ctx: ctx, catalog_manager: query_engine.engine_state().catalog_manager().clone(), shutdown_rx: rx, - batch_opts: Arc::new(BatchingModeOptions::default()), + batch_opts: incremental_batch_opts(), flow_eval_interval: None, }) .unwrap(); @@ -866,10 +1145,7 @@ async fn test_prepare_plan_for_incremental_falls_back_without_disable_on_rewrite CheckpointMode::Incremental ); - let incremental_plan = task - .prepare_plan_for_incremental(&dml_plan, None) - .await - .unwrap(); + let incremental_plan = task.prepare_plan_for_incremental(&dml_plan).await.unwrap(); assert!(incremental_plan.is_none()); let state = task.state.read().unwrap(); assert!(!state.is_incremental_disabled()); @@ -928,7 +1204,7 @@ async fn test_prepare_plan_for_incremental_group_by_without_merge_columns_uses_o query_ctx: ctx, catalog_manager: query_engine.engine_state().catalog_manager().clone(), shutdown_rx: rx, - batch_opts: Arc::new(BatchingModeOptions::default()), + batch_opts: incremental_batch_opts(), flow_eval_interval: None, }) .unwrap(); @@ -939,7 +1215,7 @@ async fn test_prepare_plan_for_incremental_group_by_without_merge_columns_uses_o .advance_checkpoints(HashMap::from([(1_u64, 10_u64)])); let incremental_plan = task - .prepare_plan_for_incremental(&dml_plan, None) + .prepare_plan_for_incremental(&dml_plan) .await .unwrap() .expect("plain GROUP BY is incremental-safe without a rewrite"); @@ -962,7 +1238,7 @@ async fn test_auto_created_sql_aggregate_sink_reaches_incremental_safe() { task.state.write().unwrap().dirty_time_windows.set_dirty(); let plan_info = task - .gen_insert_plan(&query_engine, None) + .gen_insert_plan_unlocked(&query_engine, None) .await .unwrap() .unwrap(); @@ -973,7 +1249,7 @@ async fn test_auto_created_sql_aggregate_sink_reaches_incremental_safe() { .unwrap() .advance_checkpoints(HashMap::from([(1_u64, 10_u64)])); let incremental_plan = task - .prepare_plan_for_incremental(&plan_info.plan, None) + .prepare_plan_for_incremental(&plan_info.plan) .await .unwrap(); let incremental_safe = incremental_plan.is_some(); @@ -1078,11 +1354,11 @@ async fn test_insert_plan_matching_failure_restores_consumed_dirty_marker() { register_number_only_sink(&query_engine, sink_table); task.state.write().unwrap().dirty_time_windows.set_dirty(); - let result = task.gen_insert_plan(&query_engine, None).await; + let result = task.gen_insert_plan_unlocked(&query_engine, None).await; assert!(result.is_err()); let _err = match result { - Ok(_) => panic!("gen_insert_plan should fail with a sink column mismatch"), + Ok(_) => panic!("gen_insert_plan_unlocked should fail with a sink column mismatch"), Err(err) => err, }; let state = task.state.read().unwrap(); diff --git a/src/flow/src/batching_mode/utils/test.rs b/src/flow/src/batching_mode/utils/test.rs index 5b9cf7f507..317b0a5475 100644 --- a/src/flow/src/batching_mode/utils/test.rs +++ b/src/flow/src/batching_mode/utils/test.rs @@ -1288,9 +1288,10 @@ async fn test_rewrite_incremental_aggregate_with_left_join() { #[tokio::test] async fn test_rewrite_incremental_aggregate_filters_sink_dirty_time_window() { - // This verifies the rewrite placement when callers supply an already - // inferred sink dirty-window predicate. The task-level inference rules are - // covered by `infer_sink_time_window_filter_col` tests in task.rs. + // This verifies the rewrite placement when callers supply a sink predicate. + // The production incremental flow path currently leaves sink scans + // unfiltered for correctness and relies on future dynamic filters for + // pruning. let query_engine = create_test_query_engine(); let ctx = QueryContext::arc(); let sql = "SELECT max(number) AS number, date_bin(INTERVAL '1 second', ts) AS time_window FROM numbers_with_ts GROUP BY time_window"; diff --git a/src/operator/src/statement/ddl.rs b/src/operator/src/statement/ddl.rs index aef164c6bb..cadb1cde66 100644 --- a/src/operator/src/statement/ddl.rs +++ b/src/operator/src/statement/ddl.rs @@ -35,7 +35,9 @@ use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME, is_reado use common_catalog::{format_full_flow_name, format_full_table_name}; use common_error::ext::BoxedError; use common_meta::cache_invalidator::Context; -use common_meta::ddl::create_flow::{DEFER_ON_MISSING_SOURCE_KEY, FlowType}; +use common_meta::ddl::create_flow::{ + DEFER_ON_MISSING_SOURCE_KEY, FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, FlowType, +}; use common_meta::instruction::CacheIdent; use common_meta::key::schema_name::{SchemaName, SchemaNameKey}; use common_meta::procedure_executor::ExecutorContext; @@ -114,7 +116,10 @@ struct DdlSubmitOptions { timeout: Duration, } -const ALLOWED_FLOW_OPTIONS: [&str; 1] = [DEFER_ON_MISSING_SOURCE_KEY]; +const ALLOWED_FLOW_OPTIONS: [&str; 2] = [ + DEFER_ON_MISSING_SOURCE_KEY, + FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY, +]; fn build_procedure_id_output(procedure_id: Vec) -> Result { let procedure_id = String::from_utf8_lossy(&procedure_id).to_string(); @@ -187,7 +192,9 @@ fn validate_and_normalize_flow_options( } let normalized_value = match key.as_str() { - DEFER_ON_MISSING_SOURCE_KEY => normalize_flow_bool_option(&key, &value)?, + DEFER_ON_MISSING_SOURCE_KEY | FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY => { + normalize_flow_bool_option(&key, &value)? + } _ => { return InvalidSqlSnafu { err_msg: format!( @@ -2478,12 +2485,23 @@ mod test { #[test] fn test_validate_and_normalize_flow_options_valid() { - let options = - HashMap::from([(DEFER_ON_MISSING_SOURCE_KEY.to_string(), "TRUE".to_string())]); + let options = HashMap::from([ + (DEFER_ON_MISSING_SOURCE_KEY.to_string(), "TRUE".to_string()), + ( + FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(), + "FALSE".to_string(), + ), + ]); assert_eq!( validate_and_normalize_flow_options(options).unwrap(), - HashMap::from([(DEFER_ON_MISSING_SOURCE_KEY.to_string(), "true".to_string(),)]) + HashMap::from([ + (DEFER_ON_MISSING_SOURCE_KEY.to_string(), "true".to_string(),), + ( + FLOW_EXPERIMENTAL_ENABLE_INCREMENTAL_READ_KEY.to_string(), + "false".to_string(), + ) + ]) ); } @@ -2497,7 +2515,7 @@ mod test { assert!( err.to_string() - .contains("unknown flow option 'foo', supported options: defer_on_missing_source") + .contains("unknown flow option 'foo', supported options: defer_on_missing_source, experimental_enable_incremental_read") ); } diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 6e35ed7656..5fe22359f3 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -1602,6 +1602,7 @@ experimental_grpc_max_retries = 3 experimental_frontend_scan_timeout = "30s" experimental_max_filter_num_per_query = 20 experimental_time_window_merge_threshold = 3 +experimental_enable_incremental_read = false read_preference = "Leader" [logging] diff --git a/tests/cases/standalone/common/flow/flow_incremental_aggr.result b/tests/cases/standalone/common/flow/flow_incremental_aggr.result index bb66d5362c..2273e0e821 100644 --- a/tests/cases/standalone/common/flow/flow_incremental_aggr.result +++ b/tests/cases/standalone/common/flow/flow_incremental_aggr.result @@ -1,3 +1,31 @@ +-- Incremental aggregate reads only support append-only source tables because +-- update/upsert sources need old-value compensation. +CREATE TABLE incremental_non_append_input ( + host_id INT, + n INT, + ts TIMESTAMP TIME INDEX, + PRIMARY KEY(host_id) +); + +Affected Rows: 0 + +CREATE FLOW incremental_non_append_flow SINK TO incremental_non_append_sink +WITH (experimental_enable_incremental_read = 'true') +AS +SELECT + sum(n) AS total, + date_bin(INTERVAL '1 minute', ts, '2024-01-01 00:00:00') AS time_window +FROM + incremental_non_append_input +GROUP BY + time_window; + +Error: 3001(EngineExecuteQuery), Unsupported: Flow incremental read requires append-only source table, but source table `greptime.public.incremental_non_append_input` is not append-only. Consider setting append_mode='true' on the source table or disabling experimental_enable_incremental_read + +DROP TABLE incremental_non_append_input; + +Affected Rows: 0 + CREATE TABLE incremental_aggr_input ( host_id INT, n INT, @@ -9,7 +37,9 @@ CREATE TABLE incremental_aggr_input ( Affected Rows: 0 -CREATE FLOW incremental_aggr_flow SINK TO incremental_aggr_sink AS +CREATE FLOW incremental_aggr_flow SINK TO incremental_aggr_sink +WITH (experimental_enable_incremental_read = 'true') +AS SELECT sum(n) AS total, min(n) AS min_n, diff --git a/tests/cases/standalone/common/flow/flow_incremental_aggr.sql b/tests/cases/standalone/common/flow/flow_incremental_aggr.sql index 51dd431fef..4c012aef23 100644 --- a/tests/cases/standalone/common/flow/flow_incremental_aggr.sql +++ b/tests/cases/standalone/common/flow/flow_incremental_aggr.sql @@ -1,3 +1,25 @@ +-- Incremental aggregate reads only support append-only source tables because +-- update/upsert sources need old-value compensation. +CREATE TABLE incremental_non_append_input ( + host_id INT, + n INT, + ts TIMESTAMP TIME INDEX, + PRIMARY KEY(host_id) +); + +CREATE FLOW incremental_non_append_flow SINK TO incremental_non_append_sink +WITH (experimental_enable_incremental_read = 'true') +AS +SELECT + sum(n) AS total, + date_bin(INTERVAL '1 minute', ts, '2024-01-01 00:00:00') AS time_window +FROM + incremental_non_append_input +GROUP BY + time_window; + +DROP TABLE incremental_non_append_input; + CREATE TABLE incremental_aggr_input ( host_id INT, n INT, @@ -7,7 +29,9 @@ CREATE TABLE incremental_aggr_input ( append_mode = 'true' ); -CREATE FLOW incremental_aggr_flow SINK TO incremental_aggr_sink AS +CREATE FLOW incremental_aggr_flow SINK TO incremental_aggr_sink +WITH (experimental_enable_incremental_read = 'true') +AS SELECT sum(n) AS total, min(n) AS min_n, diff --git a/tests/cases/standalone/common/flow/flow_incremental_memtable.result b/tests/cases/standalone/common/flow/flow_incremental_memtable.result index 1e452b21ad..67326e1261 100644 --- a/tests/cases/standalone/common/flow/flow_incremental_memtable.result +++ b/tests/cases/standalone/common/flow/flow_incremental_memtable.result @@ -12,7 +12,9 @@ CREATE TABLE flow_incr_memtable_input ( Affected Rows: 0 -CREATE FLOW flow_incr_memtable SINK TO flow_incr_memtable_sink AS +CREATE FLOW flow_incr_memtable SINK TO flow_incr_memtable_sink +WITH (experimental_enable_incremental_read = 'true') +AS SELECT sum(n) AS total, min(n) AS min_n, diff --git a/tests/cases/standalone/common/flow/flow_incremental_memtable.sql b/tests/cases/standalone/common/flow/flow_incremental_memtable.sql index 66dccbb8b3..6dbbf6064f 100644 --- a/tests/cases/standalone/common/flow/flow_incremental_memtable.sql +++ b/tests/cases/standalone/common/flow/flow_incremental_memtable.sql @@ -10,7 +10,9 @@ CREATE TABLE flow_incr_memtable_input ( append_mode = 'true' ); -CREATE FLOW flow_incr_memtable SINK TO flow_incr_memtable_sink AS +CREATE FLOW flow_incr_memtable SINK TO flow_incr_memtable_sink +WITH (experimental_enable_incremental_read = 'true') +AS SELECT sum(n) AS total, min(n) AS min_n, diff --git a/tests/cases/standalone/common/flow/flow_incremental_partitioned.result b/tests/cases/standalone/common/flow/flow_incremental_partitioned.result index b56b390abd..0899d4acb0 100644 --- a/tests/cases/standalone/common/flow/flow_incremental_partitioned.result +++ b/tests/cases/standalone/common/flow/flow_incremental_partitioned.result @@ -17,7 +17,9 @@ WITH ( Affected Rows: 0 -CREATE FLOW flow_incr_part SINK TO flow_incr_part_sink AS +CREATE FLOW flow_incr_part SINK TO flow_incr_part_sink +WITH (experimental_enable_incremental_read = 'true') +AS SELECT sum(n) AS total, min(n) AS min_n, diff --git a/tests/cases/standalone/common/flow/flow_incremental_partitioned.sql b/tests/cases/standalone/common/flow/flow_incremental_partitioned.sql index 234c9b9085..18cece1889 100644 --- a/tests/cases/standalone/common/flow/flow_incremental_partitioned.sql +++ b/tests/cases/standalone/common/flow/flow_incremental_partitioned.sql @@ -15,7 +15,9 @@ WITH ( append_mode = 'true' ); -CREATE FLOW flow_incr_part SINK TO flow_incr_part_sink AS +CREATE FLOW flow_incr_part SINK TO flow_incr_part_sink +WITH (experimental_enable_incremental_read = 'true') +AS SELECT sum(n) AS total, min(n) AS min_n, diff --git a/tests/cases/standalone/common/flow/show_create_flow.result b/tests/cases/standalone/common/flow/show_create_flow.result index 113822cd18..431d1dfbb5 100644 --- a/tests/cases/standalone/common/flow/show_create_flow.result +++ b/tests/cases/standalone/common/flow/show_create_flow.result @@ -476,7 +476,7 @@ SINK TO out_num_cnt_show WITH (access_key_id = [true]) AS SELECT number AS n1 FROM numbers_input_show where number > 10; -Error: 1004(InvalidArguments), Invalid SQL, error: unknown flow option 'access_key_id', supported options: defer_on_missing_source +Error: 1004(InvalidArguments), Invalid SQL, error: unknown flow option 'access_key_id', supported options: defer_on_missing_source, experimental_enable_incremental_read DROP FLOW filter_numbers_show; From 33ffa41ac2c1b10784e08902583cf5a7e026ed27 Mon Sep 17 00:00:00 2001 From: LFC <990479+MichaelScofield@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:45:38 +0800 Subject: [PATCH 06/29] refactor: use last compaction scheduled time instead of finished time (#8190) * refactor: use last compaction scheduled time instead of finished time Signed-off-by: luofucong * add some logs for debugging Signed-off-by: luofucong --------- Signed-off-by: luofucong --- src/mito2/src/compaction.rs | 136 +++++++++++++++++++--- src/mito2/src/engine/edit_region_test.rs | 41 +++++-- src/mito2/src/region.rs | 21 ++-- src/mito2/src/region/opener.rs | 4 +- src/mito2/src/worker/handle_compaction.rs | 49 ++++++-- 5 files changed, 201 insertions(+), 50 deletions(-) diff --git a/src/mito2/src/compaction.rs b/src/mito2/src/compaction.rs index 5cf3c444a8..c91809e253 100644 --- a/src/mito2/src/compaction.rs +++ b/src/mito2/src/compaction.rs @@ -150,6 +150,7 @@ impl CompactionScheduler { } /// Schedules a compaction for the region. + /// Returns whether a compaction is scheduled. #[allow(clippy::too_many_arguments)] pub(crate) async fn schedule_compaction( &mut self, @@ -161,7 +162,7 @@ impl CompactionScheduler { manifest_ctx: &ManifestContextRef, schema_metadata_manager: SchemaMetadataManagerRef, max_parallelism: usize, - ) -> Result<()> { + ) -> Result { // skip compaction if region is in staging state let current_state = manifest_ctx.current_state(); if current_state == RegionRoleState::Leader(RegionLeaderState::Staging) { @@ -170,7 +171,7 @@ impl CompactionScheduler { region_id, compact_options ); waiter.send(Ok(0)); - return Ok(()); + return Ok(false); } if let Some(status) = self.region_status.get_mut(®ion_id) { @@ -192,7 +193,7 @@ impl CompactionScheduler { ); } } - return Ok(()); + return Ok(false); } // The region can compact directly. @@ -209,7 +210,7 @@ impl CompactionScheduler { max_parallelism, ); - let result = match self + match self .schedule_compaction_request(request, compact_options) .await { @@ -220,14 +221,12 @@ impl CompactionScheduler { status.active_compaction = Some(active_compaction); self.region_status.insert(region_id, status); - Ok(()) + self.listener.on_compaction_scheduled(region_id); + Ok(true) } - Ok(None) => Ok(()), + Ok(None) => Ok(false), Err(e) => Err(e), - }; - - self.listener.on_compaction_scheduled(region_id); - result + } } // Handle pending manual compaction request for the region. @@ -334,6 +333,27 @@ impl CompactionScheduler { // And skip try to schedule next compaction task. return pending_ddl_requests; } + Vec::new() + } + + pub(crate) fn is_compacting(&self, region_id: RegionId) -> bool { + self.region_status + .get(®ion_id) + .map(|status| status.active_compaction.is_some()) + .unwrap_or(false) + } + + /// Schedules next compaction upon a finished compaction. + /// Returns whether the compaction is scheduled. + pub(crate) async fn schedule_next_compaction( + &mut self, + region_id: RegionId, + manifest_ctx: &ManifestContextRef, + schema_metadata_manager: SchemaMetadataManagerRef, + ) -> bool { + let Some(status) = self.region_status.get_mut(®ion_id) else { + return false; + }; // We should always try to compact the region until picker returns None. let request = status.new_compaction_request( @@ -364,20 +384,21 @@ impl CompactionScheduler { "Successfully scheduled next compaction for region id: {}", region_id ); + true } Ok(None) => { // No further compaction tasks can be scheduled; cleanup the `CompactionStatus` for this region. // All DDL requests and pending compaction requests have already been processed. // Safe to remove the region from status tracking. self.region_status.remove(®ion_id); + false } Err(e) => { error!(e; "Failed to schedule next compaction for region {}", region_id); self.remove_region_on_failure(region_id, Arc::new(e)); + false } } - - Vec::new() } /// Notifies the scheduler that the compaction job is cancelled cooperatively. @@ -1434,7 +1455,7 @@ mod tests { let manifest_ctx = env .mock_manifest_context(version_control.current().version.metadata.clone()) .await; - scheduler + let scheduled = scheduler .schedule_compaction( builder.region_id(), compact_request::Options::Regular(Default::default()), @@ -1447,6 +1468,7 @@ mod tests { ) .await .unwrap(); + assert!(!scheduled); let output = output_rx.await.unwrap().unwrap(); assert_eq!(output, 0); assert!(scheduler.region_status.is_empty()); @@ -1455,7 +1477,7 @@ mod tests { let version_control = Arc::new(builder.push_l0_file(0, 1000).build()); let (output_tx, output_rx) = oneshot::channel(); let waiter = OptionOutputTx::from(output_tx); - scheduler + let scheduled = scheduler .schedule_compaction( builder.region_id(), compact_request::Options::Regular(Default::default()), @@ -1468,11 +1490,67 @@ mod tests { ) .await .unwrap(); + assert!(!scheduled); let output = output_rx.await.unwrap().unwrap(); assert_eq!(output, 0); assert!(scheduler.region_status.is_empty()); } + #[tokio::test] + async fn test_schedule_compaction_returns_true_when_task_scheduled() { + let job_scheduler = Arc::new(VecScheduler::default()); + let env = SchedulerEnv::new().await.scheduler(job_scheduler.clone()); + let (tx, _rx) = mpsc::channel(4); + let mut scheduler = env.mock_compaction_scheduler(tx); + let mut builder = VersionControlBuilder::new(); + let region_id = builder.region_id(); + let end = 1000 * 1000; + // Five overlapping L0 files are enough for the regular picker to create a task. + let version_control = Arc::new( + builder + .push_l0_file(0, end) + .push_l0_file(10, end) + .push_l0_file(50, end) + .push_l0_file(80, end) + .push_l0_file(90, end) + .build(), + ); + let manifest_ctx = env + .mock_manifest_context(version_control.current().version.metadata.clone()) + .await; + let (schema_metadata_manager, kv_backend) = mock_schema_metadata_manager(); + schema_metadata_manager + .register_region_table_info( + region_id.table_id(), + "test_table", + "test_catalog", + "test_schema", + None, + kv_backend, + ) + .await; + + let scheduled = scheduler + .schedule_compaction( + region_id, + Options::Regular(Default::default()), + &version_control, + &env.access_layer, + OptionOutputTx::none(), + &manifest_ctx, + schema_metadata_manager, + 1, + ) + .await + .unwrap(); + + // The boolean result is what the worker uses to decide whether to update + // last_schedule_compaction_millis. + assert!(scheduled); + assert_eq!(1, job_scheduler.num_jobs()); + assert!(scheduler.region_status.contains_key(®ion_id)); + } + #[tokio::test] async fn test_schedule_on_finished() { common_telemetry::init_default_ut_logging(); @@ -1510,7 +1588,7 @@ mod tests { let manifest_ctx = env .mock_manifest_context(version_control.current().version.metadata.clone()) .await; - scheduler + let scheduled = scheduler .schedule_compaction( region_id, compact_request::Options::Regular(Default::default()), @@ -1524,6 +1602,7 @@ mod tests { .await .unwrap(); // Should schedule 1 compaction. + assert!(scheduled); assert_eq!(1, scheduler.region_status.len()); assert_eq!(1, job_scheduler.num_jobs()); let data = version_control.current(); @@ -1542,7 +1621,7 @@ mod tests { ); // The task is pending. let (tx, _rx) = oneshot::channel(); - scheduler + let scheduled = scheduler .schedule_compaction( region_id, compact_request::Options::Regular(Default::default()), @@ -1555,6 +1634,7 @@ mod tests { ) .await .unwrap(); + assert!(!scheduled); assert_eq!(1, scheduler.region_status.len()); assert_eq!(1, job_scheduler.num_jobs()); assert!( @@ -1570,6 +1650,10 @@ mod tests { scheduler .on_compaction_finished(region_id, &manifest_ctx, schema_metadata_manager.clone()) .await; + let scheduled = scheduler + .schedule_next_compaction(region_id, &manifest_ctx, schema_metadata_manager.clone()) + .await; + assert!(scheduled); assert_eq!(1, scheduler.region_status.len()); assert_eq!(2, job_scheduler.num_jobs()); @@ -1582,7 +1666,7 @@ mod tests { ); let (tx, _rx) = oneshot::channel(); // The task is pending. - scheduler + let scheduled = scheduler .schedule_compaction( region_id, compact_request::Options::Regular(Default::default()), @@ -1595,6 +1679,7 @@ mod tests { ) .await .unwrap(); + assert!(!scheduled); assert_eq!(2, job_scheduler.num_jobs()); assert!( !scheduler @@ -2328,6 +2413,15 @@ mod tests { .await; assert!(pending_ddls.is_empty()); + assert!(scheduler.region_status.contains_key(®ion_id)); + + let (schema_metadata_manager, _kv_backend) = mock_schema_metadata_manager(); + // With no compactable files, next scheduling returns false and removes + // the status without creating a background task. + let scheduled = scheduler + .schedule_next_compaction(region_id, &manifest_ctx, schema_metadata_manager) + .await; + assert!(!scheduled); assert!(!scheduler.region_status.contains_key(®ion_id)); } @@ -2370,6 +2464,14 @@ mod tests { .await; assert!(pending_ddls.is_empty()); + assert!(scheduler.region_status.contains_key(®ion_id)); + + let (schema_metadata_manager, _kv_backend) = mock_schema_metadata_manager(); + // The failing scheduler simulates a submit error; callers must see false. + let scheduled = scheduler + .schedule_next_compaction(region_id, &manifest_ctx, schema_metadata_manager) + .await; + assert!(!scheduled); assert!(!scheduler.region_status.contains_key(®ion_id)); } diff --git a/src/mito2/src/engine/edit_region_test.rs b/src/mito2/src/engine/edit_region_test.rs index e05e1a847a..1d9bf2101b 100644 --- a/src/mito2/src/engine/edit_region_test.rs +++ b/src/mito2/src/engine/edit_region_test.rs @@ -21,6 +21,7 @@ use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; use common_recordbatch::DfRecordBatch; use common_test_util::flight::encode_to_flight_data; +use common_time::Timestamp; use common_time::util::current_time_millis; use datatypes::arrow::array::{ArrayRef, Float64Array, StringArray, TimestampMillisecondArray}; use datatypes::arrow::datatypes::{DataType, Field, Schema, TimeUnit}; @@ -67,7 +68,8 @@ async fn test_edit_region_schedule_compaction_with_format(flat_format: bool) { default_flat_format: flat_format, ..Default::default() }; - let time_provider = Arc::new(MockTimeProvider::new(current_time_millis())); + let initial_time = current_time_millis(); + let time_provider = Arc::new(MockTimeProvider::new(initial_time)); let engine = env .create_engine_with_time( config.clone(), @@ -99,14 +101,22 @@ async fn test_edit_region_schedule_compaction_with_format(flat_format: bool) { .await .unwrap(); let region = engine.get_region(region_id).unwrap(); + let initial_schedule_time = region.last_schedule_compaction_millis(); + assert_eq!(initial_time, initial_schedule_time); - let new_edit = || RegionEdit { - files_to_add: vec![FileMeta { - region_id: region.region_id, - file_id: FileId::random(), - level: 0, - ..Default::default() - }], + let new_edit = |file_starts: &[i64]| RegionEdit { + files_to_add: file_starts + .iter() + .map(|start| FileMeta { + region_id: region.region_id, + file_id: FileId::random(), + time_range: ( + Timestamp::new_millisecond(*start), + Timestamp::new_millisecond(1000 * 1000), + ), + ..Default::default() + }) + .collect(), files_to_remove: vec![], timestamp_ms: None, compaction_time_window: None, @@ -115,19 +125,23 @@ async fn test_edit_region_schedule_compaction_with_format(flat_format: bool) { committed_sequence: None, }; engine - .edit_region(region.region_id, new_edit()) + .edit_region(region.region_id, new_edit(&[0, 10, 50, 80])) .await .unwrap(); // Asserts that the compaction of the region is not scheduled, // because the minimum time interval between two compactions is not passed. assert_eq!(rx.try_recv(), Err(oneshot::error::TryRecvError::Empty)); + assert_eq!( + initial_schedule_time, + region.last_schedule_compaction_millis() + ); // Simulates the time has passed the min compaction interval, - time_provider - .set_now(current_time_millis() + config.min_compaction_interval.as_millis() as i64); + let next_schedule_time = initial_time + config.min_compaction_interval.as_millis() as i64; + time_provider.set_now(next_schedule_time); // ... then edits the region again, engine - .edit_region(region.region_id, new_edit()) + .edit_region(region.region_id, new_edit(&[90])) .await .unwrap(); // ... finally asserts that the compaction of the region is scheduled. @@ -136,6 +150,9 @@ async fn test_edit_region_schedule_compaction_with_format(flat_format: bool) { .unwrap() .unwrap(); assert_eq!(region_id, actual); + // Wait for the `last_schedule_compaction_millis` to update. + tokio::time::sleep(Duration::from_millis(100)).await; + assert_eq!(next_schedule_time, region.last_schedule_compaction_millis()); } #[tokio::test] diff --git a/src/mito2/src/region.rs b/src/mito2/src/region.rs index 9d214caed3..4acec5a893 100644 --- a/src/mito2/src/region.rs +++ b/src/mito2/src/region.rs @@ -157,8 +157,8 @@ pub struct MitoRegion { pub(crate) provider: Provider, /// Last flush time in millis. last_flush_millis: AtomicI64, - /// Last compaction time in millis. - last_compaction_millis: AtomicI64, + /// Last schedule compaction time in millis. + last_schedule_compaction_millis: AtomicI64, /// Provider to get current time. time_provider: TimeProviderRef, /// The topic's latest entry id since the region's last flushing. @@ -251,15 +251,16 @@ impl MitoRegion { self.last_flush_millis.store(now, Ordering::Relaxed); } - /// Returns last compaction timestamp in millis. - pub(crate) fn last_compaction_millis(&self) -> i64 { - self.last_compaction_millis.load(Ordering::Relaxed) + /// Returns last schedule compaction timestamp in millis. + pub(crate) fn last_schedule_compaction_millis(&self) -> i64 { + self.last_schedule_compaction_millis.load(Ordering::Relaxed) } - /// Update compaction time to current time. - pub(crate) fn update_compaction_millis(&self) { + /// Update schedule compaction time to current time. + pub(crate) fn update_schedule_compaction_millis(&self) { let now = self.time_provider.current_time_millis(); - self.last_compaction_millis.store(now, Ordering::Relaxed); + self.last_schedule_compaction_millis + .store(now, Ordering::Relaxed); } /// Returns the table dir. @@ -1727,7 +1728,7 @@ mod tests { file_purger: crate::test_util::new_noop_file_purger(), provider: Provider::noop_provider(), last_flush_millis: Default::default(), - last_compaction_millis: Default::default(), + last_schedule_compaction_millis: Default::default(), time_provider: Arc::new(StdTimeProvider), topic_latest_entry_id: Default::default(), written_bytes: Arc::new(AtomicU64::new(0)), @@ -2084,7 +2085,7 @@ mod tests { file_purger: crate::test_util::new_noop_file_purger(), provider: Provider::noop_provider(), last_flush_millis: Default::default(), - last_compaction_millis: Default::default(), + last_schedule_compaction_millis: Default::default(), time_provider: Arc::new(StdTimeProvider), topic_latest_entry_id: Default::default(), written_bytes: Arc::new(AtomicU64::new(0)), diff --git a/src/mito2/src/region/opener.rs b/src/mito2/src/region/opener.rs index 6785181b48..3142a87c38 100644 --- a/src/mito2/src/region/opener.rs +++ b/src/mito2/src/region/opener.rs @@ -345,7 +345,7 @@ impl RegionOpener { ), provider, last_flush_millis: AtomicI64::new(now), - last_compaction_millis: AtomicI64::new(now), + last_schedule_compaction_millis: AtomicI64::new(now), time_provider: self.time_provider.clone(), topic_latest_entry_id: AtomicU64::new(0), written_bytes: Arc::new(AtomicU64::new(0)), @@ -581,7 +581,7 @@ impl RegionOpener { file_purger, provider: provider.clone(), last_flush_millis: AtomicI64::new(now), - last_compaction_millis: AtomicI64::new(now), + last_schedule_compaction_millis: AtomicI64::new(now), time_provider: self.time_provider.clone(), topic_latest_entry_id: AtomicU64::new(topic_latest_entry_id), written_bytes: Arc::new(AtomicU64::new(0)), diff --git a/src/mito2/src/worker/handle_compaction.rs b/src/mito2/src/worker/handle_compaction.rs index fd021771b1..3abdf0e349 100644 --- a/src/mito2/src/worker/handle_compaction.rs +++ b/src/mito2/src/worker/handle_compaction.rs @@ -13,7 +13,7 @@ // limitations under the License. use api::v1::region::compact_request; -use common_telemetry::{error, info, warn}; +use common_telemetry::{debug, error, info}; use store_api::logstore::LogStore; use store_api::region_request::RegionCompactRequest; use store_api::storage::RegionId; @@ -80,7 +80,6 @@ impl RegionWorkerLoop { return; } }; - region.update_compaction_millis(); region.version_control.apply_edit( Some(request.edit.clone()), @@ -118,6 +117,31 @@ impl RegionWorkerLoop { ) .await; self.handle_ddl_requests(&mut pending_ddls).await; + + if self.compaction_scheduler.is_compacting(region_id) { + return; + } + + let now = self.time_provider.current_time_millis(); + if now - region.last_schedule_compaction_millis() + >= self.config.min_compaction_interval.as_millis() as i64 + { + debug!( + "minimal compaction interval time {:?} has passed, scheduling next compaction", + self.config.min_compaction_interval + ); + if self + .compaction_scheduler + .schedule_next_compaction( + region_id, + ®ion.manifest_ctx, + self.schema_metadata_manager.clone(), + ) + .await + { + region.update_schedule_compaction_millis(); + } + } } pub(crate) async fn handle_compaction_cancelled( @@ -160,9 +184,14 @@ impl RegionWorkerLoop { return; } let now = self.time_provider.current_time_millis(); - if now - region.last_compaction_millis() + if now - region.last_schedule_compaction_millis() >= self.config.min_compaction_interval.as_millis() as i64 - && let Err(e) = self + { + debug!( + "minimal compaction interval time {:?} has passed, scheduling next compaction", + self.config.min_compaction_interval + ); + match self .compaction_scheduler .schedule_compaction( region.region_id, @@ -175,11 +204,13 @@ impl RegionWorkerLoop { 1, // Default for automatic compaction ) .await - { - warn!( - "Failed to schedule compaction for region: {}, err: {}", - region.region_id, e - ); + { + Ok(true) => region.update_schedule_compaction_millis(), + Ok(false) => {} + Err(e) => { + error!(e; "Failed to schedule compaction for region: {}", region.region_id) + } + } } } } From a796dbf1a0759cd0bcc62ad6122c8164cd7c5d04 Mon Sep 17 00:00:00 2001 From: shuiyisong <113876041+shuiyisong@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:45:44 +0800 Subject: [PATCH 07/29] chore: update opendal to 0.57 (#8204) * chore: update opendal to 0.57 Signed-off-by: shuiyisong * chore: remove duplicate compat to use opendal's Signed-off-by: shuiyisong --------- Signed-off-by: shuiyisong --- Cargo.lock | 87 +- Cargo.toml | 2 +- src/common/datasource/Cargo.toml | 1 + src/common/datasource/src/file_format.rs | 2 +- .../datasource/src/file_format/tests.rs | 2 +- src/common/datasource/src/object_store/oss.rs | 24 +- src/common/datasource/src/test_util.rs | 4 +- src/file-engine/Cargo.toml | 1 + src/file-engine/src/query/file_stream.rs | 2 +- src/object-store/Cargo.toml | 2 +- src/object-store/src/compat.rs | 1045 ----------------- src/object-store/src/layers/mock.rs | 30 +- src/object-store/src/lib.rs | 1 - 13 files changed, 98 insertions(+), 1105 deletions(-) delete mode 100644 src/object-store/src/compat.rs diff --git a/Cargo.lock b/Cargo.lock index 63ba289947..7c6cb93c2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2278,6 +2278,7 @@ dependencies = [ "futures", "lazy_static", "object-store", + "object_store_opendal", "orc-rust", "parquet", "paste", @@ -5102,6 +5103,7 @@ dependencies = [ "datatypes", "futures", "object-store", + "object_store_opendal", "serde", "serde_json", "snafu 0.8.6", @@ -9074,8 +9076,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eb12a624a41fce745838d0ef3701ff6c47797c13cd18ad3612fd2a3134fdbd8" dependencies = [ "async-trait", "bytes", @@ -9162,8 +9165,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendal" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96c9c85ce253ff87225e7669979d877a20c98a06604ec9d6dd5f4473e08f1ae1" dependencies = [ "ctor", "opendal-core", @@ -9183,8 +9187,9 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4f8607c90e2c963a91467f50fb49fbc7fb3d573f88cea219ca59ccd3740b309" dependencies = [ "anyhow", "base64 0.22.1", @@ -9210,8 +9215,9 @@ dependencies = [ [[package]] name = "opendal-layer-concurrent-limit" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d6f81ba6960e3fae1882f253b114b21d7e444e1534f209c7737a79f6243eb6f" dependencies = [ "futures", "http 1.3.1", @@ -9221,8 +9227,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58ada45c6d81d1aa4c9305d0c7d4bc317c59c85866a0908a2d75a7a978aa5ee2" dependencies = [ "log", "opendal-core", @@ -9230,8 +9237,9 @@ dependencies = [ [[package]] name = "opendal-layer-observe-metrics-common" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "628b0228fdbd13c3d9d50eee4341f2eb82ca5b44991e4c68f07c84cc823e2d12" dependencies = [ "futures", "http 1.3.1", @@ -9240,8 +9248,9 @@ dependencies = [ [[package]] name = "opendal-layer-prometheus" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0487bdb1357097ec8654781bad03ef310282517738e2864ebde69e27aaafc5ec" dependencies = [ "opendal-core", "opendal-layer-observe-metrics-common", @@ -9250,8 +9259,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2a25a718afb81fad81cb9a0580a1cb989221fa2317f888c6a37f8dad408eb7" dependencies = [ "backon", "log", @@ -9260,8 +9270,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e91f731724c213af81e9d03517859c8fc47b4578e64ad61ae4f099f10fe36e3" dependencies = [ "opendal-core", "tokio", @@ -9269,8 +9280,9 @@ dependencies = [ [[package]] name = "opendal-layer-tracing" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90c6fc9df6da1f0dafbdf55fa48525f1643aefbe7da8f46936e869e2a5b8a34f" dependencies = [ "futures", "http 1.3.1", @@ -9280,8 +9292,9 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0030644366ef5d8cbe3a4a5822bf99a4aafddc1666e9d24b44d158d9062fc76a" dependencies = [ "base64 0.22.1", "bytes", @@ -9300,8 +9313,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b489f13c42e69d69bdd72952b634356ec43a7881a20259b38b540fcecdf4051" dependencies = [ "http 1.3.1", "opendal-core", @@ -9309,8 +9323,9 @@ dependencies = [ [[package]] name = "opendal-service-fs" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e89a665fef0e6bd249cf5ea47fc174b7ba892159bee4b9382528b1ca873a2c" dependencies = [ "bytes", "log", @@ -9322,8 +9337,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48de101aac565ed06af4b47903c24eafd249075553ec1fb18256751c45148d47" dependencies = [ "async-trait", "bytes", @@ -9342,8 +9358,9 @@ dependencies = [ [[package]] name = "opendal-service-http" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb6af628a0bf14075b957179444927e1df40dc7addef382b585a05ef015a077b" dependencies = [ "http 1.3.1", "log", @@ -9353,8 +9370,9 @@ dependencies = [ [[package]] name = "opendal-service-oss" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "328fa55e8888cbdfe00826bfea2a79042422b720e8369e9e021e46121dea5ace" dependencies = [ "bytes", "http 1.3.1", @@ -9369,8 +9387,9 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.56.0" -source = "git+https://github.com/apache/opendal.git?rev=4ad2d85296ffa6fdc2882f97d3c760ee243913f7#4ad2d85296ffa6fdc2882f97d3c760ee243913f7" +version = "0.57.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "313d46c9f5ae70bca26b7c3e3fbb9b639292625f28af73aa016f47e788af9deb" dependencies = [ "base64 0.22.1", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 32407f31cf..56200a24d6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -178,7 +178,7 @@ nalgebra = "0.33" nix = { version = "0.30.1", default-features = false, features = ["event", "fs", "process"] } notify = "8.0" num_cpus = "1.16" -object_store_opendal = { git = "https://github.com/apache/opendal.git", rev = "4ad2d85296ffa6fdc2882f97d3c760ee243913f7" } +object_store_opendal = "0.57" once_cell = "1.18" opentelemetry-proto = { version = "0.31", features = [ "gen-tonic", diff --git a/src/common/datasource/Cargo.toml b/src/common/datasource/Cargo.toml index 470b5371f7..8b4053db2f 100644 --- a/src/common/datasource/Cargo.toml +++ b/src/common/datasource/Cargo.toml @@ -33,6 +33,7 @@ datatypes.workspace = true futures.workspace = true lazy_static.workspace = true object-store.workspace = true +object_store_opendal.workspace = true orc-rust = { version = "0.8", default-features = false, features = ["async"] } parquet.workspace = true paste.workspace = true diff --git a/src/common/datasource/src/file_format.rs b/src/common/datasource/src/file_format.rs index a6a358c9e4..e36f94c0d2 100644 --- a/src/common/datasource/src/file_format.rs +++ b/src/common/datasource/src/file_format.rs @@ -316,7 +316,7 @@ pub async fn file_to_stream( .with_file_compression_type(df_compression) .build(); - let store = Arc::new(object_store::compat::OpendalStore::new(store.clone())); + let store = Arc::new(object_store_opendal::OpendalStore::new(store.clone())); let file_opener = config.file_source().create_file_opener(store, &config, 0)?; let stream = FileStream::new(&config, 0, file_opener, &ExecutionPlanMetricsSet::new())?; diff --git a/src/common/datasource/src/file_format/tests.rs b/src/common/datasource/src/file_format/tests.rs index 93ab3b4409..a925f73d48 100644 --- a/src/common/datasource/src/file_format/tests.rs +++ b/src/common/datasource/src/file_format/tests.rs @@ -44,7 +44,7 @@ struct Test<'a> { impl Test<'_> { async fn run(self, store: &ObjectStore) { - let store = Arc::new(object_store::compat::OpendalStore::new(store.clone())); + let store = Arc::new(object_store_opendal::OpendalStore::new(store.clone())); let file_opener = self .file_source .create_file_opener(store, &self.config, 0) diff --git a/src/common/datasource/src/object_store/oss.rs b/src/common/datasource/src/object_store/oss.rs index aded3eca2c..aacc17ac5e 100644 --- a/src/common/datasource/src/object_store/oss.rs +++ b/src/common/datasource/src/object_store/oss.rs @@ -27,12 +27,14 @@ const ACCESS_KEY_ID: &str = "access_key_id"; const ACCESS_KEY_SECRET: &str = "access_key_secret"; const ROOT: &str = "root"; const ALLOW_ANONYMOUS: &str = "allow_anonymous"; +const SKIP_SIGNATURE: &str = "skip_signature"; /// Check if the key is supported in OSS configuration. pub fn is_supported_in_oss(key: &str) -> bool { [ ROOT, ALLOW_ANONYMOUS, + SKIP_SIGNATURE, BUCKET, ENDPOINT, ACCESS_KEY_ID, @@ -61,18 +63,23 @@ pub fn build_oss_backend( builder = builder.access_key_secret(access_key_secret); } - if let Some(allow_anonymous) = connection.get(ALLOW_ANONYMOUS) { - let allow = allow_anonymous.as_str().parse::().map_err(|e| { + if let Some((key, value)) = connection + .get(SKIP_SIGNATURE) + .map(|value| (SKIP_SIGNATURE, value)) + .or_else(|| { + connection + .get(ALLOW_ANONYMOUS) + .map(|value| (ALLOW_ANONYMOUS, value)) + }) + { + let skip_signature = value.as_str().parse::().map_err(|e| { error::InvalidConnectionSnafu { - msg: format!( - "failed to parse the option {}={}, {}", - ALLOW_ANONYMOUS, allow_anonymous, e - ), + msg: format!("failed to parse the option {}={}, {}", key, value, e), } .build() })?; - if allow { - builder = builder.allow_anonymous(); + if skip_signature { + builder = builder.skip_signature(); } } @@ -93,6 +100,7 @@ mod tests { fn test_is_supported_in_oss() { assert!(is_supported_in_oss(ROOT)); assert!(is_supported_in_oss(ALLOW_ANONYMOUS)); + assert!(is_supported_in_oss(SKIP_SIGNATURE)); assert!(is_supported_in_oss(BUCKET)); assert!(is_supported_in_oss(ENDPOINT)); assert!(is_supported_in_oss(ACCESS_KEY_ID)); diff --git a/src/common/datasource/src/test_util.rs b/src/common/datasource/src/test_util.rs index 0a13d9c6e8..ea2b0c768c 100644 --- a/src/common/datasource/src/test_util.rs +++ b/src/common/datasource/src/test_util.rs @@ -103,7 +103,7 @@ pub async fn setup_stream_to_json_test(origin_path: &str, threshold: impl Fn(usi test_util::TEST_BATCH_SIZE, schema.clone(), FileCompressionType::UNCOMPRESSED, - Arc::new(object_store::compat::OpendalStore::new(store.clone())), + Arc::new(object_store_opendal::OpendalStore::new(store.clone())), true, ); @@ -157,7 +157,7 @@ pub async fn setup_stream_to_csv_test( let csv_opener = csv_source .create_file_opener( - Arc::new(object_store::compat::OpendalStore::new(store.clone())), + Arc::new(object_store_opendal::OpendalStore::new(store.clone())), &config, 0, ) diff --git a/src/file-engine/Cargo.toml b/src/file-engine/Cargo.toml index 6c8c9e887d..9d031cb279 100644 --- a/src/file-engine/Cargo.toml +++ b/src/file-engine/Cargo.toml @@ -29,6 +29,7 @@ datafusion-expr.workspace = true datatypes.workspace = true futures.workspace = true object-store.workspace = true +object_store_opendal.workspace = true serde = { version = "1.0", features = ["derive"] } serde_json.workspace = true snafu.workspace = true diff --git a/src/file-engine/src/query/file_stream.rs b/src/file-engine/src/query/file_stream.rs index eec8f8961d..a480a50374 100644 --- a/src/file-engine/src/query/file_stream.rs +++ b/src/file-engine/src/query/file_stream.rs @@ -61,7 +61,7 @@ fn build_record_batch_stream( .with_file_group(FileGroup::new(files)) .build(); - let store = Arc::new(object_store::compat::OpendalStore::new( + let store = Arc::new(object_store_opendal::OpendalStore::new( scan_plan_config.store.clone(), )); diff --git a/src/object-store/Cargo.toml b/src/object-store/Cargo.toml index 0815f066ba..9ca68bb780 100644 --- a/src/object-store/Cargo.toml +++ b/src/object-store/Cargo.toml @@ -24,7 +24,7 @@ derive_builder = { workspace = true, optional = true } futures.workspace = true humantime-serde.workspace = true lazy_static.workspace = true -opendal = { git = "https://github.com/apache/opendal.git", rev = "4ad2d85296ffa6fdc2882f97d3c760ee243913f7", features = [ +opendal = { version = "0.57", features = [ "layers-tracing", "layers-prometheus", "services-azblob", diff --git a/src/object-store/src/compat.rs b/src/object-store/src/compat.rs deleted file mode 100644 index 4498f8f3be..0000000000 --- a/src/object-store/src/compat.rs +++ /dev/null @@ -1,1045 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use std::collections::HashMap; -use std::fmt::{self, Debug, Display, Formatter}; -use std::future::IntoFuture; -use std::io; -use std::ops::Range; -use std::sync::Arc; - -use async_trait::async_trait; -use bytes::Bytes; -use datafusion_object_store::path::Path; -use datafusion_object_store::{ - Attribute, Attributes, CopyMode, CopyOptions, GetOptions, GetRange, GetResult, - GetResultPayload, ListResult, MultipartUpload, ObjectMeta, ObjectStore as ArrowObjectStore, - PutMode, PutMultipartOptions, PutOptions, PutPayload, PutResult, UploadPart, -}; -use futures::stream::BoxStream; -use futures::{FutureExt, StreamExt, TryStreamExt}; -use opendal::options::CopyOptions as OpendalCopyOptions; -use opendal::raw::percent_decode_path; -use opendal::{Buffer, Operator, OperatorInfo, Writer}; -use tokio::sync::{Mutex, oneshot}; - -/// OpendalStore implements ObjectStore trait by using opendal. -/// -/// This allows users to use opendal as an object store without extra cost. -/// -/// Visit [`opendal::services`] for more information about supported services. -/// -/// ```no_run -/// use std::sync::Arc; -/// -/// use bytes::Bytes; -/// use object_store::path::Path; -/// use object_store::ObjectStore; -/// use object_store_opendal::OpendalStore; -/// use opendal::services::S3; -/// use opendal::{Builder, Operator}; -/// -/// #[tokio::main] -/// async fn main() { -/// let builder = S3::default() -/// .access_key_id("my_access_key") -/// .secret_access_key("my_secret_key") -/// .endpoint("my_endpoint") -/// .region("my_region"); -/// -/// // Create a new operator -/// let operator = Operator::new(builder).unwrap().finish(); -/// -/// // Create a new object store -/// let object_store = Arc::new(OpendalStore::new(operator)); -/// -/// let path = Path::from("data/nested/test.txt"); -/// let bytes = Bytes::from_static(b"hello, world! I am nested."); -/// -/// object_store.put(&path, bytes.clone().into()).await.unwrap(); -/// -/// let content = object_store -/// .get(&path) -/// .await -/// .unwrap() -/// .bytes() -/// .await -/// .unwrap(); -/// -/// assert_eq!(content, bytes); -/// } -/// ``` -#[derive(Clone)] -pub struct OpendalStore { - info: Arc, - inner: Operator, -} - -impl OpendalStore { - /// Create OpendalStore by given Operator. - pub fn new(op: Operator) -> Self { - Self { - info: op.info().into(), - inner: op, - } - } - - /// Get the Operator info. - pub fn info(&self) -> &OperatorInfo { - self.info.as_ref() - } - - /// Copy a file from one location to another. - async fn copy_request( - &self, - from: &Path, - to: &Path, - if_not_exists: bool, - ) -> datafusion_object_store::Result<()> { - let mut copy_options = OpendalCopyOptions::default(); - if if_not_exists { - copy_options.if_not_exists = true; - } - - // Perform the copy operation - self.inner - .copy_options( - &percent_decode_path(from.as_ref()), - &percent_decode_path(to.as_ref()), - copy_options, - ) - .await - .map_err(|err| { - if if_not_exists && err.kind() == opendal::ErrorKind::AlreadyExists { - datafusion_object_store::Error::AlreadyExists { - path: to.to_string(), - source: Box::new(err), - } - } else { - format_object_store_error(err, from.as_ref()) - } - })?; - - Ok(()) - } -} - -impl Debug for OpendalStore { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("OpendalStore") - .field("scheme", &self.info.scheme()) - .field("name", &self.info.name()) - .field("root", &self.info.root()) - .field("capability", &self.info.full_capability()) - .finish() - } -} - -impl Display for OpendalStore { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - let info = self.inner.info(); - write!( - f, - "Opendal({}, bucket={}, root={})", - info.scheme(), - info.name(), - info.root() - ) - } -} - -impl From for OpendalStore { - fn from(value: Operator) -> Self { - Self::new(value) - } -} - -#[async_trait] -impl ArrowObjectStore for OpendalStore { - async fn put_opts( - &self, - location: &Path, - bytes: PutPayload, - opts: PutOptions, - ) -> datafusion_object_store::Result { - let decoded_location = percent_decode_path(location.as_ref()); - let mut future_write = self - .inner - .write_with(&decoded_location, Buffer::from_iter(bytes)); - let opts_mode = opts.mode.clone(); - match opts.mode { - PutMode::Overwrite => {} - PutMode::Create => { - future_write = future_write.if_not_exists(true); - } - PutMode::Update(update_version) => { - let Some(etag) = update_version.e_tag else { - return Err(datafusion_object_store::Error::NotSupported { - source: Box::new(opendal::Error::new( - opendal::ErrorKind::Unsupported, - "etag is required for conditional put", - )), - }); - }; - future_write = future_write.if_match(etag.as_str()); - } - } - let rp = future_write.await.map_err(|err| { - match format_object_store_error(err, location.as_ref()) { - datafusion_object_store::Error::Precondition { path, source } - if opts_mode == PutMode::Create => - { - datafusion_object_store::Error::AlreadyExists { path, source } - } - e => e, - } - })?; - - let e_tag = rp.etag().map(|s| s.to_string()); - let version = rp.version().map(|s| s.to_string()); - - Ok(PutResult { e_tag, version }) - } - - async fn put_multipart_opts( - &self, - location: &Path, - opts: PutMultipartOptions, - ) -> datafusion_object_store::Result> { - const DEFAULT_CONCURRENT: usize = 8; - - let mut options = opendal::options::WriteOptions { - concurrent: DEFAULT_CONCURRENT, - ..Default::default() - }; - - let mut user_metadata = HashMap::new(); - - for (key, value) in opts.attributes.iter() { - match key { - Attribute::CacheControl => { - options.cache_control = Some(value.to_string()); - } - Attribute::ContentDisposition => { - options.content_disposition = Some(value.to_string()); - } - Attribute::ContentEncoding => { - options.content_encoding = Some(value.to_string()); - } - Attribute::ContentLanguage => continue, - Attribute::ContentType => { - options.content_type = Some(value.to_string()); - } - Attribute::Metadata(k) => { - user_metadata.insert(k.to_string(), value.to_string()); - } - _ => {} - } - } - - if !user_metadata.is_empty() { - options.user_metadata = Some(user_metadata); - } - - let decoded_location = percent_decode_path(location.as_ref()); - let writer = self - .inner - .writer_options(&decoded_location, options) - .await - .map_err(|err| format_object_store_error(err, location.as_ref()))?; - let upload = OpendalMultipartUpload::new(writer, location.clone()); - - Ok(Box::new(upload)) - } - - async fn get_opts( - &self, - location: &Path, - options: GetOptions, - ) -> datafusion_object_store::Result { - let raw_location = percent_decode_path(location.as_ref()); - let meta = { - let mut s = self.inner.stat_with(&raw_location); - if let Some(version) = &options.version { - s = s.version(version.as_str()) - } - if let Some(if_match) = &options.if_match { - s = s.if_match(if_match.as_str()); - } - if let Some(if_none_match) = &options.if_none_match { - s = s.if_none_match(if_none_match.as_str()); - } - if let Some(if_modified_since) = - options.if_modified_since.and_then(datetime_to_timestamp) - { - s = s.if_modified_since(if_modified_since); - } - if let Some(if_unmodified_since) = - options.if_unmodified_since.and_then(datetime_to_timestamp) - { - s = s.if_unmodified_since(if_unmodified_since); - } - s.await - .map_err(|err| format_object_store_error(err, location.as_ref()))? - }; - - let mut attributes = Attributes::new(); - if let Some(user_meta) = meta.user_metadata() { - for (key, value) in user_meta { - attributes.insert( - Attribute::Metadata(key.clone().into()), - value.clone().into(), - ); - } - } - - let meta = ObjectMeta { - location: location.clone(), - last_modified: meta - .last_modified() - .and_then(timestamp_to_datetime) - .unwrap_or_default(), - size: meta.content_length(), - e_tag: meta.etag().map(|x| x.to_string()), - version: meta.version().map(|x| x.to_string()), - }; - - if options.head { - return Ok(GetResult { - payload: GetResultPayload::Stream(Box::pin(futures::stream::empty())), - range: 0..0, - meta, - attributes, - }); - } - - let reader = { - let mut r = self.inner.reader_with(raw_location.as_ref()); - if let Some(version) = options.version { - r = r.version(version.as_str()); - } - if let Some(if_match) = options.if_match { - r = r.if_match(if_match.as_str()); - } - if let Some(if_none_match) = options.if_none_match { - r = r.if_none_match(if_none_match.as_str()); - } - if let Some(if_modified_since) = - options.if_modified_since.and_then(datetime_to_timestamp) - { - r = r.if_modified_since(if_modified_since); - } - if let Some(if_unmodified_since) = - options.if_unmodified_since.and_then(datetime_to_timestamp) - { - r = r.if_unmodified_since(if_unmodified_since); - } - r.await - .map_err(|err| format_object_store_error(err, location.as_ref()))? - }; - - let read_range = match options.range { - Some(GetRange::Bounded(r)) => { - if r.start >= r.end || r.start >= meta.size { - 0..0 - } else { - let end = r.end.min(meta.size); - r.start..end - } - } - Some(GetRange::Offset(r)) => { - if r < meta.size { - r..meta.size - } else { - 0..0 - } - } - Some(GetRange::Suffix(r)) if r < meta.size => (meta.size - r)..meta.size, - _ => 0..meta.size, - }; - - let stream = reader - .into_bytes_stream(read_range.start..read_range.end) - .await - .map_err(|err| format_object_store_error(err, location.as_ref()))? - .map_ok(|buf| buf) - .map_err(|err: io::Error| datafusion_object_store::Error::Generic { - store: "IoError", - source: Box::new(err), - }); - - Ok(GetResult { - payload: GetResultPayload::Stream(Box::pin(stream)), - range: read_range.start..read_range.end, - meta, - attributes, - }) - } - - async fn get_ranges( - &self, - location: &Path, - ranges: &[Range], - ) -> datafusion_object_store::Result> { - if ranges.is_empty() { - return Ok(Vec::new()); - } - - let raw_location = percent_decode_path(location.as_ref()); - let reader = self - .inner - .reader_with(raw_location.as_ref()) - .await - .map_err(|err| format_object_store_error(err, location.as_ref()))?; - let buffers = reader - .fetch(ranges.to_vec()) - .await - .map_err(|err| format_object_store_error(err, location.as_ref()))?; - - Ok(buffers.into_iter().map(|buf| buf.to_bytes()).collect()) - } - - fn delete_stream( - &self, - locations: BoxStream<'static, datafusion_object_store::Result>, - ) -> BoxStream<'static, datafusion_object_store::Result> { - let this = self.clone(); - locations - .map(move |location| { - let this = this.clone(); - async move { - let location = location?; - let decoded_location = percent_decode_path(location.as_ref()); - this.inner - .delete(&decoded_location) - .await - .map_err(|err| format_object_store_error(err, location.as_ref()))?; - Ok(location) - } - }) - .buffered(10) - .boxed() - } - - fn list( - &self, - prefix: Option<&Path>, - ) -> BoxStream<'static, datafusion_object_store::Result> { - // object_store `Path` always removes trailing slash - // need to add it back - let path = prefix.map_or("".into(), |x| { - format!("{}/", percent_decode_path(x.as_ref())) - }); - - let this = self.clone(); - let fut = async move { - let stream = this - .inner - .lister_with(&path) - .recursive(true) - .await - .map_err(|err| format_object_store_error(err, &path))?; - - let stream = stream.then(|res| async { - let entry = res.map_err(|err| format_object_store_error(err, ""))?; - let meta = entry.metadata(); - - Ok(format_object_meta(entry.path(), meta)) - }); - Ok::<_, datafusion_object_store::Error>(stream) - }; - - fut.into_stream().try_flatten().boxed() - } - - fn list_with_offset( - &self, - prefix: Option<&Path>, - offset: &Path, - ) -> BoxStream<'static, datafusion_object_store::Result> { - let path = prefix.map_or("".into(), |x| { - format!("{}/", percent_decode_path(x.as_ref())) - }); - let offset = offset.clone(); - - // clone self for 'static lifetime - // clone self is cheap - let this = self.clone(); - - let fut = async move { - let list_with_start_after = this.inner.info().full_capability().list_with_start_after; - let mut fut = this.inner.lister_with(&path).recursive(true); - - // Use native start_after support if possible. - if list_with_start_after { - fut = fut.start_after(offset.as_ref()); - } - - let lister = fut - .await - .map_err(|err| format_object_store_error(err, &path))? - .then(move |entry| { - let path = path.clone(); - let this = this.clone(); - async move { - let entry = entry.map_err(|err| format_object_store_error(err, &path))?; - let (path, metadata) = entry.into_parts(); - - // If it's a dir or last_modified is present, we can use it directly. - if metadata.is_dir() || metadata.last_modified().is_some() { - let object_meta = format_object_meta(&path, &metadata); - return Ok(object_meta); - } - - let metadata = this - .inner - .stat(&path) - .await - .map_err(|err| format_object_store_error(err, &path))?; - let object_meta = format_object_meta(&path, &metadata); - Ok::<_, datafusion_object_store::Error>(object_meta) - } - }) - .boxed(); - - let stream = if list_with_start_after { - lister - } else { - lister - .try_filter(move |entry| futures::future::ready(entry.location > offset)) - .boxed() - }; - - Ok::<_, datafusion_object_store::Error>(stream) - }; - - fut.into_stream().try_flatten().boxed() - } - - async fn list_with_delimiter( - &self, - prefix: Option<&Path>, - ) -> datafusion_object_store::Result { - let path = prefix.map_or("".into(), |x| { - format!("{}/", percent_decode_path(x.as_ref())) - }); - let mut stream = self - .inner - .lister_with(&path) - .into_future() - .await - .map_err(|err| format_object_store_error(err, &path))?; - - let mut common_prefixes = Vec::new(); - let mut objects = Vec::new(); - - while let Some(res) = stream.next().await { - let entry = res.map_err(|err| format_object_store_error(err, ""))?; - let meta = entry.metadata(); - - if meta.is_dir() { - common_prefixes.push(entry.path().into()); - } else if meta.last_modified().is_some() { - objects.push(format_object_meta(entry.path(), meta)); - } else { - let meta = self - .inner - .stat(entry.path()) - .await - .map_err(|err| format_object_store_error(err, entry.path()))?; - objects.push(format_object_meta(entry.path(), &meta)); - } - } - - Ok(ListResult { - common_prefixes, - objects, - }) - } - - async fn copy_opts( - &self, - from: &Path, - to: &Path, - options: CopyOptions, - ) -> datafusion_object_store::Result<()> { - let if_not_exists = options.mode == CopyMode::Create; - self.copy_request(from, to, if_not_exists).await - } -} - -/// `MultipartUpload` implementation based on `Writer` in opendal. -/// -/// # Notes -/// -/// OpenDAL writer can handle concurrent internally we don't generate real `UploadPart` like existing -/// implementation do. Instead, we just write the part and notify the next task to be written. -/// -/// The lock here doesn't really involve the write process, it's just for the notify mechanism. -struct OpendalMultipartUpload { - writer: Arc>, - location: Path, - next_notify: oneshot::Receiver<()>, -} - -impl OpendalMultipartUpload { - fn new(writer: Writer, location: Path) -> Self { - // an immediately dropped sender for the first part to write without waiting - let (_, rx) = oneshot::channel(); - - Self { - writer: Arc::new(Mutex::new(writer)), - location, - next_notify: rx, - } - } -} - -#[async_trait] -impl MultipartUpload for OpendalMultipartUpload { - fn put_part(&mut self, data: PutPayload) -> UploadPart { - let writer = self.writer.clone(); - let location = self.location.clone(); - - // Generate next notify which will be notified after the current part is written. - let (tx, rx) = oneshot::channel(); - // Fetch the notify for current part to wait for it to be written. - let last_rx = std::mem::replace(&mut self.next_notify, rx); - - async move { - // Wait for the previous part to be written - let _ = last_rx.await; - - let mut writer = writer.lock().await; - let result = writer - .write(Buffer::from_iter(data)) - .await - .map_err(|err| format_object_store_error(err, location.as_ref())); - - // Notify the next part to be written - drop(tx); - - result - } - .boxed() - } - - async fn complete(&mut self) -> datafusion_object_store::Result { - let mut writer = self.writer.lock().await; - let metadata = writer - .close() - .await - .map_err(|err| format_object_store_error(err, self.location.as_ref()))?; - - let e_tag = metadata.etag().map(|s| s.to_string()); - let version = metadata.version().map(|s| s.to_string()); - - Ok(PutResult { e_tag, version }) - } - - async fn abort(&mut self) -> datafusion_object_store::Result<()> { - let mut writer = self.writer.lock().await; - writer - .abort() - .await - .map_err(|err| format_object_store_error(err, self.location.as_ref())) - } -} - -impl Debug for OpendalMultipartUpload { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - f.debug_struct("OpendalMultipartUpload") - .field("location", &self.location) - .finish() - } -} - -fn format_object_store_error(err: opendal::Error, path: &str) -> datafusion_object_store::Error { - match err.kind() { - opendal::ErrorKind::NotFound => datafusion_object_store::Error::NotFound { - path: path.to_string(), - source: Box::new(err), - }, - opendal::ErrorKind::Unsupported => datafusion_object_store::Error::NotSupported { - source: Box::new(err), - }, - opendal::ErrorKind::AlreadyExists => datafusion_object_store::Error::AlreadyExists { - path: path.to_string(), - source: Box::new(err), - }, - opendal::ErrorKind::ConditionNotMatch => datafusion_object_store::Error::Precondition { - path: path.to_string(), - source: Box::new(err), - }, - kind => datafusion_object_store::Error::Generic { - store: kind.into_static(), - source: Box::new(err), - }, - } -} - -fn format_object_meta(path: &str, meta: &opendal::Metadata) -> ObjectMeta { - ObjectMeta { - location: path.into(), - last_modified: meta - .last_modified() - .and_then(timestamp_to_datetime) - .unwrap_or_default(), - size: meta.content_length(), - e_tag: meta.etag().map(|x| x.to_string()), - version: meta.version().map(|x| x.to_string()), - } -} - -fn timestamp_to_datetime(ts: opendal::raw::Timestamp) -> Option> { - let ts = ts.into_inner(); - chrono::DateTime::::from_timestamp(ts.as_second(), ts.subsec_nanosecond() as u32) -} - -fn datetime_to_timestamp(dt: chrono::DateTime) -> Option { - opendal::raw::Timestamp::new(dt.timestamp(), dt.timestamp_subsec_nanos() as i32).ok() -} - -#[cfg(test)] -mod tests { - use std::sync::Arc; - - use bytes::Bytes; - use datafusion_object_store::path::Path; - use datafusion_object_store::{ - ObjectStore as ArrowObjectStore, ObjectStoreExt, WriteMultipart, - }; - use opendal::{Operator, services}; - use rand::{Rng, RngCore}; - - use super::*; - - async fn create_test_object_store() -> Arc { - let op = Operator::new(services::Memory::default()).unwrap().finish(); - let object_store = Arc::new(OpendalStore::new(op)); - - let path: Path = "data/test.txt".into(); - let bytes = Bytes::from_static(b"hello, world!"); - object_store.put(&path, bytes.into()).await.unwrap(); - - let path: Path = "data/nested/test.txt".into(); - let bytes = Bytes::from_static(b"hello, world! I am nested."); - object_store.put(&path, bytes.into()).await.unwrap(); - - object_store - } - - #[tokio::test] - async fn test_basic() { - let op = Operator::new(services::Memory::default()).unwrap().finish(); - let object_store: Arc = Arc::new(OpendalStore::new(op)); - - // Retrieve a specific file - let path: Path = "data/test.txt".into(); - - let bytes = Bytes::from_static(b"hello, world!"); - object_store.put(&path, bytes.clone().into()).await.unwrap(); - - let meta = object_store.head(&path).await.unwrap(); - - assert_eq!(meta.size, 13); - - assert_eq!( - object_store - .get(&path) - .await - .unwrap() - .bytes() - .await - .unwrap(), - bytes - ); - } - - #[tokio::test] - async fn test_put_multipart() { - let op = Operator::new(services::Memory::default()).unwrap().finish(); - let object_store: Arc = Arc::new(OpendalStore::new(op)); - - let mut rng = rand::rng(); - - // Case complete - let path: Path = "data/test_complete.txt".into(); - let upload = object_store.put_multipart(&path).await.unwrap(); - - let mut write = WriteMultipart::new(upload); - - let mut all_bytes = vec![]; - let round = rng.random_range(1..=1024); - for _ in 0..round { - let size = rng.random_range(1..=1024); - let mut bytes = vec![0; size]; - rng.fill_bytes(&mut bytes); - - all_bytes.extend_from_slice(&bytes); - write.put(bytes.into()); - } - - let _ = write.finish().await.unwrap(); - - let meta = object_store.head(&path).await.unwrap(); - - assert_eq!(meta.size, all_bytes.len() as u64); - - assert_eq!( - object_store - .get(&path) - .await - .unwrap() - .bytes() - .await - .unwrap(), - Bytes::from(all_bytes) - ); - - // Case abort - let path: Path = "data/test_abort.txt".into(); - let mut upload = object_store.put_multipart(&path).await.unwrap(); - upload.put_part(vec![1; 1024].into()).await.unwrap(); - upload.abort().await.unwrap(); - - let res = object_store.head(&path).await; - let err = res.unwrap_err(); - - assert!(matches!( - err, - datafusion_object_store::Error::NotFound { .. } - )) - } - - #[tokio::test] - async fn test_list() { - let object_store = create_test_object_store().await; - let path: Path = "data/".into(); - let results = object_store.list(Some(&path)).collect::>().await; - assert_eq!(results.len(), 2); - let mut locations = results - .iter() - .map(|x| x.as_ref().unwrap().location.as_ref()) - .collect::>(); - - let expected_files = vec![ - ( - "data/nested/test.txt", - Bytes::from_static(b"hello, world! I am nested."), - ), - ("data/test.txt", Bytes::from_static(b"hello, world!")), - ]; - - let expected_locations = expected_files.iter().map(|x| x.0).collect::>(); - - locations.sort(); - assert_eq!(locations, expected_locations); - - for (location, bytes) in expected_files { - let path: Path = location.into(); - assert_eq!( - object_store - .get(&path) - .await - .unwrap() - .bytes() - .await - .unwrap(), - bytes - ); - } - } - - #[tokio::test] - async fn test_list_with_delimiter() { - let object_store = create_test_object_store().await; - let path: Path = "data/".into(); - let result = object_store.list_with_delimiter(Some(&path)).await.unwrap(); - assert_eq!(result.objects.len(), 1); - assert_eq!(result.common_prefixes.len(), 1); - assert_eq!(result.objects[0].location.as_ref(), "data/test.txt"); - assert_eq!(result.common_prefixes[0].as_ref(), "data/nested"); - } - - #[tokio::test] - async fn test_list_with_offset() { - let object_store = create_test_object_store().await; - let path: Path = "data/".into(); - let offset: Path = "data/nested/test.txt".into(); - let result = object_store - .list_with_offset(Some(&path), &offset) - .collect::>() - .await; - assert_eq!(result.len(), 1); - assert_eq!( - result[0].as_ref().unwrap().location.as_ref(), - "data/test.txt" - ); - } - - mod stat_counter { - use std::sync::atomic::{AtomicUsize, Ordering}; - - use super::*; - - #[derive(Debug, Clone)] - pub struct StatCounterLayer { - count: Arc, - } - - impl StatCounterLayer { - pub fn new(count: Arc) -> Self { - Self { count } - } - } - - impl opendal::raw::Layer for StatCounterLayer { - type LayeredAccess = StatCounterAccessor; - - fn layer(&self, inner: A) -> Self::LayeredAccess { - StatCounterAccessor { - inner, - count: self.count.clone(), - } - } - } - - #[derive(Debug, Clone)] - pub struct StatCounterAccessor { - inner: A, - count: Arc, - } - - impl opendal::raw::LayeredAccess for StatCounterAccessor { - type Inner = A; - type Reader = A::Reader; - type Writer = A::Writer; - type Lister = A::Lister; - type Deleter = A::Deleter; - - fn inner(&self) -> &Self::Inner { - &self.inner - } - - async fn stat( - &self, - path: &str, - args: opendal::raw::OpStat, - ) -> opendal::Result { - self.count.fetch_add(1, Ordering::SeqCst); - self.inner.stat(path, args).await - } - - async fn read( - &self, - path: &str, - args: opendal::raw::OpRead, - ) -> opendal::Result<(opendal::raw::RpRead, Self::Reader)> { - self.inner.read(path, args).await - } - - async fn write( - &self, - path: &str, - args: opendal::raw::OpWrite, - ) -> opendal::Result<(opendal::raw::RpWrite, Self::Writer)> { - self.inner.write(path, args).await - } - - async fn delete(&self) -> opendal::Result<(opendal::raw::RpDelete, Self::Deleter)> { - self.inner.delete().await - } - - async fn list( - &self, - path: &str, - args: opendal::raw::OpList, - ) -> opendal::Result<(opendal::raw::RpList, Self::Lister)> { - self.inner.list(path, args).await - } - - async fn copy( - &self, - from: &str, - to: &str, - args: opendal::raw::OpCopy, - ) -> opendal::Result { - self.inner.copy(from, to, args).await - } - - async fn rename( - &self, - from: &str, - to: &str, - args: opendal::raw::OpRename, - ) -> opendal::Result { - self.inner.rename(from, to, args).await - } - } - } - - #[tokio::test] - async fn test_get_ranges_no_stat() { - use std::sync::atomic::{AtomicUsize, Ordering}; - - // Create a stat counter and operator with tracking layer - let stat_count = Arc::new(AtomicUsize::new(0)); - let op = Operator::new(opendal::services::Memory::default()) - .unwrap() - .layer(stat_counter::StatCounterLayer::new(stat_count.clone())) - .finish(); - let store = OpendalStore::new(op); - - // Create a test file - let location = "test_get_range.txt".into(); - let value = Bytes::from_static(b"Hello, world!"); - store.put(&location, value.clone().into()).await.unwrap(); - - // Reset counter after put - stat_count.store(0, Ordering::SeqCst); - - // Test 1: get_ranges should NOT call stat() - let range = 0..5; - let ret = store - .get_ranges(&location, std::slice::from_ref(&range)) - .await - .unwrap(); - assert_eq!(vec![Bytes::from_static(b"Hello")], ret); - assert_eq!( - stat_count.load(Ordering::SeqCst), - 0, - "get_ranges should not call stat()" - ); - - // Reset counter - stat_count.store(0, Ordering::SeqCst); - - // Test 2: get_opts SHOULD call stat() to get metadata - let opts = datafusion_object_store::GetOptions { - range: Some(datafusion_object_store::GetRange::Bounded(0..5)), - ..Default::default() - }; - let ret = store.get_opts(&location, opts).await.unwrap(); - let data = ret.bytes().await.unwrap(); - assert_eq!(Bytes::from_static(b"Hello"), data); - assert!( - stat_count.load(Ordering::SeqCst) > 0, - "get_opts should call stat() to get metadata" - ); - - // Cleanup - store.delete(&location).await.unwrap(); - } -} diff --git a/src/object-store/src/layers/mock.rs b/src/object-store/src/layers/mock.rs index 3df8aae535..f4d3df54d7 100644 --- a/src/object-store/src/layers/mock.rs +++ b/src/object-store/src/layers/mock.rs @@ -21,7 +21,7 @@ pub use opendal::raw::{ Access, Layer, LayeredAccess, OpDelete, OpList, OpRead, OpWrite, RpDelete, RpList, RpRead, RpWrite, oio, }; -use opendal::raw::{OpCopy, RpCopy}; +use opendal::raw::{OpCopier, OpCopy, RpCopy}; pub use opendal::{Buffer, Error, ErrorKind, Metadata, Result}; pub type MockWriterFactory = Arc oio::Writer + Send + Sync>; @@ -146,6 +146,7 @@ impl LayeredAccess for MockAccessor { type Writer = MockWriter; type Lister = MockLister; type Deleter = MockDeleter; + type Copier = oio::Copier; fn inner(&self) -> &Self::Inner { &self.inner @@ -222,15 +223,24 @@ impl LayeredAccess for MockAccessor { } } - async fn copy(&self, from: &str, to: &str, args: OpCopy) -> Result { - let Some(copy_interceptor) = self.copy_interceptor.as_ref() else { - return self.inner.copy(from, to, args).await; - }; + async fn copy( + &self, + from: &str, + to: &str, + args: OpCopy, + opts: OpCopier, + ) -> Result<(RpCopy, Self::Copier)> { + if let Some(result) = self + .copy_interceptor + .as_ref() + .and_then(|copy_interceptor| copy_interceptor(from, to, args.clone())) + { + return result.map(|rp_copy| (rp_copy, Box::new(()) as oio::Copier)); + } - let Some(result) = copy_interceptor(from, to, args.clone()) else { - return self.inner.copy(from, to, args).await; - }; - - result + self.inner + .copy(from, to, args, opts) + .await + .map(|(rp_copy, copier)| (rp_copy, Box::new(copier) as oio::Copier)) } } diff --git a/src/object-store/src/lib.rs b/src/object-store/src/lib.rs index 3a5f72c5ce..f1f8b59082 100644 --- a/src/object-store/src/lib.rs +++ b/src/object-store/src/lib.rs @@ -18,7 +18,6 @@ pub use opendal::{ FuturesAsyncWriter, Lister, Operator as ObjectStore, Reader, Result, Writer, services, }; -pub mod compat; pub mod config; pub mod error; pub mod factory; From 6e6ea4c0c7f9eefe808b8c1f246bdabb239895b8 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Mon, 1 Jun 2026 17:45:31 +0800 Subject: [PATCH 08/29] test: fuzz unpartitioned repartition (#8195) * test: fuzz unpartitioned repartition Signed-off-by: WenyXu * fix: invalidate table info cache Signed-off-by: WenyXu * chore: set max actions 128 Signed-off-by: WenyXu * chore: use InitStrategy::VersionChecked for table info cache Signed-off-by: WenyXu * chore: refine logs Signed-off-by: WenyXu --------- Signed-off-by: WenyXu --- src/catalog/src/kvbackend/table_cache.rs | 13 ++- src/common/meta/src/cache.rs | 2 +- src/meta-srv/src/procedure/repartition.rs | 12 ++- .../repartition/update_partition_metadata.rs | 13 ++- tests-fuzz/src/context.rs | 9 ++ tests-fuzz/src/generator/create_expr.rs | 86 +++++++++++----- tests-fuzz/src/ir.rs | 4 +- tests-fuzz/src/ir/repartition_expr.rs | 10 ++ .../src/translator/mysql/repartition_expr.rs | 98 ++++++++++++++++++- tests-fuzz/src/validator/partition.rs | 20 +++- .../ddl/fuzz_repartition_metric_table.rs | 47 ++++++--- .../targets/ddl/fuzz_repartition_table.rs | 70 +++++++++++-- .../ddl/fuzz_repartition_table_chaos.rs | 79 ++++++++++++--- 13 files changed, 395 insertions(+), 68 deletions(-) diff --git a/src/catalog/src/kvbackend/table_cache.rs b/src/catalog/src/kvbackend/table_cache.rs index 42b3fbc74b..13f74a48c9 100644 --- a/src/catalog/src/kvbackend/table_cache.rs +++ b/src/catalog/src/kvbackend/table_cache.rs @@ -14,7 +14,9 @@ use std::sync::Arc; -use common_meta::cache::{CacheContainer, Initializer, TableInfoCacheRef, TableNameCacheRef}; +use common_meta::cache::{ + CacheContainer, InitStrategy, Initializer, TableInfoCacheRef, TableNameCacheRef, +}; use common_meta::error::{Result as MetaResult, ValueNotExistSnafu}; use common_meta::instruction::CacheIdent; use futures::future::BoxFuture; @@ -38,7 +40,14 @@ pub fn new_table_cache( ) -> TableCache { let init = init_factory(table_info_cache, table_name_cache); - CacheContainer::new(name, cache, Box::new(invalidator), init, filter) + CacheContainer::with_strategy( + name, + cache, + Box::new(invalidator), + init, + filter, + InitStrategy::VersionChecked, + ) } fn init_factory( diff --git a/src/common/meta/src/cache.rs b/src/common/meta/src/cache.rs index f16290937a..c26a0fab76 100644 --- a/src/common/meta/src/cache.rs +++ b/src/common/meta/src/cache.rs @@ -17,7 +17,7 @@ mod flow; mod registry; mod table; -pub use container::{CacheContainer, Initializer, Invalidator, TokenFilter}; +pub use container::{CacheContainer, InitStrategy, Initializer, Invalidator, TokenFilter}; pub use flow::{TableFlownodeSetCache, TableFlownodeSetCacheRef, new_table_flownode_set_cache}; pub use registry::{ CacheRegistry, CacheRegistryBuilder, CacheRegistryRef, LayeredCacheRegistry, diff --git a/src/meta-srv/src/procedure/repartition.rs b/src/meta-srv/src/procedure/repartition.rs index c1819cb364..8c7c1dfeff 100644 --- a/src/meta-srv/src/procedure/repartition.rs +++ b/src/meta-srv/src/procedure/repartition.rs @@ -440,7 +440,17 @@ impl Context { }; let _ = self .cache_invalidator - .invalidate(&ctx, &[CacheIdent::TableId(table_id)]) + .invalidate( + &ctx, + &[ + CacheIdent::TableId(table_id), + CacheIdent::TableName(TableName { + catalog_name: self.persistent_ctx.catalog_name.clone(), + schema_name: self.persistent_ctx.schema_name.clone(), + table_name: self.persistent_ctx.table_name.clone(), + }), + ], + ) .await; Ok(()) } diff --git a/src/meta-srv/src/procedure/repartition/update_partition_metadata.rs b/src/meta-srv/src/procedure/repartition/update_partition_metadata.rs index cc9ca1c9bb..8d68a6e8fe 100644 --- a/src/meta-srv/src/procedure/repartition/update_partition_metadata.rs +++ b/src/meta-srv/src/procedure/repartition/update_partition_metadata.rs @@ -95,10 +95,19 @@ impl State for UpdatePartitionMetadata { let mut new_table_info = table_info_value.table_info.clone(); new_table_info.meta.partition_key_indices = partition_key_indices; + common_telemetry::info!( + "Update table partition metadata, table_id: {}, partition_key_indices: {:?}, partition_columns: {:?}", + table_id, + new_table_info.meta.partition_key_indices, + new_table_info + .meta + .partition_column_names() + .cloned() + .collect::>(), + ); ctx.update_table_info(&table_info_value, table_info_value.update(new_table_info)) .await?; - // We don't invalidate cache here because the subsequent AllocateRegion step - // will update the table route and invalidate the cache accordingly. + ctx.invalidate_table_cache().await?; Ok(( Box::new(AllocateRegion::new(self.plan_entries.clone())), diff --git a/tests-fuzz/src/context.rs b/tests-fuzz/src/context.rs index 2f65ad56aa..8043e166fd 100644 --- a/tests-fuzz/src/context.rs +++ b/tests-fuzz/src/context.rs @@ -200,6 +200,15 @@ impl TableContext { partitions.remove_bound(removed_idx)?; partition_def.exprs = partitions.generate()?; } + RepartitionExpr::AlterPartitions(partition) => { + ensure!( + self.partition.is_none(), + error::UnexpectedSnafu { + violated: format!("Table {} already has partition", self.name), + } + ); + self.partition = Some(partition.partition); + } } Ok(self) diff --git a/tests-fuzz/src/generator/create_expr.rs b/tests-fuzz/src/generator/create_expr.rs index 261a310db2..401c0fa9ff 100644 --- a/tests-fuzz/src/generator/create_expr.rs +++ b/tests-fuzz/src/generator/create_expr.rs @@ -44,6 +44,7 @@ pub struct CreateTableExprGenerator { #[builder(setter(into))] engine: String, partition: usize, + partition_column: bool, if_not_exists: bool, #[builder(setter(into))] name: Ident, @@ -67,6 +68,7 @@ impl Default for CreateTableExprGenerator { engine: DEFAULT_ENGINE.to_string(), if_not_exists: false, partition: 0, + partition_column: false, name: Ident::new(""), with_clause: HashMap::default(), name_generator: Box::new(MappedGenerator::new(WordGenerator, random_capitalize_map)), @@ -95,7 +97,7 @@ impl Generator for CreateTableExprGenerato let mut builder = CreateTableExprBuilder::default(); let mut columns = Vec::with_capacity(self.columns); let mut primary_keys = vec![]; - let need_partible_column = self.partition > 1; + let need_partible_column = self.partition > 1 || self.partition_column; let mut column_names = self.name_generator.choose(rng, self.columns); if self.columns == 1 { @@ -123,13 +125,15 @@ impl Generator for CreateTableExprGenerato ) .remove(0); - // Generates partition bounds. - let partition_def = generate_partition_def( - self.partition, - column.column_type.clone(), - name.clone(), - ); - builder.partition(partition_def); + if self.partition > 1 { + // Generates partition bounds. + let partition_def = generate_partition_def( + self.partition, + column.column_type.clone(), + name.clone(), + ); + builder.partition(partition_def); + } columns.push(column); } // Generates the ts column. @@ -178,11 +182,12 @@ impl Generator for CreateTableExprGenerato } } -fn generate_partition_def( +pub fn generate_partition_def( partitions: usize, column_type: ConcreteDataType, column_name: Ident, ) -> PartitionDef { + assert!(partitions > 1, "partitions must be greater than 1"); let bounds = generate_partition_bounds(&column_type, partitions - 1); let partitions = SimplePartitions::new(column_name.clone(), bounds); let partition_exprs = partitions.generate().unwrap(); @@ -193,24 +198,23 @@ fn generate_partition_def( } } -fn generate_metric_partition(partitions: usize) -> Option<(Column, PartitionDef)> { - if partitions <= 1 { - return None; - } - - let partition_column = Column { +fn metric_partition_column() -> Column { + Column { name: Ident::new("host"), column_type: ConcreteDataType::string_datatype(), options: vec![ColumnOption::PrimaryKey], - }; + } +} + +pub fn generate_metric_partition_def(partitions: usize) -> PartitionDef { + assert!(partitions > 1, "partitions must be greater than 1"); + let partition_column = metric_partition_column(); let bounds = generate_partition_bounds(&partition_column.column_type, partitions - 1); let partitions = SimplePartitions::new(partition_column.name.clone(), bounds); - let partition_def = PartitionDef { + PartitionDef { columns: vec![partitions.column_name.clone()], exprs: partitions.generate().unwrap(), - }; - - Some((partition_column, partition_def)) + } } /// Generate a physical table with 2 columns: ts of TimestampType::Millisecond as time index and val of Float64Type. @@ -223,6 +227,8 @@ pub struct CreatePhysicalTableExprGenerator { if_not_exists: bool, #[builder(default = "0")] partition: usize, + #[builder(default = "false")] + partition_column: bool, #[builder(default, setter(into))] with_clause: HashMap, } @@ -252,11 +258,13 @@ impl Generator for CreatePhysicalTableExpr let mut partition = None; let mut primary_keys = vec![]; - if let Some((partition_column, partition_def)) = generate_metric_partition(self.partition) { - columns.push(partition_column); - partition = Some(partition_def); + if self.partition > 1 || self.partition_column { + columns.push(metric_partition_column()); primary_keys.push(columns.len() - 1); } + if self.partition > 1 { + partition = Some(generate_metric_partition_def(self.partition)); + } Ok(CreateTableExpr { table_name: self.name_generator.generate(rng), @@ -387,6 +395,7 @@ mod tests { use super::*; use crate::context::TableContext; + use crate::ir::PARTIBLE_DATA_TYPES; #[test] fn test_float64() { @@ -423,6 +432,18 @@ mod tests { .unwrap(); assert_eq!(expr.columns.len(), 10); assert!(expr.partition.is_none()); + + let expr = CreateTableExprGeneratorBuilder::default() + .columns(10) + .partition(1) + .partition_column(true) + .build() + .unwrap() + .generate(&mut rng) + .unwrap(); + assert_eq!(expr.columns.len(), 10); + assert!(expr.partition.is_none()); + assert!(PARTIBLE_DATA_TYPES.contains(&expr.columns[0].column_type)); } #[test] @@ -516,6 +537,25 @@ mod tests { assert_eq!(physical_table_expr.partition.unwrap().exprs.len(), 3); } + #[test] + fn test_create_physical_table_expr_generator_with_partition_column() { + let mut rng = rand::rng(); + let physical_table_expr = CreatePhysicalTableExprGeneratorBuilder::default() + .partition(1) + .partition_column(true) + .if_not_exists(false) + .build() + .unwrap() + .generate(&mut rng) + .unwrap(); + + assert_eq!(physical_table_expr.engine, "metric"); + assert!(physical_table_expr.partition.is_none()); + assert_eq!(physical_table_expr.columns.len(), 3); + assert_eq!(physical_table_expr.columns[2].name, Ident::new("host")); + assert_eq!(physical_table_expr.primary_keys, vec![2]); + } + #[test] fn test_create_logical_table_expr_generator_without_partition_column() { let mut rng = rand::rng(); diff --git a/tests-fuzz/src/ir.rs b/tests-fuzz/src/ir.rs index ce1628cd61..e0b57ad0d1 100644 --- a/tests-fuzz/src/ir.rs +++ b/tests-fuzz/src/ir.rs @@ -30,7 +30,7 @@ use std::time::Duration; pub use alter_expr::{AlterTableExpr, AlterTableOption}; use common_time::timestamp::TimeUnit; use common_time::{Date, Timestamp}; -pub use create_expr::{CreateDatabaseExpr, CreateTableExpr}; +pub use create_expr::{CreateDatabaseExpr, CreateTableExpr, PartitionDef}; use datatypes::data_type::ConcreteDataType; use datatypes::types::TimestampType; use datatypes::value::Value; @@ -40,7 +40,7 @@ use lazy_static::lazy_static; pub use partition_expr::SimplePartitions; use rand::Rng; use rand::seq::{IndexedRandom, SliceRandom}; -pub use repartition_expr::RepartitionExpr; +pub use repartition_expr::{AlterTablePartitionsExpr, RepartitionExpr}; use serde::{Deserialize, Serialize}; use self::insert_expr::RowValues; diff --git a/tests-fuzz/src/ir/repartition_expr.rs b/tests-fuzz/src/ir/repartition_expr.rs index 5c8b401c8d..d17ed90d2c 100644 --- a/tests-fuzz/src/ir/repartition_expr.rs +++ b/tests-fuzz/src/ir/repartition_expr.rs @@ -16,6 +16,7 @@ use partition::expr::PartitionExpr; use serde::{Deserialize, Serialize}; use crate::ir::Ident; +use crate::ir::create_expr::PartitionDef; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SplitPartitionExpr { @@ -34,10 +35,19 @@ pub struct MergePartitionExpr { pub wait: bool, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AlterTablePartitionsExpr { + pub table_name: Ident, + pub partition: PartitionDef, + #[serde(default = "default_wait")] + pub wait: bool, +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub enum RepartitionExpr { Split(SplitPartitionExpr), Merge(MergePartitionExpr), + AlterPartitions(AlterTablePartitionsExpr), } const fn default_wait() -> bool { diff --git a/tests-fuzz/src/translator/mysql/repartition_expr.rs b/tests-fuzz/src/translator/mysql/repartition_expr.rs index 56c27594b3..43ca8e57ef 100644 --- a/tests-fuzz/src/translator/mysql/repartition_expr.rs +++ b/tests-fuzz/src/translator/mysql/repartition_expr.rs @@ -15,7 +15,10 @@ use partition::expr::PartitionExpr; use crate::error::Result; -use crate::ir::repartition_expr::{MergePartitionExpr, RepartitionExpr, SplitPartitionExpr}; +use crate::ir::create_expr::PartitionDef; +use crate::ir::repartition_expr::{ + AlterTablePartitionsExpr, MergePartitionExpr, RepartitionExpr, SplitPartitionExpr, +}; use crate::translator::DslTranslator; pub struct RepartitionExprTranslator; @@ -59,10 +62,38 @@ impl DslTranslator for RepartitionExprTranslator { table_name, merge_exprs, wait_clause )) } + RepartitionExpr::AlterPartitions(AlterTablePartitionsExpr { + table_name, + partition, + wait, + }) => { + let partition_clause = format_partition_clause(partition); + let wait_clause = format_wait_clause(*wait); + Ok(format!( + "ALTER TABLE {} {}{};", + table_name, partition_clause, wait_clause + )) + } } } } +fn format_partition_clause(partition: &PartitionDef) -> String { + let columns = partition + .columns + .iter() + .map(|column| column.to_string()) + .collect::>() + .join(", "); + let exprs = partition + .exprs + .iter() + .map(format_partition_expr_sql) + .collect::>() + .join(",\n "); + format!("PARTITION ON COLUMNS ({columns}) (\n {exprs}\n)") +} + fn format_partition_expr_sql(expr: &PartitionExpr) -> String { expr.to_parser_expr().to_string() } @@ -79,9 +110,15 @@ fn format_wait_clause(wait: bool) -> String { mod tests { use datatypes::value::Value; use partition::expr::col; + use sql::dialect::GreptimeDbDialect; + use sql::parser::{ParseOptions, ParserContext}; use super::RepartitionExprTranslator; - use crate::ir::repartition_expr::{MergePartitionExpr, RepartitionExpr, SplitPartitionExpr}; + use crate::ir::Ident; + use crate::ir::create_expr::PartitionDef; + use crate::ir::repartition_expr::{ + AlterTablePartitionsExpr, MergePartitionExpr, RepartitionExpr, SplitPartitionExpr, + }; use crate::translator::DslTranslator; #[test] @@ -149,4 +186,61 @@ mod tests { );"#; assert_eq!(sql, expected); } + + #[test] + fn test_translate_alter_table_partitions_expr() { + let expr = RepartitionExpr::AlterPartitions(AlterTablePartitionsExpr { + table_name: "demo".into(), + partition: PartitionDef { + columns: vec![Ident::new("id")], + exprs: vec![ + col("id").lt(Value::Int32(10)), + col("id") + .gt_eq(Value::Int32(10)) + .and(col("id").lt(Value::Int32(20))), + col("id").gt_eq(Value::Int32(20)), + ], + }, + wait: true, + }); + let sql = RepartitionExprTranslator.translate(&expr).unwrap(); + let expected = r#"ALTER TABLE demo PARTITION ON COLUMNS (id) ( + id < 10, + id >= 10 AND id < 20, + id >= 20 +);"#; + assert_eq!(sql, expected); + assert_repartition_sql_parseable(&sql); + } + + #[test] + fn test_translate_alter_table_partitions_expr_wait_false() { + let expr = RepartitionExpr::AlterPartitions(AlterTablePartitionsExpr { + table_name: "demo".into(), + partition: PartitionDef { + columns: vec![Ident::new("host")], + exprs: vec![ + col("host").lt(Value::from("m")), + col("host").gt_eq(Value::from("m")), + ], + }, + wait: false, + }); + let sql = RepartitionExprTranslator.translate(&expr).unwrap(); + let expected = r#"ALTER TABLE demo PARTITION ON COLUMNS (host) ( + host < 'm', + host >= 'm' +) WITH ( + WAIT = false +);"#; + assert_eq!(sql, expected); + assert_repartition_sql_parseable(&sql); + } + + fn assert_repartition_sql_parseable(sql: &str) { + let statements = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) + .unwrap(); + assert_eq!(statements.len(), 1); + } } diff --git a/tests-fuzz/src/validator/partition.rs b/tests-fuzz/src/validator/partition.rs index 3fbc8e7f86..581fb8a635 100644 --- a/tests-fuzz/src/validator/partition.rs +++ b/tests-fuzz/src/validator/partition.rs @@ -21,7 +21,8 @@ use crate::ir::Ident; use crate::ir::create_expr::PartitionDef; const PARTITIONS_INFO_SCHEMA_SQL: &str = "SELECT table_catalog, table_schema, table_name, \ -partition_name, partition_expression, partition_description, greptime_partition_id, \ +partition_name, COALESCE(partition_expression, '') AS partition_expression, \ +COALESCE(partition_description, '') AS partition_description, greptime_partition_id, \ partition_ordinal_position FROM information_schema.partitions WHERE table_name = ? \ ORDER BY partition_ordinal_position;"; @@ -91,3 +92,20 @@ pub fn assert_partitions(expected: &PartitionDef, actual: &[PartitionInfo]) -> R Ok(()) } + +/// Asserts that the table has no partition metadata in information schema. +pub fn assert_unpartitioned(actual: &[PartitionInfo]) -> Result<()> { + let has_no_partition_metadata = actual.is_empty() + || (actual.len() == 1 + && actual[0].partition_expression.is_empty() + && actual[0].partition_description.is_empty()); + + ensure!( + has_no_partition_metadata, + error::AssertSnafu { + reason: format!("Expected unpartitioned table, got partitions: {actual:?}"), + } + ); + + Ok(()) +} diff --git a/tests-fuzz/targets/ddl/fuzz_repartition_metric_table.rs b/tests-fuzz/targets/ddl/fuzz_repartition_metric_table.rs index 7932bc7759..8a6bd81fa8 100644 --- a/tests-fuzz/targets/ddl/fuzz_repartition_metric_table.rs +++ b/tests-fuzz/targets/ddl/fuzz_repartition_metric_table.rs @@ -36,14 +36,15 @@ use tests_fuzz::fake::{ use tests_fuzz::generator::Generator; use tests_fuzz::generator::create_expr::{ CreateLogicalTableExprGeneratorBuilder, CreatePhysicalTableExprGeneratorBuilder, + generate_metric_partition_def, }; use tests_fuzz::generator::insert_expr::InsertExprGeneratorBuilder; use tests_fuzz::generator::repartition_expr::{ MergePartitionExprGeneratorBuilder, SplitPartitionExprGeneratorBuilder, }; use tests_fuzz::ir::{ - CreateTableExpr, Ident, InsertIntoExpr, RepartitionExpr, generate_random_value, - generate_unique_timestamp_for_mysql_with_clock, + AlterTablePartitionsExpr, CreateTableExpr, Ident, InsertIntoExpr, PartitionDef, + RepartitionExpr, generate_random_value, generate_unique_timestamp_for_mysql_with_clock, }; use tests_fuzz::translator::DslTranslator; use tests_fuzz::translator::csv::InsertExprToCsvRecordsTranslator; @@ -94,6 +95,7 @@ fn generate_create_physical_table_expr( )))) .if_not_exists(rng.random_bool(0.5)) .partition(partitions) + .partition_column(partitions <= 1) .build() .unwrap() .generate(rng) @@ -158,12 +160,6 @@ async fn create_metric_tables( })?; info!("Create physical table: {create_physical_sql}, result: {result:?}"); let physical_table_ctx = Arc::new(TableContext::from(&create_physical_expr)); - ensure!( - physical_table_ctx.partition.is_some(), - error::AssertSnafu { - reason: "Physical metric table must have partition".to_string() - } - ); let mut logical_tables = BTreeMap::new(); let mut create_logical_sqls = HashMap::new(); @@ -436,6 +432,11 @@ fn repartition_operation( table_ctx: &TableContextRef, rng: &mut R, ) -> Result { + if table_ctx.partition.is_none() { + let partition = generate_metric_partition_def(rng.random_range(2..8)); + return Ok(alter_table_partitions_expr(table_ctx, partition, true)); + } + let split = rng.random_bool(0.5); if table_ctx.partition.as_ref().unwrap().exprs.len() <= 2 || split { let expr = SplitPartitionExprGeneratorBuilder::default() @@ -454,19 +455,35 @@ fn repartition_operation( } } +fn alter_table_partitions_expr( + table_ctx: &TableContextRef, + partition: PartitionDef, + wait: bool, +) -> RepartitionExpr { + RepartitionExpr::AlterPartitions(AlterTablePartitionsExpr { + table_name: table_ctx.name.clone(), + partition, + wait, + }) +} + impl Arbitrary<'_> for FuzzInput { fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result { let seed = get_fuzz_override::("SEED").unwrap_or(u.int_in_range(u64::MIN..=u64::MAX)?); let mut rng = ChaChaRng::seed_from_u64(seed); - let partitions = - get_fuzz_override::("PARTITIONS").unwrap_or_else(|| rng.random_range(2..8)); + let partitions = get_fuzz_override::("PARTITIONS").unwrap_or_else(|| { + if rng.random_bool(0.5) { + 1 + } else { + rng.random_range(2..8) + } + }); let max_tables = get_gt_fuzz_input_max_tables(); let tables = get_fuzz_override::("TABLES") .unwrap_or_else(|| rng.random_range(1..=std::cmp::max(1, max_tables))); - let max_actions = get_gt_fuzz_input_max_alter_actions(); + let max_actions = std::cmp::min(128, get_gt_fuzz_input_max_alter_actions()); let actions = get_fuzz_override::("ACTIONS") .unwrap_or_else(|| rng.random_range(1..max_actions)); - Ok(FuzzInput { seed, actions, @@ -536,7 +553,11 @@ async fn execute_repartition_metric_table(ctx: FuzzContext, input: FuzzInput) -> tokio::time::sleep(Duration::from_millis(100)).await; for i in 0..input.actions { - let partition_num = physical_table_ctx.partition.as_ref().unwrap().exprs.len(); + let partition_num = physical_table_ctx + .partition + .as_ref() + .map(|partition| partition.exprs.len()) + .unwrap_or_default(); info!( "partition_num: {partition_num}, action: {}/{}, table: {}, logical table num: {}", i + 1, diff --git a/tests-fuzz/targets/ddl/fuzz_repartition_table.rs b/tests-fuzz/targets/ddl/fuzz_repartition_table.rs index d4b9e9fd7a..4f6a014f2e 100644 --- a/tests-fuzz/targets/ddl/fuzz_repartition_table.rs +++ b/tests-fuzz/targets/ddl/fuzz_repartition_table.rs @@ -33,14 +33,15 @@ use tests_fuzz::fake::{ uppercase_and_keyword_backtick_map, }; use tests_fuzz::generator::Generator; -use tests_fuzz::generator::create_expr::CreateTableExprGeneratorBuilder; +use tests_fuzz::generator::create_expr::{CreateTableExprGeneratorBuilder, generate_partition_def}; use tests_fuzz::generator::insert_expr::InsertExprGeneratorBuilder; use tests_fuzz::generator::repartition_expr::{ MergePartitionExprGeneratorBuilder, SplitPartitionExprGeneratorBuilder, }; use tests_fuzz::ir::{ - CreateTableExpr, InsertIntoExpr, MySQLTsColumnTypeGenerator, RepartitionExpr, RowValue, - SimplePartitions, generate_partition_value, generate_unique_timestamp_for_mysql_with_clock, + AlterTablePartitionsExpr, CreateTableExpr, InsertIntoExpr, MySQLTsColumnTypeGenerator, + PartitionDef, RepartitionExpr, RowValue, SimplePartitions, generate_partition_value, + generate_unique_timestamp_for_mysql_with_clock, }; use tests_fuzz::translator::DslTranslator; use tests_fuzz::translator::mysql::create_expr::CreateTableExprTranslator; @@ -75,8 +76,13 @@ impl Arbitrary<'_> for FuzzInput { fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result { let seed = get_fuzz_override::("SEED").unwrap_or(u.int_in_range(u64::MIN..=u64::MAX)?); let mut rng = ChaChaRng::seed_from_u64(seed); - let partitions = - get_fuzz_override::("PARTITIONS").unwrap_or_else(|| rng.random_range(2..8)); + let partitions = get_fuzz_override::("PARTITIONS").unwrap_or_else(|| { + if rng.random_bool(0.5) { + 1 + } else { + rng.random_range(2..8) + } + }); let max_actions = get_gt_fuzz_input_max_alter_actions(); let actions = get_fuzz_override::("ACTIONS") .unwrap_or_else(|| rng.random_range(1..max_actions)); @@ -99,6 +105,7 @@ fn generate_create_expr( ))) .columns(5) .partition(input.partitions) + .partition_column(input.partitions <= 1) .engine("mito") .ts_column_type_generator(Box::new(MySQLTsColumnTypeGenerator)) .build() @@ -122,7 +129,7 @@ fn build_insert_expr( let ts_value_generator = generate_unique_timestamp_for_mysql_with_clock(clock.clone()); let counter = Arc::new(AtomicUsize::new(0)); let counter_clone = counter.clone(); - let partition_len = table_ctx.partition.as_ref().unwrap().exprs.len(); + let partition_len = partitions.bounds.len() + 1; let row = rng.random_range(partition_len..partition_len * 2); let moved_partitions = partitions.clone(); @@ -150,6 +157,28 @@ fn build_insert_expr( insert_generator.generate(rng).unwrap() } +fn alter_table_partitions_expr( + table_ctx: &TableContextRef, + partition: PartitionDef, + wait: bool, +) -> RepartitionExpr { + RepartitionExpr::AlterPartitions(AlterTablePartitionsExpr { + table_name: table_ctx.name.clone(), + partition, + wait, + }) +} + +fn alter_table_partitions_expr_from_table_ctx( + table_ctx: &TableContextRef, + rng: &mut R, + wait: bool, +) -> RepartitionExpr { + let column = table_ctx.columns[0].clone(); + let partition = generate_partition_def(rng.random_range(2..8), column.column_type, column.name); + alter_table_partitions_expr(table_ctx, partition, wait) +} + async fn execute_insert_with_retry(ctx: &FuzzContext, sql: &str) -> Result<()> { let mut delay = Duration::from_millis(100); let mut attempt = 0; @@ -236,9 +265,36 @@ async fn execute_repartition_table(ctx: FuzzContext, input: FuzzInput) -> Result inserted_rows: 0, })); + let mut action_start = 0; + if table_ctx.partition.is_none() { + let expr = alter_table_partitions_expr_from_table_ctx(&table_ctx, &mut rng, true); + let translator = RepartitionExprTranslator; + let sql = translator.translate(&expr)?; + info!("Initial partition sql: {sql}"); + let result = sqlx::query(&sql) + .execute(&ctx.greptime) + .await + .context(error::ExecuteQuerySnafu { sql: &sql })?; + info!("Initial partition result: {result:?}"); + table_ctx = Arc::new(Arc::unwrap_or_clone(table_ctx).repartition(expr).unwrap()); + shared_state.lock().unwrap().table_ctx = table_ctx.clone(); + + let partition_entries = validator::partition::fetch_partitions_info_schema( + &ctx.greptime, + "public".into(), + &table_ctx.name, + ) + .await?; + validator::partition::assert_partitions( + table_ctx.partition.as_ref().unwrap(), + &partition_entries, + )?; + action_start = 1; + } + let writer_rng = ChaChaRng::seed_from_u64(input.seed); let writer_task = tokio::spawn(write_loop(writer_rng, ctx.clone(), shared_state.clone())); - for i in 0..input.actions { + for i in action_start..input.actions { let partition_num = table_ctx.partition.as_ref().unwrap().exprs.len(); info!( "partition_num: {partition_num}, action: {}/{}", diff --git a/tests-fuzz/targets/ddl/fuzz_repartition_table_chaos.rs b/tests-fuzz/targets/ddl/fuzz_repartition_table_chaos.rs index d3789b696c..9d8faeebf5 100644 --- a/tests-fuzz/targets/ddl/fuzz_repartition_table_chaos.rs +++ b/tests-fuzz/targets/ddl/fuzz_repartition_table_chaos.rs @@ -34,14 +34,15 @@ use tests_fuzz::fake::{ uppercase_and_keyword_backtick_map, }; use tests_fuzz::generator::Generator; -use tests_fuzz::generator::create_expr::CreateTableExprGeneratorBuilder; +use tests_fuzz::generator::create_expr::{CreateTableExprGeneratorBuilder, generate_partition_def}; use tests_fuzz::generator::insert_expr::InsertExprGeneratorBuilder; use tests_fuzz::generator::repartition_expr::{ MergePartitionExprGeneratorBuilder, SplitPartitionExprGeneratorBuilder, }; use tests_fuzz::ir::{ - CreateTableExpr, InsertIntoExpr, MySQLTsColumnTypeGenerator, RepartitionExpr, RowValue, - SimplePartitions, generate_partition_value, generate_unique_timestamp_for_mysql_with_clock, + AlterTablePartitionsExpr, CreateTableExpr, InsertIntoExpr, MySQLTsColumnTypeGenerator, + PartitionDef, RepartitionExpr, RowValue, SimplePartitions, generate_partition_value, + generate_unique_timestamp_for_mysql_with_clock, }; use tests_fuzz::translator::DslTranslator; use tests_fuzz::translator::mysql::create_expr::CreateTableExprTranslator; @@ -93,13 +94,17 @@ impl Arbitrary<'_> for FuzzInput { let mut rng = ChaChaRng::seed_from_u64(seed); let rows = get_fuzz_override::("ROWS") .unwrap_or_else(|| rng.random_range(2..get_gt_fuzz_input_max_rows())); - let partitions = - get_fuzz_override::("PARTITIONS").unwrap_or_else(|| rng.random_range(2..8)); + let partitions = get_fuzz_override::("PARTITIONS").unwrap_or_else(|| { + if rng.random_bool(0.5) { + 1 + } else { + rng.random_range(2..8) + } + }); let chaos_delay_ms = get_fuzz_override::("CHAOS_DELAY_MS").unwrap_or_else(|| rng.random_range(0..5000)); let chaos_hold_secs = get_fuzz_override::("CHAOS_HOLD_SECS").unwrap_or_else(|| rng.random_range(10..20)); - Ok(FuzzInput { seed, rows, @@ -127,6 +132,7 @@ fn generate_create_expr( ))) .columns(5) .partition(input.partitions) + .partition_column(input.partitions <= 1) .engine("mito") .ts_column_type_generator(Box::new(MySQLTsColumnTypeGenerator)) .build() @@ -144,7 +150,7 @@ fn build_insert_expr( let ts_value_generator = generate_unique_timestamp_for_mysql_with_clock(clock.clone()); let counter = Arc::new(AtomicUsize::new(0)); let counter_clone = counter.clone(); - let partition_len = table_ctx.partition.as_ref().unwrap().exprs.len(); + let partition_len = partitions.bounds.len() + 1; let moved_partitions = partitions.clone(); let insert_generator = InsertExprGeneratorBuilder::default() .table_ctx(table_ctx.clone()) @@ -202,10 +208,12 @@ async fn create_table(ctx: &FuzzContext, expr: &CreateTableExpr) -> Result( ctx: &FuzzContext, table_ctx: &TableContextRef, + partition_def: &PartitionDef, rng: &mut R, rows: usize, ) -> Result { - let partitions = SimplePartitions::from_table_ctx(table_ctx).unwrap(); + let partitions = + SimplePartitions::from_exprs(partition_def.columns[0].clone(), &partition_def.exprs)?; let clock = Arc::new(Mutex::new(Timestamp::current_millis())); let insert_expr = build_insert_expr(table_ctx, rng, &partitions, &clock, rows); let inserted_rows = insert_expr.values_list.len() as u64; @@ -260,6 +268,28 @@ fn repartition_operation( } } +fn alter_table_partitions_expr( + table_ctx: &TableContextRef, + partition: PartitionDef, + wait: bool, +) -> RepartitionExpr { + RepartitionExpr::AlterPartitions(AlterTablePartitionsExpr { + table_name: table_ctx.name.clone(), + partition, + wait, + }) +} + +fn alter_table_partitions_expr_from_table_ctx( + table_ctx: &TableContextRef, + rng: &mut R, + wait: bool, +) -> RepartitionExpr { + let column = table_ctx.columns[0].clone(); + let partition = generate_partition_def(rng.random_range(2..8), column.column_type, column.name); + alter_table_partitions_expr(table_ctx, partition, wait) +} + async fn submit_repartition_procedure(ctx: &FuzzContext, expr: &RepartitionExpr) -> Result { let translator = RepartitionExprTranslator; let sql = translator.translate(expr)?; @@ -334,10 +364,13 @@ async fn validate_terminal_metadata( after_table_ctx.partition.as_ref().unwrap(), &partition_entries, )?, - ProcedureTerminalState::Failed => validator::partition::assert_partitions( - before_table_ctx.partition.as_ref().unwrap(), - &partition_entries, - )?, + ProcedureTerminalState::Failed => { + if let Some(partition) = before_table_ctx.partition.as_ref() { + validator::partition::assert_partitions(partition, &partition_entries)?; + } else { + validator::partition::assert_unpartitioned(&partition_entries)?; + } + } } Ok(()) @@ -359,7 +392,21 @@ async fn execute_repartition_chaos(ctx: FuzzContext, input: FuzzInput) -> Result let create_expr = generate_create_expr(&input, &mut rng)?; let before_table_ctx = create_table(&ctx, &create_expr).await?; - let inserted_rows = insert_initial_rows(&ctx, &before_table_ctx, &mut rng, input.rows).await?; + let insert_partition = create_expr.partition.clone().unwrap_or_else(|| { + generate_partition_def( + 2, + before_table_ctx.columns[0].column_type.clone(), + before_table_ctx.columns[0].name.clone(), + ) + }); + let inserted_rows = insert_initial_rows( + &ctx, + &before_table_ctx, + &insert_partition, + &mut rng, + input.rows, + ) + .await?; validate_table_rows(&ctx, &before_table_ctx, inserted_rows).await?; let before_entries = validator::partition::fetch_partitions_info_schema( @@ -370,7 +417,11 @@ async fn execute_repartition_chaos(ctx: FuzzContext, input: FuzzInput) -> Result .await?; info!("Before repartition partition entries: {before_entries:?}"); - let repartition_expr = repartition_operation(&before_table_ctx, &mut rng, false)?; + let repartition_expr = if before_table_ctx.partition.is_some() { + repartition_operation(&before_table_ctx, &mut rng, false)? + } else { + alter_table_partitions_expr_from_table_ctx(&before_table_ctx, &mut rng, false) + }; let after_table_ctx = Arc::new( Arc::unwrap_or_clone(before_table_ctx.clone()) .repartition(repartition_expr.clone()) From bce925b34dcb5d39630647e676d051235605a8f8 Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Tue, 2 Jun 2026 06:31:50 +0800 Subject: [PATCH 09/29] RFC: Table Semantic Layer (#8199) * docs: RFC for Table Semantic Layer Signed-off-by: Dennis Zhuang * chore: address comments Signed-off-by: Dennis Zhuang * chore: address comments Signed-off-by: Dennis Zhuang * chore: remove source_protocol and future work for sdk versions Signed-off-by: Dennis Zhuang --------- Signed-off-by: Dennis Zhuang --- docs/rfcs/2026-05-28-table-semantic-layer.md | 157 +++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 docs/rfcs/2026-05-28-table-semantic-layer.md diff --git a/docs/rfcs/2026-05-28-table-semantic-layer.md b/docs/rfcs/2026-05-28-table-semantic-layer.md new file mode 100644 index 0000000000..e4d899d704 --- /dev/null +++ b/docs/rfcs/2026-05-28-table-semantic-layer.md @@ -0,0 +1,157 @@ +--- +Feature Name: Table Semantic Layer +Tracking Issue: TBD +Date: 2026-05-28 +Author: "Dennis Zhuang " +--- + +# Summary + +Attach a thin layer of semantic metadata to each table so machine consumers — LLM agents, alert generators, dashboard builders, MCP servers, ETL pipelines — can align it with the observability concepts they already know (OTel instrument kinds, Prometheus naming conventions, UCUM units, semantic conventions, severity numbers, OTel ↔ Prometheus translation rules). + +The mechanism reuses what already exists in `table_options` (the same slot that today carries `table_data_model` and `otlp_metric_compat`): a reserved `greptime.semantic.*` namespace, plus standard SQL column `COMMENT` for field-level supplements, plus an `information_schema.semantic_tables` view as the discovery entry point. No new protocol, no new DDL keyword. + +Per-table identity only. Cross-table relationships are deferred. + +# Motivation + +GreptimeDB already ingests OTLP metrics / traces / logs and Prometheus remote write. Each protocol carries rich metadata on the wire (instrument kind, temporality, unit, scope, resource, semantic-conventions version), and most of it is dropped when rows land in a table: + +- An `opentelemetry_traces` table looks like any wide table; signal type, source, and field provenance must be guessed from naming. +- The OTel-to-Prometheus translation in v0.16+ actively drops scope attributes and most resource attributes; the table never records *what was dropped*. +- Prometheus remote write v1 metadata is unreliable by protocol, but downstream tables do not flag whether `counter` typing was *declared* or *inferred* from the `_total` suffix. +- Mixed-temporality data (OTel delta + Prometheus cumulative in the same table) is unrecoverable from schema alone. + +The audience is broader than LLM agents. Alert generators need to choose between `rate()` and absolute thresholds, and need units to pick sensible bounds. Dashboard builders pick visualisations by signal type. MCP servers surface a structured tool catalog instead of free-text descriptions. ETL pipelines need lineage to know whether a `service_name` column is `resource.service.name` or a free-form label. All of them currently guess from column names; the metadata to remove the guess already exists at ingest time, we just do not preserve it. + +# Goals + +1. Tag every ingested table with a stable identity using existing SQL surfaces — no new protocol, no new DDL keyword. +2. Record the lossy transformations the ingestion path performs (dropped attributes, scope handling, type inference vs. declaration). +3. Expose one `information_schema` view as the consumer-facing discovery entry point. +4. Keep the layer optional and additive — tables without these options keep working unchanged. + +# Non-Goals + +- Cross-table relationship modelling. Deferred to a follow-up RFC. +- Bespoke storage. Reuse `table_options` and column `COMMENT`. +- Semantic enforcement at query time. The layer is descriptive, not coercive. +- New wire protocol. Upstream standardisation is mentioned only as a future direction. + +# Proposal + +## Three mechanisms + +1. **`greptime.semantic.*` table options** — table-level identity and lineage. Carried inside the existing `table_options` blob. This is the same slot that today carries `table_data_model = 'greptime_trace_v1'` and `otlp_metric_compat = 'prom'`, so the mechanism is generalising what the OTLP trace auto-create path already does. +2. **Column `COMMENT`** — column-level supplements ("this column is `resource.service.name`"; "this column carries delta values"). Standard SQL. +3. **`information_schema.semantic_tables` view** — a denormalised projection of the options, registered through the existing `with_extra_table_factories()` hook. Tables without a `greptime.semantic.*` option do not appear in the view. + +## Vocabulary + +All keys are flat strings under the `greptime.semantic.` prefix; values are strings; unknown keys are tolerated so the vocabulary can grow without coordinated rollouts. + +**Common (all signals)** + +| Key | Example | +| --- | --- | +| `greptime.semantic.signal_type` | `trace` / `log` / `metric` / `event` | +| `greptime.semantic.source` | `opentelemetry` / `prometheus` / `elasticsearch` / `loki` / `custom` | +| `greptime.semantic.source_version` | protocol or SDK version, e.g. `v2` (Prom remote write), `1.30.0` (optional) | +| `greptime.semantic.pipeline` | `greptime_trace_v1` (subsumes the existing `table_data_model` value) | + +**Trace**: `greptime.semantic.trace.conventions` (e.g. `otel-semconv-1.27`, lifted from `schema_url`, which is the version of the OpenTelemetry semantic conventions used in this table), `greptime.semantic.trace.has_events`, `greptime.semantic.trace.has_links`. + +**Metric** — v1 assumes one metric type per table, which is how both Prom RW and the post-v0.16 OTel ingestion path land data today; mixed-type tables are a follow-up. + +| Key | Example | +| --- | --- | +| `greptime.semantic.metric.type` | `counter` / `gauge` / `histogram` / `summary` / `updown_counter` / `gauge_histogram` / `info` / `stateset` | +| `greptime.semantic.metric.unit` | UCUM, e.g. `s`, `By`, `{request}` | +| `greptime.semantic.metric.temporality` | `cumulative` / `delta` (OTel only) | +| `greptime.semantic.metric.monotonic` | `true` / `false` | +| `greptime.semantic.metric.metadata_quality` | `declared` (OTLP / Prom RW v2 / exposition) or `inferred` (Prom RW v1, name-suffix guess) | +| `greptime.semantic.metric.original_name` | Pre-translation OTel name when the table name was Prometheus-ised | + +`metadata_quality = inferred` is the load-bearing field for confidence-aware tooling: an inferred counter should be re-checked before betting on `rate()`-style semantics. + +**Log**: `greptime.semantic.log.severity_scheme` (`otlp` / `syslog` / `custom`), `greptime.semantic.log.body_format` (`string` / `json` / `mixed`). + +**Resource / scope preservation**: `greptime.semantic.resource.attributes_preserved` (JSON array string of attrs promoted to columns), `greptime.semantic.resource.attributes_dropped` (boolean), `greptime.semantic.scope.preserved` (boolean). These answer the most common downstream question: "is this data missing because it was dropped, or because it lives on a different column than I think?" List-shaped values use JSON array strings rather than comma-separated text to avoid escaping and ordering ambiguity. + +## Conflict and update semantics + +Two design decisions worth pinning down up front, because they constrain everything else: + +- **Conflict.** Some table-level keys (`trace.conventions` lifted from `schema_url`, `metric.temporality`, ...) cannot represent the truth when a long-lived table sees rows from multiple sources. v1 records `mixed` or `unknown` rather than a fictitious single value. Downstream consumers must treat any single-valued semantic key as best-effort, not strong evidence. +- **Update.** Semantic options are stamped at table creation. v1 does not specify an update path; promoting `metadata_quality` from `inferred` to `declared`, refreshing `resource.attributes_preserved`, or revising `trace.conventions` on later writes is deferred. If real usage shows update is needed, it lands as a separate RFC. + +## `information_schema.semantic_tables` + +A consumer's first SQL on connect: + +```sql +SELECT table_catalog, table_schema, table_name, signal_type, source, pipeline +FROM information_schema.semantic_tables; +``` + +returns one row per semantic-tagged table. The view exposes a stable set of core columns (`table_catalog`, `table_schema`, `table_name`, `signal_type`, `source`, `source_version`, `pipeline`) plus a `semantic_options` JSON column carrying the rest of the `greptime.semantic.*` keys verbatim. Future keys appear inside `semantic_options` without forcing a view-schema change; only widely-used keys are ever promoted to first-class columns. + +# Implementation Plan + +Four phases, each independently shippable. + +1. **Identity.** Stamp `signal_type` and `source` on every auto-create path. The OTLP paths already have natural injection points; Prom remote write is the one non-trivial path because metric-engine logical tables share physical storage (see Open Question 2). +2. **Metric specifics.** Add type / unit / temporality / monotonic / metadata_quality / original_name at OTel metric and Prom RW ingestion sites; the data is already at hand inside the OTel translator. +3. **Resource / scope lineage.** Record what the OTel-to-Prometheus translation kept and dropped. +4. **`information_schema.semantic_tables` view + documentation** as a stable user-facing contract. + +# Relationship to OpenTelemetry standardisation + +OTel today standardises what producers emit and how data collectors are managed; the read side — what a backend exposes back to clients — is deliberately vendor turf. OTLP is one-way; OpAMP is agent management; OTEP-0243 (App Telemetry Schema) is producer-side; `schema_url` is producer-stated with no reverse. Adjacent precedents — Prometheus `/api/v1/metadata`, Loki labels API, Tempo tags, Jaeger services, ad-hoc MCP servers — are all vendor-specific. + +This is a real gap. The shape we propose locally (signal-agnostic, `schema_url`-aware, structured around a small vocabulary) is deliberately close to what a future upstream OTEP for a backend-catalog read API could look like, with Weaver's *Resolved Telemetry Schema* as the natural data model. We do not commit to driving such an OTEP here; we do commit to keeping the local shape close enough that a future upstream proposal does not force a breaking migration. + +# Alternatives + +- **New DDL syntax (`SEMANTIC trace WITH (...)`).** Cleaner-looking but non-standard and forces every client to learn it. The metadata is not interesting enough to justify a new keyword. +- **Dedicated `_semantic` system table.** Doubles the storage path for what is static per-table KV and adds lifecycle questions (drop, backfill). A view over `table_options` covers the same access pattern. +- **Column comments only.** Discovery (`WHERE signal_type = 'trace'`) becomes a full-text problem. Comments are good for column-level supplements, not for identity. +- **Encode everything into the table name.** What we do today. Every new field becomes a new naming convention. + +# Open Questions + +1. **Namespace prefix.** `greptime.semantic.*` vs. bare `semantic.*`. v1 picks the vendored prefix; alias or migrate if a community standard later emerges. +2. **Prom RW injection point.** Metric-engine logical tables share physical storage, so per-logical-table options need a hook that does not exist as cleanly as the OTLP trace branch. A short spike before Phase 1 lands for Prom RW. +3. **Mixed-type metric tables.** When ingestion modes that pack multiple metric types into one table appear, `metric.type` migrates from table-level to row-level. v1 leaves a `metric.type = 'mixed'` marker and punts. +4. **Stability surface.** Top-level keys (`signal_type`, `source`) are stable; sub-namespaces (`metric.*`, ...) are evolving until v1.0 of the layer is declared. + +# Future Work + +- **Cross-table relationships.** Paired trace/services tables, metric/info pairing, JOIN hints. Its own RFC. +- **Producer SDK/client identity.** An optional `greptime.semantic.source.sdk` key recording the emitting client (e.g. `opentelemetry-go`, `opentelemetry-java`, `opentelemetry-collector`). Because a single table can receive data from multiple SDKs (a shared trace table is the common case), mixed producers collapse to `mixed`, following the same conflict rule as the table-level keys above. +- **Backfill** for tables created before this feature shipped. +- **Upstream proposal.** Carry the shape into a community proposal — likely an OTEP for an OTLP-Catalog read API plus an MCP binding — informed by Greptime's local usage data. + +# References + +OpenTelemetry: +- [OTLP specification](https://opentelemetry.io/docs/specs/otlp/) +- [OTel Schemas (`schema_url`)](https://opentelemetry.io/docs/specs/otel/schemas/) +- [Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/) +- [OTEP-0243: App Telemetry Schema](https://github.com/open-telemetry/oteps/blob/main/text/0243-app-telemetry-schema-vision-roadmap.md) +- [OpAMP specification](https://github.com/open-telemetry/opamp-spec/blob/main/specification.md) +- [Weaver: Resolved Telemetry Schema](https://github.com/open-telemetry/weaver) +- [2025 Stability Proposal](https://opentelemetry.io/blog/2025/stability-proposal-announcement/) + +Prometheus / OpenMetrics: +- [Prometheus Remote Write 1.0](https://prometheus.io/docs/specs/prw/remote_write_spec/) +- [Prometheus Remote Write 2.0](https://prometheus.io/docs/specs/prw/remote_write_spec_2_0/) +- [Prometheus exposition formats](https://prometheus.io/docs/instrumenting/exposition_formats/) +- [Prometheus HTTP API: `/api/v1/metadata`](https://prometheus.io/docs/prometheus/latest/querying/api/#querying-metric-metadata) + +Units and conventions: +- [UCUM — Unified Code for Units of Measure](https://ucum.org/) + +GreptimeDB: +- [OTLP ingestion guide](https://docs.greptime.com/user-guide/ingest-data/for-observability/opentelemetry/) +- [Trace data model](https://docs.greptime.com/user-guide/traces/data-model/) From 3e89cba3368afef0850853e4c6d0685c24739d34 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jun 2026 17:27:02 -0700 Subject: [PATCH 10/29] chore(deps): bump tar from 0.4.45 to 0.4.46 (#8209) Bumps [tar](https://github.com/composefs/tar-rs) from 0.4.45 to 0.4.46. - [Release notes](https://github.com/composefs/tar-rs/releases) - [Commits](https://github.com/composefs/tar-rs/compare/0.4.45...0.4.46) --- updated-dependencies: - dependency-name: tar dependency-version: 0.4.46 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7c6cb93c2e..ac416f68d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14121,9 +14121,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.45" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" dependencies = [ "filetime", "libc", From f25ab67af12f59e9242ec157adfb14e055bd2594 Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Tue, 2 Jun 2026 09:56:29 +0800 Subject: [PATCH 11/29] feat: table semantic layer identity (Phase 1) (#8210) * feat: table semantic layer identity (Phase 1) Attach a thin layer of semantic metadata to ingested tables via the existing `table_options` slot, so machine consumers (LLM agents, alert/dashboard builders, MCP servers, ETL) can align a table with the observability concept it stands for without guessing from column names. See docs/rfcs/2026-05-28-table-semantic-layer.md. Phase 1 (identity) only: - New `table::requests::semantic` module: the `greptime.semantic.*` vocabulary (signal/source/source_version/pipeline + trace/metric/log/resource-scope keys, defined now, populated by later phases), value constants, the internal `greptime.internal.semantic.per_table_index` transport key (reserved for Phase 2, deliberately outside the public namespace), and `is_semantic_option_key`. - `validate_table_option` accepts the `greptime.semantic.*` prefix, so the keys are valid both on the auto-create path and on explicit `CREATE TABLE ... WITH (...)`. - `fill_table_options_for_create` copies every semantic ctx extension into the new table's options (prefix passthrough alongside the fixed allowlist). - Frontend stamps identity on the context at each ingest entry: OTLP metrics (metric/opentelemetry), traces (+pipeline, has_events/has_links/conventions for the v1 model), logs (log/opentelemetry), and Prometheus remote write (metric/prometheus, metadata_quality=inferred). OTLP metric metadata_quality is left for Phase 2 (declared). - Trace identity is stamped only on the main span table; the derived `_services` / `_operations` lookup tables keep the unstamped context and carry no semantic identity (cross-table relationships are out of scope). Semantic options appear in SHOW CREATE TABLE (like table_data_model / otlp_metric_compat) and in information_schema, so an LLM inspecting a table sees its semantics directly. Tests: unit (validation prefix + internal-key rejection, ctx passthrough) and integration assertions that the common keys land for OTLP metrics (metric-engine logical table), traces, logs, and Prometheus remote write; SHOW CREATE goldens updated. Signed-off-by: Dennis Zhuang * fix: prom batcher not cover and white list for semantic keys/values Signed-off-by: Dennis Zhuang * fix: typo Signed-off-by: Dennis Zhuang --------- Signed-off-by: Dennis Zhuang --- src/frontend/src/instance/otlp.rs | 45 ++++- src/operator/src/insert.rs | 36 ++++ src/servers/src/http/prom_store.rs | 11 ++ src/sql/src/parsers/utils.rs | 16 +- src/sql/src/statements/create.rs | 20 ++- src/table/src/requests.rs | 17 ++ src/table/src/requests/semantic.rs | 280 +++++++++++++++++++++++++++++ tests-integration/src/test_util.rs | 40 ++++- tests-integration/tests/http.rs | 124 +++++++++++-- 9 files changed, 568 insertions(+), 21 deletions(-) create mode 100644 src/table/src/requests/semantic.rs diff --git a/src/frontend/src/instance/otlp.rs b/src/frontend/src/instance/otlp.rs index 434a413fed..757f657c1e 100644 --- a/src/frontend/src/instance/otlp.rs +++ b/src/frontend/src/instance/otlp.rs @@ -43,7 +43,12 @@ use servers::query_handler::{ }; use session::context::QueryContextRef; use snafu::{IntoError, ResultExt}; -use table::requests::{OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM}; +use table::requests::{ + OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM, SEMANTIC_PIPELINE, SEMANTIC_SIGNAL_TYPE, + SEMANTIC_SOURCE, SEMANTIC_TRACE_CONVENTIONS, SEMANTIC_TRACE_HAS_EVENTS, + SEMANTIC_TRACE_HAS_LINKS, SEMANTIC_VALUE_UNKNOWN, SIGNAL_TYPE_LOG, SIGNAL_TYPE_METRIC, + SIGNAL_TYPE_TRACE, SOURCE_OPENTELEMETRY, TABLE_DATA_MODEL_TRACE_V1, +}; use crate::instance::Instance; use crate::instance::otlp::trace_semconv::trace_semconv_fixed_type; @@ -131,12 +136,14 @@ impl OpenTelemetryProtocolHandler for Instance { let (requests, rows) = otlp::metrics::to_grpc_insert_requests(request, &mut metric_ctx)?; OTLP_METRICS_ROWS.inc_by(rows as u64); - let ctx = if !is_legacy { + let ctx = { let mut c = (*ctx).clone(); - c.set_extension(OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM.to_string()); + c.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_METRIC); + c.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY); + if !is_legacy { + c.set_extension(OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM.to_string()); + } Arc::new(c) - } else { - ctx }; // If the user uses the legacy path, it is by default without metric engine. @@ -211,6 +218,15 @@ impl OpenTelemetryProtocolHandler for Instance { .get::>(); interceptor_ref.pre_execute(ctx.clone())?; + // `as_req_iter` clones this ctx into each `temp_ctx`, so identity set here + // reaches the context that drives table auto-create. + let ctx = { + let mut c = (*ctx).clone(); + c.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_LOG); + c.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY); + Arc::new(c) + }; + let opt_req = otlp::logs::to_grpc_insert_requests( request, pipeline, @@ -256,6 +272,23 @@ impl Instance { ctx: QueryContextRef, ) -> ServerResult { let is_trace_v1_model = matches!(pipeline, PipelineWay::OtlpTraceDirectV1); + + // Only the main span table gets the identity; the derived `_services` / + // `_operations` lookup tables keep the unstamped `ctx`. + let main_ctx = { + let mut c = (*ctx).clone(); + c.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_TRACE); + c.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY); + if is_trace_v1_model { + c.set_extension(SEMANTIC_PIPELINE, TABLE_DATA_MODEL_TRACE_V1); + c.set_extension(SEMANTIC_TRACE_HAS_EVENTS, "true"); + c.set_extension(SEMANTIC_TRACE_HAS_LINKS, "true"); + // schema_url is row-level, so conventions is unknown at table level. + c.set_extension(SEMANTIC_TRACE_CONVENTIONS, SEMANTIC_VALUE_UNKNOWN); + } + Arc::new(c) + }; + let ingest_ctx = TraceChunkIngestContext { pipeline_handler, pipeline, @@ -278,7 +311,7 @@ impl Instance { .map(|chunk| chunk.collect::>()) .collect::>(); for chunk in chunks { - self.ingest_trace_chunk(&ingest_ctx, chunk, ctx.clone(), &mut ingest_state) + self.ingest_trace_chunk(&ingest_ctx, chunk, main_ctx.clone(), &mut ingest_state) .await?; } } diff --git a/src/operator/src/insert.rs b/src/operator/src/insert.rs index 9ef4d2ff94..317c324429 100644 --- a/src/operator/src/insert.rs +++ b/src/operator/src/insert.rs @@ -63,6 +63,7 @@ use table::metadata::TableInfo; use table::requests::{ AUTO_CREATE_TABLE_KEY, InsertRequest as TableInsertRequest, TABLE_DATA_MODEL, TABLE_DATA_MODEL_TRACE_V1, TRACE_TABLE_PARTITIONS_HINT_KEY, VALID_TABLE_OPTION_KEYS, + is_semantic_option_key, }; use table::table_reference::TableReference; @@ -1089,6 +1090,13 @@ pub fn fill_table_options_for_create( } } + // Semantic keys are prefix-matched, not in the fixed allowlist above. + for (key, value) in ctx.extensions() { + if is_semantic_option_key(&key) { + table_options.insert(key, value); + } + } + match create_type { AutoCreateTableType::Logical(physical_table) => { table_options.insert( @@ -1391,6 +1399,34 @@ mod tests { assert!(!table_options.contains_key(APPEND_MODE_KEY)); } + #[test] + fn test_fill_table_options_copies_semantic_extensions() { + use table::requests::{ + SEMANTIC_PER_TABLE_INDEX_KEY, SEMANTIC_SIGNAL_TYPE, SEMANTIC_SOURCE, + SIGNAL_TYPE_METRIC, SOURCE_OPENTELEMETRY, + }; + + let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME); + ctx.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_METRIC); + ctx.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY); + // The internal transport key must NOT be copied into table options. + ctx.set_extension(SEMANTIC_PER_TABLE_INDEX_KEY, "{}"); + let ctx = Arc::new(ctx); + let mut table_options = Default::default(); + + fill_table_options_for_create(&mut table_options, &AutoCreateTableType::Physical, &ctx); + + assert_eq!( + Some(SIGNAL_TYPE_METRIC), + table_options.get(SEMANTIC_SIGNAL_TYPE).map(String::as_str) + ); + assert_eq!( + Some(SOURCE_OPENTELEMETRY), + table_options.get(SEMANTIC_SOURCE).map(String::as_str) + ); + assert!(!table_options.contains_key(SEMANTIC_PER_TABLE_INDEX_KEY)); + } + #[test] fn test_last_non_null_create_options_preserve_default_with_append_mode_false() { let mut ctx = QueryContext::with(DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME); diff --git a/src/servers/src/http/prom_store.rs b/src/servers/src/http/prom_store.rs index bfc072e84e..280c0655d7 100644 --- a/src/servers/src/http/prom_store.rs +++ b/src/servers/src/http/prom_store.rs @@ -31,6 +31,10 @@ use prost::Message; use serde::{Deserialize, Serialize}; use session::context::{Channel, QueryContext}; use snafu::prelude::*; +use table::requests::{ + METADATA_QUALITY_INFERRED, SEMANTIC_METRIC_METADATA_QUALITY, SEMANTIC_SIGNAL_TYPE, + SEMANTIC_SOURCE, SIGNAL_TYPE_METRIC, SOURCE_PROMETHEUS, +}; use crate::error::{self, InternalSnafu, PipelineSnafu, Result}; use crate::http::extractor::PipelineInfo; @@ -108,6 +112,13 @@ pub async fn remote_write( .clone() .unwrap_or_else(|| GREPTIME_PHYSICAL_TABLE.to_string()); query_ctx.set_extension(PHYSICAL_TABLE_PARAM, physical_table.clone()); + // Stamp the Prometheus metric identity here, before `as_req_iter` splits into the + // batched and direct write paths, so both inherit it (the batched path bypasses + // `PromStoreProtocolHandler::write`). Prom RW v1 metadata is weak, so the type is + // inferred from naming. + query_ctx.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_METRIC); + query_ctx.set_extension(SEMANTIC_SOURCE, SOURCE_PROMETHEUS); + query_ctx.set_extension(SEMANTIC_METRIC_METADATA_QUALITY, METADATA_QUALITY_INFERRED); let query_ctx = Arc::new(query_ctx); let _timer = crate::metrics::METRIC_HTTP_PROM_STORE_WRITE_ELAPSED .with_label_values(&[db.as_str()]) diff --git a/src/sql/src/parsers/utils.rs b/src/sql/src/parsers/utils.rs index 0306bd859d..239f3155ca 100644 --- a/src/sql/src/parsers/utils.rs +++ b/src/sql/src/parsers/utils.rs @@ -40,7 +40,7 @@ use snafu::{ResultExt, ensure}; use sqlparser::dialect::Dialect; use sqlparser::keywords::Keyword; use sqlparser::parser::Parser; -use table::requests::validate_table_option; +use table::requests::{SEMANTIC_PREFIX, validate_semantic_option, validate_table_option}; use crate::error::{ ConvertToLogicalExpressionSnafu, InvalidSqlSnafu, InvalidTableOptionSnafu, ParseSqlValueSnafu, @@ -395,8 +395,18 @@ pub fn parse_with_options(parser: &mut Parser) -> Result { .into_iter() .map(parse_option_string) .collect::>>()?; - for key in options.keys() { - ensure!(validate_table_option(key), InvalidTableOptionSnafu { key }); + for (key, value) in &options { + if key.starts_with(SEMANTIC_PREFIX) { + // Semantic keys are whitelisted and value-checked against their domain, + // so a user cannot set an unknown key or an out-of-range value. + let value = value.as_string().unwrap_or_default(); + ensure!( + validate_semantic_option(key, value), + InvalidTableOptionSnafu { key } + ); + } else { + ensure!(validate_table_option(key), InvalidTableOptionSnafu { key }); + } } Ok(OptionMap::new(options)) } diff --git a/src/sql/src/statements/create.rs b/src/sql/src/statements/create.rs index 74ab8aee18..67742b853d 100644 --- a/src/sql/src/statements/create.rs +++ b/src/sql/src/statements/create.rs @@ -868,7 +868,25 @@ ENGINE=mito "; let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); - assert_matches!(result, Err(Error::InvalidTableOption { .. })) + assert_matches!(result, Err(Error::InvalidTableOption { .. })); + + // A whitelisted semantic key with an in-domain value is accepted. + let semantic = |with: &str| { + let sql = + format!("create table demo(host string, ts timestamp time index) with({with});"); + ParserContext::create_with_dialect(&sql, &GreptimeDbDialect {}, ParseOptions::default()) + }; + assert!(semantic("'greptime.semantic.signal_type'='metric'").is_ok()); + // An out-of-domain value is rejected. + assert_matches!( + semantic("'greptime.semantic.signal_type'='spans'"), + Err(Error::InvalidTableOption { .. }) + ); + // An unknown key under the semantic prefix is rejected. + assert_matches!( + semantic("'greptime.semantic.bogus'='x'"), + Err(Error::InvalidTableOption { .. }) + ); } #[test] diff --git a/src/table/src/requests.rs b/src/table/src/requests.rs index 4506c6ae65..6f27d5a20b 100644 --- a/src/table/src/requests.rs +++ b/src/table/src/requests.rs @@ -48,6 +48,9 @@ use crate::error::{ParseTableOptionSnafu, Result}; use crate::metadata::{TableId, TableVersion}; use crate::table_reference::TableReference; +mod semantic; +pub use semantic::*; + pub const FILE_TABLE_META_KEY: &str = "__private.file_table_meta"; pub const FILE_TABLE_LOCATION_KEY: &str = "location"; pub const FILE_TABLE_PATTERN_KEY: &str = "pattern"; @@ -129,6 +132,12 @@ pub fn validate_table_option(key: &str) -> bool { return true; } + // Semantic-layer keys share a reserved prefix instead of a fixed allowlist so + // the vocabulary can grow without touching this gate. See `semantic` module. + if is_semantic_option_key(key) { + return true; + } + VALID_TABLE_OPTION_KEYS.contains(&key) || VALID_DDL_OPTION_KEYS.contains(&key) } @@ -490,6 +499,14 @@ mod tests { assert!(validate_table_option(STORAGE_KEY)); assert!(validate_table_option(MEMTABLE_BULK_MERGE_THRESHOLD)); assert!(!validate_table_option("foo")); + + // Only whitelisted semantic keys are accepted. + assert!(validate_table_option(SEMANTIC_SIGNAL_TYPE)); + assert!(validate_table_option(SEMANTIC_METRIC_TYPE)); + // Unknown semantic key, near-miss, and the internal transport key are rejected. + assert!(!validate_table_option("greptime.semantic.future.key")); + assert!(!validate_table_option("greptime.semanticx")); + assert!(!validate_table_option(SEMANTIC_PER_TABLE_INDEX_KEY)); } #[test] diff --git a/src/table/src/requests/semantic.rs b/src/table/src/requests/semantic.rs new file mode 100644 index 0000000000..66d5096293 --- /dev/null +++ b/src/table/src/requests/semantic.rs @@ -0,0 +1,280 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Table semantic layer vocabulary. +//! +//! A thin layer of semantic metadata attached to a table via `table_options`, so +//! machine consumers (LLM agents, alert/dashboard builders, MCP servers, ETL) can +//! align a table with the observability concept it stands for without guessing +//! from column names. See `docs/rfcs/2026-05-28-table-semantic-layer.md`. +//! +//! All public table-option keys share the [`SEMANTIC_PREFIX`] namespace and are +//! string-valued. [`is_semantic_option_key`] gates them through +//! [`crate::requests::validate_table_option`], so they are accepted both on the +//! ingestion auto-create path and on explicit `CREATE TABLE ... WITH (...)` DDL. + +/// Reserved prefix for every public semantic table-option key. +pub const SEMANTIC_PREFIX: &str = "greptime.semantic."; + +/// Internal `QueryContext` extension key carrying the per-table semantic index +/// (a `{table_name -> {semantic_key: value}}` JSON blob) from the ingestion +/// encode path to the auto-create site. Deliberately OUTSIDE [`SEMANTIC_PREFIX`] +/// so it is not a valid table option and never leaks into a table's options. +pub const SEMANTIC_PER_TABLE_INDEX_KEY: &str = "greptime.internal.semantic.per_table_index"; + +// ---- Common keys (all signals) ---- + +/// Signal kind: one of [`SIGNAL_TYPE_TRACE`] / [`SIGNAL_TYPE_LOG`] / +/// [`SIGNAL_TYPE_METRIC`] / [`SIGNAL_TYPE_EVENT`]. +pub const SEMANTIC_SIGNAL_TYPE: &str = "greptime.semantic.signal_type"; +/// Ingestion ecosystem, e.g. [`SOURCE_OPENTELEMETRY`] / [`SOURCE_PROMETHEUS`]. +pub const SEMANTIC_SOURCE: &str = "greptime.semantic.source"; +/// Optional protocol or SDK version string, e.g. `v2` (Prom remote write), `1.30.0`. +pub const SEMANTIC_SOURCE_VERSION: &str = "greptime.semantic.source_version"; +/// Internal ingestion pipeline / data model, e.g. `greptime_trace_v1`. +pub const SEMANTIC_PIPELINE: &str = "greptime.semantic.pipeline"; + +// ---- Trace keys ---- + +/// Semantic-conventions version the rows conform to (e.g. `otel-semconv-1.27`), +/// or [`SEMANTIC_VALUE_UNKNOWN`] / [`SEMANTIC_VALUE_MIXED`] when not single-valued. +pub const SEMANTIC_TRACE_CONVENTIONS: &str = "greptime.semantic.trace.conventions"; +/// Whether `span_events` are preserved on the table. +pub const SEMANTIC_TRACE_HAS_EVENTS: &str = "greptime.semantic.trace.has_events"; +/// Whether `span_links` are preserved on the table. +pub const SEMANTIC_TRACE_HAS_LINKS: &str = "greptime.semantic.trace.has_links"; + +// ---- Metric keys (populated in Phase 2) ---- + +/// Instrument kind: `counter` / `gauge` / `histogram` / `summary` / +/// `updown_counter` / `gauge_histogram` / `info` / `stateset`. +pub const SEMANTIC_METRIC_TYPE: &str = "greptime.semantic.metric.type"; +/// UCUM unit, e.g. `s`, `By`, `{request}`. +pub const SEMANTIC_METRIC_UNIT: &str = "greptime.semantic.metric.unit"; +/// `cumulative` / `delta` (OTel only). +pub const SEMANTIC_METRIC_TEMPORALITY: &str = "greptime.semantic.metric.temporality"; +/// `true` / `false` for sum / counter typed data. +pub const SEMANTIC_METRIC_MONOTONIC: &str = "greptime.semantic.metric.monotonic"; +/// [`METADATA_QUALITY_DECLARED`] when the protocol stated the type, or +/// [`METADATA_QUALITY_INFERRED`] when guessed from a name suffix. +pub const SEMANTIC_METRIC_METADATA_QUALITY: &str = "greptime.semantic.metric.metadata_quality"; +/// Pre-translation OTel metric name when the table name was Prometheus-ised. +pub const SEMANTIC_METRIC_ORIGINAL_NAME: &str = "greptime.semantic.metric.original_name"; + +// ---- Log keys (populated in Phase 3) ---- + +/// `otlp` / `syslog` / `custom` — which mapping to use for `severity_number`. +pub const SEMANTIC_LOG_SEVERITY_SCHEME: &str = "greptime.semantic.log.severity_scheme"; +/// `string` / `json` / `mixed` — how to parse `body`. +pub const SEMANTIC_LOG_BODY_FORMAT: &str = "greptime.semantic.log.body_format"; + +// ---- Resource / scope preservation keys (populated in Phase 3) ---- + +/// JSON array string of resource attributes promoted to first-class columns. +pub const SEMANTIC_RESOURCE_ATTRIBUTES_PRESERVED: &str = + "greptime.semantic.resource.attributes_preserved"; +/// `true` / `false` — whether any resource attribute was dropped at ingest. +pub const SEMANTIC_RESOURCE_ATTRIBUTES_DROPPED: &str = + "greptime.semantic.resource.attributes_dropped"; +/// `true` / `false` — whether `scope.name` / `scope.version` survive on the row. +pub const SEMANTIC_SCOPE_PRESERVED: &str = "greptime.semantic.scope.preserved"; + +// ---- Value constants ---- + +pub const SIGNAL_TYPE_TRACE: &str = "trace"; +pub const SIGNAL_TYPE_LOG: &str = "log"; +pub const SIGNAL_TYPE_METRIC: &str = "metric"; +pub const SIGNAL_TYPE_EVENT: &str = "event"; + +pub const SOURCE_OPENTELEMETRY: &str = "opentelemetry"; +pub const SOURCE_PROMETHEUS: &str = "prometheus"; + +pub const METADATA_QUALITY_DECLARED: &str = "declared"; +pub const METADATA_QUALITY_INFERRED: &str = "inferred"; + +/// Sentinel for a key that cannot be determined at stamp time. +pub const SEMANTIC_VALUE_UNKNOWN: &str = "unknown"; +/// Sentinel for a single-valued key that saw conflicting sources. +pub const SEMANTIC_VALUE_MIXED: &str = "mixed"; + +/// Every recognised public semantic table-option key. The set is a closed +/// whitelist: keys under [`SEMANTIC_PREFIX`] that are not listed here are rejected, +/// so an unknown key like `greptime.semantic.unknown_key` does not silently land +/// in a table's options. Adding a key to the vocabulary means adding it here. +pub const SEMANTIC_OPTION_KEYS: &[&str] = &[ + SEMANTIC_SIGNAL_TYPE, + SEMANTIC_SOURCE, + SEMANTIC_SOURCE_VERSION, + SEMANTIC_PIPELINE, + SEMANTIC_TRACE_CONVENTIONS, + SEMANTIC_TRACE_HAS_EVENTS, + SEMANTIC_TRACE_HAS_LINKS, + SEMANTIC_METRIC_TYPE, + SEMANTIC_METRIC_UNIT, + SEMANTIC_METRIC_TEMPORALITY, + SEMANTIC_METRIC_MONOTONIC, + SEMANTIC_METRIC_METADATA_QUALITY, + SEMANTIC_METRIC_ORIGINAL_NAME, + SEMANTIC_LOG_SEVERITY_SCHEME, + SEMANTIC_LOG_BODY_FORMAT, + SEMANTIC_RESOURCE_ATTRIBUTES_PRESERVED, + SEMANTIC_RESOURCE_ATTRIBUTES_DROPPED, + SEMANTIC_SCOPE_PRESERVED, +]; + +/// Returns true if `key` is a recognised semantic table-option key (whitelist). +/// +/// Note this is membership, not a prefix test: unknown keys under +/// [`SEMANTIC_PREFIX`] are rejected, and the internal +/// [`SEMANTIC_PER_TABLE_INDEX_KEY`] (outside the prefix) never matches. +pub fn is_semantic_option_key(key: &str) -> bool { + SEMANTIC_OPTION_KEYS.contains(&key) +} + +/// Validates a `greptime.semantic.*` option's `value` against its allowed domain. +/// +/// Open-value keys (unit, original_name, version, pipeline, conventions, the +/// preserved-attributes list) accept any non-empty string. Closed-domain keys +/// accept a fixed set, plus the `unknown` sentinel, plus `mixed` for the keys +/// where one long-lived table can legitimately see multiple values. Keys not in +/// [`SEMANTIC_OPTION_KEYS`] are rejected. +pub fn validate_semantic_option(key: &str, value: &str) -> bool { + match key { + SEMANTIC_SOURCE_VERSION + | SEMANTIC_PIPELINE + | SEMANTIC_METRIC_UNIT + | SEMANTIC_METRIC_ORIGINAL_NAME + | SEMANTIC_TRACE_CONVENTIONS + | SEMANTIC_RESOURCE_ATTRIBUTES_PRESERVED => !value.is_empty(), + + SEMANTIC_SIGNAL_TYPE => matches!(value, "trace" | "log" | "metric" | "event" | "unknown"), + SEMANTIC_SOURCE => matches!( + value, + "opentelemetry" + | "prometheus" + | "elasticsearch" + | "loki" + | "custom" + | "mixed" + | "unknown" + ), + SEMANTIC_METRIC_TYPE => matches!( + value, + "counter" + | "gauge" + | "histogram" + | "summary" + | "updown_counter" + | "gauge_histogram" + | "info" + | "stateset" + | "mixed" + | "unknown" + ), + SEMANTIC_METRIC_TEMPORALITY => { + matches!(value, "cumulative" | "delta" | "mixed" | "unknown") + } + SEMANTIC_METRIC_MONOTONIC + | SEMANTIC_TRACE_HAS_EVENTS + | SEMANTIC_TRACE_HAS_LINKS + | SEMANTIC_RESOURCE_ATTRIBUTES_DROPPED + | SEMANTIC_SCOPE_PRESERVED => matches!(value, "true" | "false" | "unknown"), + SEMANTIC_METRIC_METADATA_QUALITY => matches!(value, "declared" | "inferred" | "unknown"), + SEMANTIC_LOG_SEVERITY_SCHEME => matches!(value, "otlp" | "syslog" | "custom" | "unknown"), + SEMANTIC_LOG_BODY_FORMAT => matches!(value, "string" | "json" | "mixed" | "unknown"), + + _ => false, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_semantic_option_key() { + assert!(is_semantic_option_key(SEMANTIC_SIGNAL_TYPE)); + assert!(is_semantic_option_key(SEMANTIC_METRIC_TYPE)); + + // Unknown keys under the prefix are not whitelisted. + assert!(!is_semantic_option_key("greptime.semantic.future.key")); + assert!(!is_semantic_option_key("greptime.semantic.unknown_key")); + // Near-misses must not match. + assert!(!is_semantic_option_key("greptime.semanticx")); + assert!(!is_semantic_option_key("semantic.signal_type")); + assert!(!is_semantic_option_key("table_data_model")); + // The internal transport key must never be treated as a table option. + assert!(!is_semantic_option_key(SEMANTIC_PER_TABLE_INDEX_KEY)); + } + + #[test] + fn test_validate_semantic_option() { + // Enum keys reject out-of-domain values. + assert!(validate_semantic_option(SEMANTIC_SIGNAL_TYPE, "metric")); + assert!(!validate_semantic_option(SEMANTIC_SIGNAL_TYPE, "spans")); + assert!(validate_semantic_option(SEMANTIC_METRIC_TYPE, "counter")); + assert!(validate_semantic_option(SEMANTIC_METRIC_TYPE, "mixed")); + assert!(!validate_semantic_option(SEMANTIC_METRIC_TYPE, "bogus")); + + // Booleans, sentinels, open values. + assert!(validate_semantic_option(SEMANTIC_TRACE_HAS_EVENTS, "true")); + assert!(!validate_semantic_option(SEMANTIC_TRACE_HAS_EVENTS, "yes")); + assert!(validate_semantic_option( + SEMANTIC_METRIC_TEMPORALITY, + "unknown" + )); + assert!(validate_semantic_option(SEMANTIC_METRIC_UNIT, "By")); + assert!(!validate_semantic_option(SEMANTIC_METRIC_UNIT, "")); + + // Unknown key is rejected regardless of value. + assert!(!validate_semantic_option( + "greptime.semantic.future.key", + "x" + )); + + // Drift guard: every value stamped by the ingestion path must validate. + assert!(validate_semantic_option( + SEMANTIC_SIGNAL_TYPE, + SIGNAL_TYPE_TRACE + )); + assert!(validate_semantic_option( + SEMANTIC_SIGNAL_TYPE, + SIGNAL_TYPE_METRIC + )); + assert!(validate_semantic_option( + SEMANTIC_SIGNAL_TYPE, + SIGNAL_TYPE_LOG + )); + assert!(validate_semantic_option( + SEMANTIC_SOURCE, + SOURCE_OPENTELEMETRY + )); + assert!(validate_semantic_option(SEMANTIC_SOURCE, SOURCE_PROMETHEUS)); + assert!(validate_semantic_option( + SEMANTIC_METRIC_METADATA_QUALITY, + METADATA_QUALITY_INFERRED + )); + assert!(validate_semantic_option( + SEMANTIC_TRACE_CONVENTIONS, + SEMANTIC_VALUE_UNKNOWN + )); + // An empty value never validates, for any whitelisted key. + for key in SEMANTIC_OPTION_KEYS { + assert!( + !validate_semantic_option(key, ""), + "empty value should never validate for {key}" + ); + } + } +} diff --git a/tests-integration/src/test_util.rs b/tests-integration/src/test_util.rs index 60e0d9143e..ff231f6054 100644 --- a/tests-integration/src/test_util.rs +++ b/tests-integration/src/test_util.rs @@ -16,6 +16,7 @@ use std::env; use std::fmt::Display; use std::net::SocketAddr; use std::sync::Arc; +use std::time::Duration; use auth::{DefaultPermissionChecker, PermissionCheckerRef, UserProviderRef}; use axum::Router; @@ -49,6 +50,7 @@ use servers::http::{HttpOptions, HttpServerBuilder}; use servers::metrics_handler::MetricsHandler; use servers::mysql::server::{MysqlServer, MysqlSpawnConfig, MysqlSpawnRef}; use servers::otel_arrow::OtelArrowServiceHandler; +use servers::pending_rows_batcher::PendingRowsBatcher; use servers::postgres::PostgresServer; use servers::prom_remote_write::validation::PromValidationMode; use servers::query_handler::sql::SqlQueryHandler; @@ -564,6 +566,24 @@ async fn run_sql(sql: &str, instance: &GreptimeDbStandalone) { pub async fn setup_test_prom_app_with_frontend( store_type: StorageType, name: &str, +) -> (Router, TestGuard) { + setup_test_prom_app_with_frontend_inner(store_type, name, false).await +} + +/// Like [`setup_test_prom_app_with_frontend`] but enables the pending-rows batcher, +/// so Prometheus remote write goes through the batched (metric-engine) path instead +/// of the direct `PromStoreProtocolHandler::write` path. +pub async fn setup_test_prom_app_with_frontend_batched( + store_type: StorageType, + name: &str, +) -> (Router, TestGuard) { + setup_test_prom_app_with_frontend_inner(store_type, name, true).await +} + +async fn setup_test_prom_app_with_frontend_inner( + store_type: StorageType, + name: &str, + enable_batcher: bool, ) -> (Router, TestGuard) { unsafe { std::env::set_var("TZ", "UTC"); @@ -617,6 +637,24 @@ pub async fn setup_test_prom_app_with_frontend( ..Default::default() }; let frontend_ref = instance.fe_instance().clone(); + // Mirror the production wiring at `frontend::server`: build the batcher from the + // instance's managers. A short flush interval keeps the test responsive. + let pending_rows_batcher = if enable_batcher { + PendingRowsBatcher::try_new( + frontend_ref.partition_manager().clone(), + frontend_ref.node_manager().clone(), + frontend_ref.catalog_manager().clone(), + true, + frontend_ref.clone(), + Duration::from_millis(50), + 1000, + 4, + 64, + 64, + ) + } else { + None + }; let http_server = HttpServerBuilder::new(http_opts) .with_sql_handler(frontend_ref.clone()) .with_logs_handler(instance.fe_instance().clone()) @@ -625,7 +663,7 @@ pub async fn setup_test_prom_app_with_frontend( Some(frontend_ref.clone()), true, PromValidationMode::Strict, - None, + pending_rows_batcher, ) .with_prometheus_handler(frontend_ref) .with_greptime_config_options(instance.opts.datanode_options().to_toml().unwrap()) diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 5fe22359f3..9e757a743c 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -71,6 +71,7 @@ use tests_integration::test_util::{ StorageType, setup_test_http_app, setup_test_http_app_with_frontend, setup_test_http_app_with_frontend_and_slow_query_threshold, setup_test_http_app_with_frontend_and_user_provider, setup_test_prom_app_with_frontend, + setup_test_prom_app_with_frontend_batched, }; use urlencoding::encode; use yaml_rust::YamlLoader; @@ -117,6 +118,7 @@ macro_rules! http_tests { test_dashboard_path, test_dashboard_api, test_prometheus_remote_write, + test_prometheus_remote_write_batched, test_prometheus_remote_special_labels, test_prometheus_remote_schema_labels, test_prometheus_remote_write_with_pipeline, @@ -1956,6 +1958,18 @@ pub async fn test_prometheus_remote_write(store_type: StorageType) { ) .await; + // Prom RW tables carry the metric identity; type is inferred from naming. + validate_data( + "prometheus_remote_write_semantic_identity", + &client, + "select count(*) from information_schema.tables where table_name = 'metric2' \ + and create_options like '%greptime.semantic.signal_type=metric%' \ + and create_options like '%greptime.semantic.source=prometheus%' \ + and create_options like '%greptime.semantic.metric.metadata_quality=inferred%';", + "[[1]]", + ) + .await; + // Write snappy encoded data with new labels let write_request = WriteRequest { timeseries: mock_timeseries_new_label(), @@ -1977,6 +1991,48 @@ pub async fn test_prometheus_remote_write(store_type: StorageType) { guard.remove_all().await; } +/// Covers the batched (pending-rows-batcher) Prometheus remote write path, which +/// bypasses `PromStoreProtocolHandler::write`. Verifies the metric table is created +/// asynchronously and still carries the Prometheus semantic identity stamped on the +/// shared request context. +pub async fn test_prometheus_remote_write_batched(store_type: StorageType) { + common_telemetry::init_default_ut_logging(); + let (app, mut guard) = + setup_test_prom_app_with_frontend_batched(store_type, "prometheus_remote_write_batched") + .await; + let client = TestClient::new(app).await; + + let write_request = WriteRequest { + timeseries: prom_store::mock_timeseries(), + ..Default::default() + }; + let serialized_request = write_request.encode_to_vec(); + let compressed_request = + prom_store::snappy_compress(&serialized_request).expect("failed to encode snappy"); + + let res = client + .post("/v1/prometheus/write") + .header("Content-Encoding", "snappy") + .body(compressed_request) + .send() + .await; + assert_eq!(res.status(), StatusCode::NO_CONTENT); + + // The batcher flushes asynchronously, so poll until the table exists and carries + // the semantic identity (signal_type/source/metadata_quality). + wait_for_data( + &client, + "select count(*) from information_schema.tables where table_name = 'metric2' \ + and create_options like '%greptime.semantic.signal_type=metric%' \ + and create_options like '%greptime.semantic.source=prometheus%' \ + and create_options like '%greptime.semantic.metric.metadata_quality=inferred%'", + "[[1]]", + ) + .await; + + guard.remove_all().await; +} + pub async fn test_prometheus_remote_special_labels(store_type: StorageType) { common_telemetry::init_default_ut_logging(); let (app, mut guard) = @@ -2025,7 +2081,7 @@ pub async fn test_prometheus_remote_special_labels(store_type: StorageType) { expected, ) .await; - let expected = "[[\"idc3_lo_table\",\"CREATE TABLE IF NOT EXISTS \\\"idc3_lo_table\\\" (\\n \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n \\\"greptime_value\\\" DOUBLE NULL,\\n TIME INDEX (\\\"greptime_timestamp\\\")\\n)\\n\\nENGINE=metric\\nWITH(\\n \'comment\' = 'Created on insertion',\\n on_physical_table = 'f1'\\n)\"]]"; + let expected = "[[\"idc3_lo_table\",\"CREATE TABLE IF NOT EXISTS \\\"idc3_lo_table\\\" (\\n \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n \\\"greptime_value\\\" DOUBLE NULL,\\n TIME INDEX (\\\"greptime_timestamp\\\")\\n)\\n\\nENGINE=metric\\nWITH(\\n \'comment\' = 'Created on insertion',\\n 'greptime.semantic.metric.metadata_quality' = 'inferred',\\n 'greptime.semantic.signal_type' = 'metric',\\n 'greptime.semantic.source' = 'prometheus',\\n on_physical_table = 'f1'\\n)\"]]"; validate_data( "test_prometheus_remote_special_labels_idc3_show_create_table", &client, @@ -2051,7 +2107,7 @@ pub async fn test_prometheus_remote_special_labels(store_type: StorageType) { expected, ) .await; - let expected = "[[\"idc4_local_table\",\"CREATE TABLE IF NOT EXISTS \\\"idc4_local_table\\\" (\\n \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n \\\"greptime_value\\\" DOUBLE NULL,\\n TIME INDEX (\\\"greptime_timestamp\\\")\\n)\\n\\nENGINE=metric\\nWITH(\\n \'comment\' = 'Created on insertion',\\n on_physical_table = 'f2'\\n)\"]]"; + let expected = "[[\"idc4_local_table\",\"CREATE TABLE IF NOT EXISTS \\\"idc4_local_table\\\" (\\n \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n \\\"greptime_value\\\" DOUBLE NULL,\\n TIME INDEX (\\\"greptime_timestamp\\\")\\n)\\n\\nENGINE=metric\\nWITH(\\n \'comment\' = 'Created on insertion',\\n 'greptime.semantic.metric.metadata_quality' = 'inferred',\\n 'greptime.semantic.signal_type' = 'metric',\\n 'greptime.semantic.source' = 'prometheus',\\n on_physical_table = 'f2'\\n)\"]]"; validate_data( "test_prometheus_remote_special_labels_idc4_show_create_table", &client, @@ -5027,6 +5083,28 @@ pub async fn test_otlp_metrics_new(store_type: StorageType) { let expected = "[[\"claude_code_cost_usage_USD_total\"],[\"claude_code_token_usage_tokens_total\"],[\"demo\"],[\"greptime_physical_table\"],[\"numbers\"]]"; validate_data("otlp_metrics_all_tables", &client, "show tables;", expected).await; + // Metric-engine logical table carries the semantic identity. Match substrings + // because extra_options ordering is not stable. + validate_data( + "otlp_metrics_semantic_identity", + &client, + "select count(*) from information_schema.tables where table_name = 'claude_code_cost_usage_USD_total' \ + and create_options like '%greptime.semantic.signal_type=metric%' \ + and create_options like '%greptime.semantic.source=opentelemetry%';", + "[[1]]", + ) + .await; + // OTLP metric type is declared, so Phase 1 must not stamp `metadata_quality` + // here (Phase 2 adds it as `declared`). + validate_data( + "otlp_metrics_no_metadata_quality", + &client, + "select count(*) from information_schema.tables where table_name = 'claude_code_cost_usage_USD_total' \ + and create_options like '%metadata_quality%';", + "[[0]]", + ) + .await; + // CREATE TABLE IF NOT EXISTS "claude_code_cost_usage_USD_total" ( // "greptime_timestamp" TIMESTAMP(3) NOT NULL, // "greptime_value" DOUBLE NULL, @@ -5051,7 +5129,7 @@ pub async fn test_otlp_metrics_new(store_type: StorageType) { // on_physical_table = 'greptime_physical_table', // otlp_metric_compat = 'prom' // ) - let expected = "[[\"claude_code_cost_usage_USD_total\",\"CREATE TABLE IF NOT EXISTS \\\"claude_code_cost_usage_USD_total\\\" (\\n \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n \\\"greptime_value\\\" DOUBLE NULL,\\n \\\"host_arch\\\" STRING NULL,\\n \\\"job\\\" STRING NULL,\\n \\\"model\\\" STRING NULL,\\n \\\"os_version\\\" STRING NULL,\\n \\\"otel_scope_name\\\" STRING NULL,\\n \\\"otel_scope_schema_url\\\" STRING NULL,\\n \\\"otel_scope_version\\\" STRING NULL,\\n \\\"service_name\\\" STRING NULL,\\n \\\"service_version\\\" STRING NULL,\\n \\\"session_id\\\" STRING NULL,\\n \\\"terminal_type\\\" STRING NULL,\\n \\\"user_id\\\" STRING NULL,\\n TIME INDEX (\\\"greptime_timestamp\\\"),\\n PRIMARY KEY (\\\"host_arch\\\", \\\"job\\\", \\\"model\\\", \\\"os_version\\\", \\\"otel_scope_name\\\", \\\"otel_scope_schema_url\\\", \\\"otel_scope_version\\\", \\\"service_name\\\", \\\"service_version\\\", \\\"session_id\\\", \\\"terminal_type\\\", \\\"user_id\\\")\\n)\\n\\nENGINE=metric\\nWITH(\\n \'comment\' = 'Created on insertion',\\n on_physical_table = 'greptime_physical_table',\\n otlp_metric_compat = 'prom'\\n)\"]]"; + let expected = "[[\"claude_code_cost_usage_USD_total\",\"CREATE TABLE IF NOT EXISTS \\\"claude_code_cost_usage_USD_total\\\" (\\n \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n \\\"greptime_value\\\" DOUBLE NULL,\\n \\\"host_arch\\\" STRING NULL,\\n \\\"job\\\" STRING NULL,\\n \\\"model\\\" STRING NULL,\\n \\\"os_version\\\" STRING NULL,\\n \\\"otel_scope_name\\\" STRING NULL,\\n \\\"otel_scope_schema_url\\\" STRING NULL,\\n \\\"otel_scope_version\\\" STRING NULL,\\n \\\"service_name\\\" STRING NULL,\\n \\\"service_version\\\" STRING NULL,\\n \\\"session_id\\\" STRING NULL,\\n \\\"terminal_type\\\" STRING NULL,\\n \\\"user_id\\\" STRING NULL,\\n TIME INDEX (\\\"greptime_timestamp\\\"),\\n PRIMARY KEY (\\\"host_arch\\\", \\\"job\\\", \\\"model\\\", \\\"os_version\\\", \\\"otel_scope_name\\\", \\\"otel_scope_schema_url\\\", \\\"otel_scope_version\\\", \\\"service_name\\\", \\\"service_version\\\", \\\"session_id\\\", \\\"terminal_type\\\", \\\"user_id\\\")\\n)\\n\\nENGINE=metric\\nWITH(\\n \'comment\' = 'Created on insertion',\\n 'greptime.semantic.signal_type' = 'metric',\\n 'greptime.semantic.source' = 'opentelemetry',\\n on_physical_table = 'greptime_physical_table',\\n otlp_metric_compat = 'prom'\\n)\"]]"; validate_data( "otlp_metrics_all_show_create_table", &client, @@ -5124,7 +5202,7 @@ pub async fn test_otlp_metrics_new(store_type: StorageType) { // on_physical_table = 'greptime_physical_table', // otlp_metric_compat = 'prom' // ) - let expected = "[[\"claude_code_cost_usage_USD_total\",\"CREATE TABLE IF NOT EXISTS \\\"claude_code_cost_usage_USD_total\\\" (\\n \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n \\\"greptime_value\\\" DOUBLE NULL,\\n \\\"job\\\" STRING NULL,\\n \\\"model\\\" STRING NULL,\\n \\\"os_type\\\" STRING NULL,\\n \\\"os_version\\\" STRING NULL,\\n \\\"service_name\\\" STRING NULL,\\n \\\"service_version\\\" STRING NULL,\\n \\\"session_id\\\" STRING NULL,\\n \\\"terminal_type\\\" STRING NULL,\\n \\\"user_id\\\" STRING NULL,\\n TIME INDEX (\\\"greptime_timestamp\\\"),\\n PRIMARY KEY (\\\"job\\\", \\\"model\\\", \\\"os_type\\\", \\\"os_version\\\", \\\"service_name\\\", \\\"service_version\\\", \\\"session_id\\\", \\\"terminal_type\\\", \\\"user_id\\\")\\n)\\n\\nENGINE=metric\\nWITH(\\n 'comment' = 'Created on insertion',\\n on_physical_table = 'greptime_physical_table',\\n otlp_metric_compat = 'prom'\\n)\"]]"; + let expected = "[[\"claude_code_cost_usage_USD_total\",\"CREATE TABLE IF NOT EXISTS \\\"claude_code_cost_usage_USD_total\\\" (\\n \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n \\\"greptime_value\\\" DOUBLE NULL,\\n \\\"job\\\" STRING NULL,\\n \\\"model\\\" STRING NULL,\\n \\\"os_type\\\" STRING NULL,\\n \\\"os_version\\\" STRING NULL,\\n \\\"service_name\\\" STRING NULL,\\n \\\"service_version\\\" STRING NULL,\\n \\\"session_id\\\" STRING NULL,\\n \\\"terminal_type\\\" STRING NULL,\\n \\\"user_id\\\" STRING NULL,\\n TIME INDEX (\\\"greptime_timestamp\\\"),\\n PRIMARY KEY (\\\"job\\\", \\\"model\\\", \\\"os_type\\\", \\\"os_version\\\", \\\"service_name\\\", \\\"service_version\\\", \\\"session_id\\\", \\\"terminal_type\\\", \\\"user_id\\\")\\n)\\n\\nENGINE=metric\\nWITH(\\n 'comment' = 'Created on insertion',\\n 'greptime.semantic.signal_type' = 'metric',\\n 'greptime.semantic.source' = 'opentelemetry',\\n on_physical_table = 'greptime_physical_table',\\n otlp_metric_compat = 'prom'\\n)\"]]"; validate_data( "otlp_metrics_show_create_table", &client, @@ -5188,7 +5266,7 @@ pub async fn test_otlp_metrics_new(store_type: StorageType) { // on_physical_table = 'greptime_physical_table', // otlp_metric_compat = 'prom' // ) - let expected = "[[\"claude_code_cost_usage_USD_total\",\"CREATE TABLE IF NOT EXISTS \\\"claude_code_cost_usage_USD_total\\\" (\\n \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n \\\"greptime_value\\\" DOUBLE NULL,\\n \\\"job\\\" STRING NULL,\\n \\\"model\\\" STRING NULL,\\n \\\"service_name\\\" STRING NULL,\\n \\\"service_version\\\" STRING NULL,\\n \\\"session_id\\\" STRING NULL,\\n \\\"terminal_type\\\" STRING NULL,\\n \\\"user_id\\\" STRING NULL,\\n TIME INDEX (\\\"greptime_timestamp\\\"),\\n PRIMARY KEY (\\\"job\\\", \\\"model\\\", \\\"service_name\\\", \\\"service_version\\\", \\\"session_id\\\", \\\"terminal_type\\\", \\\"user_id\\\")\\n)\\n\\nENGINE=metric\\nWITH(\\n 'comment' = 'Created on insertion',\\n on_physical_table = 'greptime_physical_table',\\n otlp_metric_compat = 'prom'\\n)\"]]"; + let expected = "[[\"claude_code_cost_usage_USD_total\",\"CREATE TABLE IF NOT EXISTS \\\"claude_code_cost_usage_USD_total\\\" (\\n \\\"greptime_timestamp\\\" TIMESTAMP(3) NOT NULL,\\n \\\"greptime_value\\\" DOUBLE NULL,\\n \\\"job\\\" STRING NULL,\\n \\\"model\\\" STRING NULL,\\n \\\"service_name\\\" STRING NULL,\\n \\\"service_version\\\" STRING NULL,\\n \\\"session_id\\\" STRING NULL,\\n \\\"terminal_type\\\" STRING NULL,\\n \\\"user_id\\\" STRING NULL,\\n TIME INDEX (\\\"greptime_timestamp\\\"),\\n PRIMARY KEY (\\\"job\\\", \\\"model\\\", \\\"service_name\\\", \\\"service_version\\\", \\\"session_id\\\", \\\"terminal_type\\\", \\\"user_id\\\")\\n)\\n\\nENGINE=metric\\nWITH(\\n 'comment' = 'Created on insertion',\\n 'greptime.semantic.signal_type' = 'metric',\\n 'greptime.semantic.source' = 'opentelemetry',\\n on_physical_table = 'greptime_physical_table',\\n otlp_metric_compat = 'prom'\\n)\"]]"; validate_data( "otlp_metrics_show_create_table_none", &client, @@ -5495,7 +5573,22 @@ pub async fn test_otlp_traces_v1(store_type: StorageType) { let expected = r#"[[1736480942444376000,1736480942444499000,123000,null,"c05d7a4ec8e1f231f02ed6e8da8655b4","d24f921c75f68e23","SPAN_KIND_CLIENT","lets-go","STATUS_CODE_UNSET","","","telemetrygen","","telemetrygen","1.2.3.4","telemetrygen-server",[],[]],[1736480942444376000,1736480942444499000,123000,"d24f921c75f68e23","c05d7a4ec8e1f231f02ed6e8da8655b4","9630f2916e2f7909","SPAN_KIND_SERVER","okey-dokey-0","STATUS_CODE_UNSET","","","telemetrygen","","telemetrygen","1.2.3.4","telemetrygen-client",[],[]],[1736480942444589000,1736480942444712000,123000,null,"cc9e0991a2e63d274984bd44ee669203","eba7be77e3558179","SPAN_KIND_CLIENT","lets-go","STATUS_CODE_UNSET","","","telemetrygen","","telemetrygen","1.2.3.4","telemetrygen-server",[],[]],[1736480942444589000,1736480942444712000,123000,"eba7be77e3558179","cc9e0991a2e63d274984bd44ee669203","8f847259b0f6e1ab","SPAN_KIND_SERVER","okey-dokey-0","STATUS_CODE_UNSET","","","telemetrygen","","telemetrygen","1.2.3.4","telemetrygen-client",[],[]]]"#; validate_data("otlp_traces", &client, "select * from mytable;", expected).await; - let expected_ddl = r#"[["mytable","CREATE TABLE IF NOT EXISTS \"mytable\" (\n \"timestamp\" TIMESTAMP(9) NOT NULL,\n \"timestamp_end\" TIMESTAMP(9) NULL,\n \"duration_nano\" BIGINT UNSIGNED NULL,\n \"parent_span_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"trace_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"span_id\" STRING NULL,\n \"span_kind\" STRING NULL,\n \"span_name\" STRING NULL,\n \"span_status_code\" STRING NULL,\n \"span_status_message\" STRING NULL,\n \"trace_state\" STRING NULL,\n \"scope_name\" STRING NULL,\n \"scope_version\" STRING NULL,\n \"service_name\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"span_attributes.net.peer.ip\" STRING NULL,\n \"span_attributes.peer.service\" STRING NULL,\n \"span_events\" JSON NULL,\n \"span_links\" JSON NULL,\n TIME INDEX (\"timestamp\"),\n PRIMARY KEY (\"service_name\")\n)\nPARTITION ON COLUMNS (\"trace_id\") (\n trace_id < '1',\n trace_id >= '1' AND trace_id < '2',\n trace_id >= '2' AND trace_id < '3',\n trace_id >= '3' AND trace_id < '4',\n trace_id >= '4' AND trace_id < '5',\n trace_id >= '5' AND trace_id < '6',\n trace_id >= '6' AND trace_id < '7',\n trace_id >= '7' AND trace_id < '8',\n trace_id >= '8' AND trace_id < '9',\n trace_id >= '9' AND trace_id < 'a',\n trace_id >= 'a' AND trace_id < 'b',\n trace_id >= 'b' AND trace_id < 'c',\n trace_id >= 'c' AND trace_id < 'd',\n trace_id >= 'd' AND trace_id < 'e',\n trace_id >= 'e' AND trace_id < 'f',\n trace_id >= 'f'\n)\nENGINE=mito\nWITH(\n 'comment' = 'Created on insertion',\n append_mode = 'true',\n table_data_model = 'greptime_trace_v1'\n)"]]"#; + // The trace v1 main table carries the trace identity (events/links preserved as + // JSON columns by the v1 model). + validate_data( + "otlp_traces_semantic_identity", + &client, + "select count(*) from information_schema.tables where table_name = 'mytable' \ + and create_options like '%greptime.semantic.signal_type=trace%' \ + and create_options like '%greptime.semantic.source=opentelemetry%' \ + and create_options like '%greptime.semantic.pipeline=greptime_trace_v1%' \ + and create_options like '%greptime.semantic.trace.has_events=true%' \ + and create_options like '%greptime.semantic.trace.has_links=true%';", + "[[1]]", + ) + .await; + + let expected_ddl = r#"[["mytable","CREATE TABLE IF NOT EXISTS \"mytable\" (\n \"timestamp\" TIMESTAMP(9) NOT NULL,\n \"timestamp_end\" TIMESTAMP(9) NULL,\n \"duration_nano\" BIGINT UNSIGNED NULL,\n \"parent_span_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"trace_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"span_id\" STRING NULL,\n \"span_kind\" STRING NULL,\n \"span_name\" STRING NULL,\n \"span_status_code\" STRING NULL,\n \"span_status_message\" STRING NULL,\n \"trace_state\" STRING NULL,\n \"scope_name\" STRING NULL,\n \"scope_version\" STRING NULL,\n \"service_name\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"span_attributes.net.peer.ip\" STRING NULL,\n \"span_attributes.peer.service\" STRING NULL,\n \"span_events\" JSON NULL,\n \"span_links\" JSON NULL,\n TIME INDEX (\"timestamp\"),\n PRIMARY KEY (\"service_name\")\n)\nPARTITION ON COLUMNS (\"trace_id\") (\n trace_id < '1',\n trace_id >= '1' AND trace_id < '2',\n trace_id >= '2' AND trace_id < '3',\n trace_id >= '3' AND trace_id < '4',\n trace_id >= '4' AND trace_id < '5',\n trace_id >= '5' AND trace_id < '6',\n trace_id >= '6' AND trace_id < '7',\n trace_id >= '7' AND trace_id < '8',\n trace_id >= '8' AND trace_id < '9',\n trace_id >= '9' AND trace_id < 'a',\n trace_id >= 'a' AND trace_id < 'b',\n trace_id >= 'b' AND trace_id < 'c',\n trace_id >= 'c' AND trace_id < 'd',\n trace_id >= 'd' AND trace_id < 'e',\n trace_id >= 'e' AND trace_id < 'f',\n trace_id >= 'f'\n)\nENGINE=mito\nWITH(\n 'comment' = 'Created on insertion',\n append_mode = 'true',\n 'greptime.semantic.pipeline' = 'greptime_trace_v1',\n 'greptime.semantic.signal_type' = 'trace',\n 'greptime.semantic.source' = 'opentelemetry',\n 'greptime.semantic.trace.conventions' = 'unknown',\n 'greptime.semantic.trace.has_events' = 'true',\n 'greptime.semantic.trace.has_links' = 'true',\n table_data_model = 'greptime_trace_v1'\n)"]]"#; validate_data( "otlp_traces", &client, @@ -5548,7 +5641,7 @@ pub async fn test_otlp_traces_v1(store_type: StorageType) { ) .await; assert_eq!(StatusCode::OK, res.status()); - let expected_ddl = r#"[["trace_table_part1","CREATE TABLE IF NOT EXISTS \"trace_table_part1\" (\n \"timestamp\" TIMESTAMP(9) NOT NULL,\n \"timestamp_end\" TIMESTAMP(9) NULL,\n \"duration_nano\" BIGINT UNSIGNED NULL,\n \"parent_span_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"trace_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"span_id\" STRING NULL,\n \"span_kind\" STRING NULL,\n \"span_name\" STRING NULL,\n \"span_status_code\" STRING NULL,\n \"span_status_message\" STRING NULL,\n \"trace_state\" STRING NULL,\n \"scope_name\" STRING NULL,\n \"scope_version\" STRING NULL,\n \"service_name\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"span_attributes.net.peer.ip\" STRING NULL,\n \"span_attributes.peer.service\" STRING NULL,\n \"span_events\" JSON NULL,\n \"span_links\" JSON NULL,\n TIME INDEX (\"timestamp\"),\n PRIMARY KEY (\"service_name\")\n)\n\nENGINE=mito\nWITH(\n 'comment' = 'Created on insertion',\n append_mode = 'true',\n table_data_model = 'greptime_trace_v1'\n)"]]"#; + let expected_ddl = r#"[["trace_table_part1","CREATE TABLE IF NOT EXISTS \"trace_table_part1\" (\n \"timestamp\" TIMESTAMP(9) NOT NULL,\n \"timestamp_end\" TIMESTAMP(9) NULL,\n \"duration_nano\" BIGINT UNSIGNED NULL,\n \"parent_span_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"trace_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"span_id\" STRING NULL,\n \"span_kind\" STRING NULL,\n \"span_name\" STRING NULL,\n \"span_status_code\" STRING NULL,\n \"span_status_message\" STRING NULL,\n \"trace_state\" STRING NULL,\n \"scope_name\" STRING NULL,\n \"scope_version\" STRING NULL,\n \"service_name\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"span_attributes.net.peer.ip\" STRING NULL,\n \"span_attributes.peer.service\" STRING NULL,\n \"span_events\" JSON NULL,\n \"span_links\" JSON NULL,\n TIME INDEX (\"timestamp\"),\n PRIMARY KEY (\"service_name\")\n)\n\nENGINE=mito\nWITH(\n 'comment' = 'Created on insertion',\n append_mode = 'true',\n 'greptime.semantic.pipeline' = 'greptime_trace_v1',\n 'greptime.semantic.signal_type' = 'trace',\n 'greptime.semantic.source' = 'opentelemetry',\n 'greptime.semantic.trace.conventions' = 'unknown',\n 'greptime.semantic.trace.has_events' = 'true',\n 'greptime.semantic.trace.has_links' = 'true',\n table_data_model = 'greptime_trace_v1'\n)"]]"#; validate_data( "otlp_traces", &client, @@ -5585,7 +5678,7 @@ pub async fn test_otlp_traces_v1(store_type: StorageType) { ) .await; assert_eq!(StatusCode::OK, res.status()); - let expected_ddl = r#"[["trace_table_part4","CREATE TABLE IF NOT EXISTS \"trace_table_part4\" (\n \"timestamp\" TIMESTAMP(9) NOT NULL,\n \"timestamp_end\" TIMESTAMP(9) NULL,\n \"duration_nano\" BIGINT UNSIGNED NULL,\n \"parent_span_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"trace_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"span_id\" STRING NULL,\n \"span_kind\" STRING NULL,\n \"span_name\" STRING NULL,\n \"span_status_code\" STRING NULL,\n \"span_status_message\" STRING NULL,\n \"trace_state\" STRING NULL,\n \"scope_name\" STRING NULL,\n \"scope_version\" STRING NULL,\n \"service_name\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"span_attributes.net.peer.ip\" STRING NULL,\n \"span_attributes.peer.service\" STRING NULL,\n \"span_events\" JSON NULL,\n \"span_links\" JSON NULL,\n TIME INDEX (\"timestamp\"),\n PRIMARY KEY (\"service_name\")\n)\nPARTITION ON COLUMNS (\"trace_id\") (\n trace_id < '4',\n trace_id >= '4' AND trace_id < '8',\n trace_id >= '8' AND trace_id < 'c',\n trace_id >= 'c'\n)\nENGINE=mito\nWITH(\n 'comment' = 'Created on insertion',\n append_mode = 'true',\n table_data_model = 'greptime_trace_v1'\n)"]]"#; + let expected_ddl = r#"[["trace_table_part4","CREATE TABLE IF NOT EXISTS \"trace_table_part4\" (\n \"timestamp\" TIMESTAMP(9) NOT NULL,\n \"timestamp_end\" TIMESTAMP(9) NULL,\n \"duration_nano\" BIGINT UNSIGNED NULL,\n \"parent_span_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"trace_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"span_id\" STRING NULL,\n \"span_kind\" STRING NULL,\n \"span_name\" STRING NULL,\n \"span_status_code\" STRING NULL,\n \"span_status_message\" STRING NULL,\n \"trace_state\" STRING NULL,\n \"scope_name\" STRING NULL,\n \"scope_version\" STRING NULL,\n \"service_name\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"span_attributes.net.peer.ip\" STRING NULL,\n \"span_attributes.peer.service\" STRING NULL,\n \"span_events\" JSON NULL,\n \"span_links\" JSON NULL,\n TIME INDEX (\"timestamp\"),\n PRIMARY KEY (\"service_name\")\n)\nPARTITION ON COLUMNS (\"trace_id\") (\n trace_id < '4',\n trace_id >= '4' AND trace_id < '8',\n trace_id >= '8' AND trace_id < 'c',\n trace_id >= 'c'\n)\nENGINE=mito\nWITH(\n 'comment' = 'Created on insertion',\n append_mode = 'true',\n 'greptime.semantic.pipeline' = 'greptime_trace_v1',\n 'greptime.semantic.signal_type' = 'trace',\n 'greptime.semantic.source' = 'opentelemetry',\n 'greptime.semantic.trace.conventions' = 'unknown',\n 'greptime.semantic.trace.has_events' = 'true',\n 'greptime.semantic.trace.has_links' = 'true',\n table_data_model = 'greptime_trace_v1'\n)"]]"#; validate_data( "otlp_traces", &client, @@ -5622,7 +5715,7 @@ pub async fn test_otlp_traces_v1(store_type: StorageType) { ) .await; assert_eq!(StatusCode::OK, res.status()); - let expected_ddl = r#"[["trace_table_part32","CREATE TABLE IF NOT EXISTS \"trace_table_part32\" (\n \"timestamp\" TIMESTAMP(9) NOT NULL,\n \"timestamp_end\" TIMESTAMP(9) NULL,\n \"duration_nano\" BIGINT UNSIGNED NULL,\n \"parent_span_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"trace_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"span_id\" STRING NULL,\n \"span_kind\" STRING NULL,\n \"span_name\" STRING NULL,\n \"span_status_code\" STRING NULL,\n \"span_status_message\" STRING NULL,\n \"trace_state\" STRING NULL,\n \"scope_name\" STRING NULL,\n \"scope_version\" STRING NULL,\n \"service_name\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"span_attributes.net.peer.ip\" STRING NULL,\n \"span_attributes.peer.service\" STRING NULL,\n \"span_events\" JSON NULL,\n \"span_links\" JSON NULL,\n TIME INDEX (\"timestamp\"),\n PRIMARY KEY (\"service_name\")\n)\nPARTITION ON COLUMNS (\"trace_id\") (\n trace_id < '08',\n trace_id >= '08' AND trace_id < '10',\n trace_id >= '10' AND trace_id < '18',\n trace_id >= '18' AND trace_id < '20',\n trace_id >= '20' AND trace_id < '28',\n trace_id >= '28' AND trace_id < '30',\n trace_id >= '30' AND trace_id < '38',\n trace_id >= '38' AND trace_id < '40',\n trace_id >= '40' AND trace_id < '48',\n trace_id >= '48' AND trace_id < '50',\n trace_id >= '50' AND trace_id < '58',\n trace_id >= '58' AND trace_id < '60',\n trace_id >= '60' AND trace_id < '68',\n trace_id >= '68' AND trace_id < '70',\n trace_id >= '70' AND trace_id < '78',\n trace_id >= '78' AND trace_id < '80',\n trace_id >= '80' AND trace_id < '88',\n trace_id >= '88' AND trace_id < '90',\n trace_id >= '90' AND trace_id < '98',\n trace_id >= '98' AND trace_id < 'a0',\n trace_id >= 'a0' AND trace_id < 'a8',\n trace_id >= 'a8' AND trace_id < 'b0',\n trace_id >= 'b0' AND trace_id < 'b8',\n trace_id >= 'b8' AND trace_id < 'c0',\n trace_id >= 'c0' AND trace_id < 'c8',\n trace_id >= 'c8' AND trace_id < 'd0',\n trace_id >= 'd0' AND trace_id < 'd8',\n trace_id >= 'd8' AND trace_id < 'e0',\n trace_id >= 'e0' AND trace_id < 'e8',\n trace_id >= 'e8' AND trace_id < 'f0',\n trace_id >= 'f0' AND trace_id < 'f8',\n trace_id >= 'f8'\n)\nENGINE=mito\nWITH(\n 'comment' = 'Created on insertion',\n append_mode = 'true',\n table_data_model = 'greptime_trace_v1'\n)"]]"#; + let expected_ddl = r#"[["trace_table_part32","CREATE TABLE IF NOT EXISTS \"trace_table_part32\" (\n \"timestamp\" TIMESTAMP(9) NOT NULL,\n \"timestamp_end\" TIMESTAMP(9) NULL,\n \"duration_nano\" BIGINT UNSIGNED NULL,\n \"parent_span_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"trace_id\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"span_id\" STRING NULL,\n \"span_kind\" STRING NULL,\n \"span_name\" STRING NULL,\n \"span_status_code\" STRING NULL,\n \"span_status_message\" STRING NULL,\n \"trace_state\" STRING NULL,\n \"scope_name\" STRING NULL,\n \"scope_version\" STRING NULL,\n \"service_name\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\n \"span_attributes.net.peer.ip\" STRING NULL,\n \"span_attributes.peer.service\" STRING NULL,\n \"span_events\" JSON NULL,\n \"span_links\" JSON NULL,\n TIME INDEX (\"timestamp\"),\n PRIMARY KEY (\"service_name\")\n)\nPARTITION ON COLUMNS (\"trace_id\") (\n trace_id < '08',\n trace_id >= '08' AND trace_id < '10',\n trace_id >= '10' AND trace_id < '18',\n trace_id >= '18' AND trace_id < '20',\n trace_id >= '20' AND trace_id < '28',\n trace_id >= '28' AND trace_id < '30',\n trace_id >= '30' AND trace_id < '38',\n trace_id >= '38' AND trace_id < '40',\n trace_id >= '40' AND trace_id < '48',\n trace_id >= '48' AND trace_id < '50',\n trace_id >= '50' AND trace_id < '58',\n trace_id >= '58' AND trace_id < '60',\n trace_id >= '60' AND trace_id < '68',\n trace_id >= '68' AND trace_id < '70',\n trace_id >= '70' AND trace_id < '78',\n trace_id >= '78' AND trace_id < '80',\n trace_id >= '80' AND trace_id < '88',\n trace_id >= '88' AND trace_id < '90',\n trace_id >= '90' AND trace_id < '98',\n trace_id >= '98' AND trace_id < 'a0',\n trace_id >= 'a0' AND trace_id < 'a8',\n trace_id >= 'a8' AND trace_id < 'b0',\n trace_id >= 'b0' AND trace_id < 'b8',\n trace_id >= 'b8' AND trace_id < 'c0',\n trace_id >= 'c0' AND trace_id < 'c8',\n trace_id >= 'c8' AND trace_id < 'd0',\n trace_id >= 'd0' AND trace_id < 'd8',\n trace_id >= 'd8' AND trace_id < 'e0',\n trace_id >= 'e0' AND trace_id < 'e8',\n trace_id >= 'e8' AND trace_id < 'f0',\n trace_id >= 'f0' AND trace_id < 'f8',\n trace_id >= 'f8'\n)\nENGINE=mito\nWITH(\n 'comment' = 'Created on insertion',\n append_mode = 'true',\n 'greptime.semantic.pipeline' = 'greptime_trace_v1',\n 'greptime.semantic.signal_type' = 'trace',\n 'greptime.semantic.source' = 'opentelemetry',\n 'greptime.semantic.trace.conventions' = 'unknown',\n 'greptime.semantic.trace.has_events' = 'true',\n 'greptime.semantic.trace.has_links' = 'true',\n table_data_model = 'greptime_trace_v1'\n)"]]"#; validate_data( "otlp_traces", &client, @@ -6285,6 +6378,17 @@ pub async fn test_otlp_logs(store_type: StorageType) { expected, ) .await; + + // The auto-created log table carries the log identity. + validate_data( + "otlp_logs_semantic_identity", + &client, + "select count(*) from information_schema.tables where table_name = 'opentelemetry_logs' \ + and create_options like '%greptime.semantic.signal_type=log%' \ + and create_options like '%greptime.semantic.source=opentelemetry%';", + "[[1]]", + ) + .await; } { @@ -7720,7 +7824,7 @@ pub async fn test_jaeger_query_api_for_trace_v1(store_type: StorageType) { .await; assert_eq!(StatusCode::OK, res.status()); - let trace_table_sql = "[[\"mytable\",\"CREATE TABLE IF NOT EXISTS \\\"mytable\\\" (\\n \\\"timestamp\\\" TIMESTAMP(9) NOT NULL,\\n \\\"timestamp_end\\\" TIMESTAMP(9) NULL,\\n \\\"duration_nano\\\" BIGINT UNSIGNED NULL,\\n \\\"parent_span_id\\\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\\n \\\"trace_id\\\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\\n \\\"span_id\\\" STRING NULL,\\n \\\"span_kind\\\" STRING NULL,\\n \\\"span_name\\\" STRING NULL,\\n \\\"span_status_code\\\" STRING NULL,\\n \\\"span_status_message\\\" STRING NULL,\\n \\\"trace_state\\\" STRING NULL,\\n \\\"scope_name\\\" STRING NULL,\\n \\\"scope_version\\\" STRING NULL,\\n \\\"service_name\\\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\\n \\\"span_attributes.operation.type\\\" STRING NULL,\\n \\\"span_attributes.net.peer.ip\\\" STRING NULL,\\n \\\"span_attributes.peer.service\\\" STRING NULL,\\n \\\"span_events\\\" JSON NULL,\\n \\\"span_links\\\" JSON NULL,\\n TIME INDEX (\\\"timestamp\\\"),\\n PRIMARY KEY (\\\"service_name\\\")\\n)\\nPARTITION ON COLUMNS (\\\"trace_id\\\") (\\n trace_id < '1',\\n trace_id >= '1' AND trace_id < '2',\\n trace_id >= '2' AND trace_id < '3',\\n trace_id >= '3' AND trace_id < '4',\\n trace_id >= '4' AND trace_id < '5',\\n trace_id >= '5' AND trace_id < '6',\\n trace_id >= '6' AND trace_id < '7',\\n trace_id >= '7' AND trace_id < '8',\\n trace_id >= '8' AND trace_id < '9',\\n trace_id >= '9' AND trace_id < 'a',\\n trace_id >= 'a' AND trace_id < 'b',\\n trace_id >= 'b' AND trace_id < 'c',\\n trace_id >= 'c' AND trace_id < 'd',\\n trace_id >= 'd' AND trace_id < 'e',\\n trace_id >= 'e' AND trace_id < 'f',\\n trace_id >= 'f'\\n)\\nENGINE=mito\\nWITH(\\n 'comment' = 'Created on insertion',\\n append_mode = 'true',\\n table_data_model = 'greptime_trace_v1',\\n ttl = '7days'\\n)\"]]"; + let trace_table_sql = "[[\"mytable\",\"CREATE TABLE IF NOT EXISTS \\\"mytable\\\" (\\n \\\"timestamp\\\" TIMESTAMP(9) NOT NULL,\\n \\\"timestamp_end\\\" TIMESTAMP(9) NULL,\\n \\\"duration_nano\\\" BIGINT UNSIGNED NULL,\\n \\\"parent_span_id\\\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\\n \\\"trace_id\\\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\\n \\\"span_id\\\" STRING NULL,\\n \\\"span_kind\\\" STRING NULL,\\n \\\"span_name\\\" STRING NULL,\\n \\\"span_status_code\\\" STRING NULL,\\n \\\"span_status_message\\\" STRING NULL,\\n \\\"trace_state\\\" STRING NULL,\\n \\\"scope_name\\\" STRING NULL,\\n \\\"scope_version\\\" STRING NULL,\\n \\\"service_name\\\" STRING NULL SKIPPING INDEX WITH(false_positive_rate = '0.01', granularity = '10240', type = 'BLOOM'),\\n \\\"span_attributes.operation.type\\\" STRING NULL,\\n \\\"span_attributes.net.peer.ip\\\" STRING NULL,\\n \\\"span_attributes.peer.service\\\" STRING NULL,\\n \\\"span_events\\\" JSON NULL,\\n \\\"span_links\\\" JSON NULL,\\n TIME INDEX (\\\"timestamp\\\"),\\n PRIMARY KEY (\\\"service_name\\\")\\n)\\nPARTITION ON COLUMNS (\\\"trace_id\\\") (\\n trace_id < '1',\\n trace_id >= '1' AND trace_id < '2',\\n trace_id >= '2' AND trace_id < '3',\\n trace_id >= '3' AND trace_id < '4',\\n trace_id >= '4' AND trace_id < '5',\\n trace_id >= '5' AND trace_id < '6',\\n trace_id >= '6' AND trace_id < '7',\\n trace_id >= '7' AND trace_id < '8',\\n trace_id >= '8' AND trace_id < '9',\\n trace_id >= '9' AND trace_id < 'a',\\n trace_id >= 'a' AND trace_id < 'b',\\n trace_id >= 'b' AND trace_id < 'c',\\n trace_id >= 'c' AND trace_id < 'd',\\n trace_id >= 'd' AND trace_id < 'e',\\n trace_id >= 'e' AND trace_id < 'f',\\n trace_id >= 'f'\\n)\\nENGINE=mito\\nWITH(\\n 'comment' = 'Created on insertion',\\n append_mode = 'true',\\n 'greptime.semantic.pipeline' = 'greptime_trace_v1',\\n 'greptime.semantic.signal_type' = 'trace',\\n 'greptime.semantic.source' = 'opentelemetry',\\n 'greptime.semantic.trace.conventions' = 'unknown',\\n 'greptime.semantic.trace.has_events' = 'true',\\n 'greptime.semantic.trace.has_links' = 'true',\\n table_data_model = 'greptime_trace_v1',\\n ttl = '7days'\\n)\"]]"; validate_data( "trace_v1_create_table", &client, From 7f6400e4605ad803302f8f369ba2b63659a0a945 Mon Sep 17 00:00:00 2001 From: LFC <990479+MichaelScofield@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:14:17 +0800 Subject: [PATCH 12/29] refactor: optimize compaction's pick time (#8197) refactor: optimize compaction's pick Signed-off-by: luofucong --- Cargo.lock | 1 + src/mito2/Cargo.toml | 1 + src/mito2/src/compaction/run.rs | 139 +++++++++++++++++++++++++++++++ src/mito2/src/compaction/twcs.rs | 100 ++++++++++++---------- src/mito2/src/lib.rs | 1 + 5 files changed, 197 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ac416f68d4..a65159d26a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8322,6 +8322,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datatypes", + "derive_more", "dotenv", "either", "futures", diff --git a/src/mito2/Cargo.toml b/src/mito2/Cargo.toml index 3e3a18a24d..99e3439879 100644 --- a/src/mito2/Cargo.toml +++ b/src/mito2/Cargo.toml @@ -50,6 +50,7 @@ datafusion-common.workspace = true datafusion-expr.workspace = true datatypes.workspace = true dashmap.workspace = true +derive_more.workspace = true dotenv.workspace = true either.workspace = true futures.workspace = true diff --git a/src/mito2/src/compaction/run.rs b/src/mito2/src/compaction/run.rs index ecd19666e2..1ba4569021 100644 --- a/src/mito2/src/compaction/run.rs +++ b/src/mito2/src/compaction/run.rs @@ -15,6 +15,9 @@ //! This file contains code to find sorted runs in a set if ranged items and //! along with the best way to merge these items to satisfy the desired run count. +use std::cmp::Ordering; +use std::collections::BinaryHeap; + use bytes::{Buf, Bytes}; use common_base::BitVec; use common_base::readable_size::ReadableSize; @@ -423,6 +426,133 @@ where runs } +pub(crate) fn find_sorted_runs_by_time_range(items: &mut [T]) -> Vec> +where + T: Item, +{ + if items.is_empty() { + return vec![]; + } + sort_ranged_items(items); + + use derive_more::{Eq, PartialEq}; + + /// `SortedRun` with a creation sequence `i`. + #[derive(PartialEq, Eq)] + struct Run { + i: usize, + #[partial_eq(skip)] + run: SortedRun, + } + + impl Run { + fn new(i: usize, item: &T) -> Run { + let mut run = SortedRun::default(); + run.push_item(item.clone()); + Run { i, run } + } + + fn push_item(&mut self, item: &T) { + self.run.push_item(item.clone()); + } + } + + impl PartialOrd for Run { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + + /// Sort by run's `end` desc then `start` asc. + impl Ord for Run { + fn cmp(&self, other: &Self) -> Ordering { + let l_run = &self.run; + let r_run = &other.run; + + // Safety: `start` and `end` must both exist because it's guaranteed that whenever a + // `Run` is created, an item is pushed into it immediately (see its `new` method above). + // And there are no other ways to create a `Run` beyond its `new` method in this + // function's scope. + let l_end = l_run.end.unwrap(); + let r_end = r_run.end.unwrap(); + r_end + .cmp(&l_end) + .then_with(|| { + let l_start = l_run.start.unwrap(); + let r_start = r_run.start.unwrap(); + l_start.cmp(&r_start) + }) + .then_with(|| self.i.cmp(&other.i)) + } + } + + /// Wrapper around the `Run` above, to support sorting them by their creation sequence `i`. + #[derive(PartialEq, Eq)] + struct Wrapper(Run); + + impl PartialOrd for Wrapper { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } + } + + impl Ord for Wrapper { + fn cmp(&self, other: &Self) -> Ordering { + other.0.i.cmp(&self.0.i) + } + } + + // Two heaps for finding a run that is both: + // 1. not overlapping with item's range, + // 2. and is created earliest, + // when iterating the items. + // + // Heap 1 (`runs_sorted_by_end`) is for storing the runs of which top has the minimal "end" + // just about to overlap with the current selected item. + // + // Heap 2 (`runs_sort_by_index`) is for storing the runs that all have "end"s non-overlap with + // the current selected item, and of which top is the earliest created run. + // + // The finding of a suitable run basically works like this: + // 1. moves the runs in heap 1 to heap 2, until the top is overlapping with the current item; + // 2. now heap 2 has all the runs that can accept the current item, pop its top; + // 3. the top is the earliest created run, push the current item; + // 4. because the run has changed, push it back to heap 1; + // 5. check the next item. Important: we don't need to push the runs in heap 2 to 1, because + // the items are sorted by "start". When checking the next item, heap 2's runs must all have + // "end"s smaller than next item's "start". + // + // Actually the heap 2 is only for aligning with the runs selection outcomes in the original + // `find_sorted_runs` implementation. If we just need the invariant that each run has the + // non-overlapping items, we can get rid of heap 2 and make the codes simpler. + + let mut runs_sort_by_end = BinaryHeap::>::new(); + let mut runs_sort_by_index = BinaryHeap::>::new(); + let mut i = 0; + + for item in items { + let (start, _) = item.range(); + + while let Some(run) = runs_sort_by_end.pop_if(|x| x.run.end.unwrap() <= start) { + runs_sort_by_index.push(Wrapper(run)); + } + + let Some(mut run) = runs_sort_by_index.pop() else { + i += 1; + runs_sort_by_end.push(Run::new(i, item)); + continue; + }; + + run.0.push_item(item); + runs_sort_by_end.push(run.0); + } + + let mut runs = runs_sort_by_end.into_vec(); + runs.extend(runs_sort_by_index.into_vec().into_iter().map(|x| x.0)); + runs.sort_unstable_by_key(|run| run.i); + runs.into_iter().map(|x| x.run).collect() +} + /// Finds a set of files with minimum penalty to merge that can reduce the total num of runs. /// The penalty of merging is defined as the size of all overlapping files between two runs. pub fn reduce_runs(mut runs: Vec>) -> Vec { @@ -599,6 +729,8 @@ mod tests { expected_runs: &[Vec<(i64, i64)>], ) -> Vec> { let mut files = build_items(ranges); + let mut files_clone = files.clone(); + let runs = find_sorted_runs(&mut files); let result_file_ranges: Vec> = runs @@ -606,6 +738,13 @@ mod tests { .map(|r| r.items.iter().map(|f| f.range()).collect()) .collect(); assert_eq!(&expected_runs, &result_file_ranges); + + let runs_by_time_range = find_sorted_runs_by_time_range(&mut files_clone); + let results: Vec> = runs_by_time_range + .iter() + .map(|r| r.items.iter().map(|f| f.range()).collect()) + .collect(); + assert_eq!(&expected_runs, &results); runs } diff --git a/src/mito2/src/compaction/twcs.rs b/src/mito2/src/compaction/twcs.rs index 87dcac7279..cfa44d5045 100644 --- a/src/mito2/src/compaction/twcs.rs +++ b/src/mito2/src/compaction/twcs.rs @@ -22,14 +22,15 @@ use common_telemetry::{debug, info}; use common_time::Timestamp; use common_time::timestamp::TimeUnit; use common_time::timestamp_millis::BucketAligned; +use rayon::prelude::*; use store_api::storage::RegionId; use crate::compaction::buckets::infer_time_bucket; use crate::compaction::compactor::CompactionRegion; use crate::compaction::picker::{Picker, PickerOutput}; use crate::compaction::run::{ - FileGroup, Item, Ranged, find_sorted_runs, merge_primary_key_ranges, merge_seq_files, - primary_key_ranges_overlap, reduce_runs, + FileGroup, Item, Ranged, find_sorted_runs, find_sorted_runs_by_time_range, + merge_primary_key_ranges, merge_seq_files, primary_key_ranges_overlap, reduce_runs, }; use crate::compaction::{CompactionOutput, get_expired_ssts}; use crate::sst::file::{FileHandle, Level, overlaps}; @@ -64,11 +65,10 @@ impl TwcsPicker { time_windows: &mut BTreeMap, active_window: Option, ) -> Vec { - let mut output = vec![]; - for (window, files) in time_windows { - if files.files.is_empty() { - continue; - } + let find_inputs = |files: &Window, + windows: &BTreeMap| + -> (Vec, bool) { + let window = &files.time_window; let mut files_to_merge: Vec<_> = files.files().cloned().collect(); // Filter out large files in append mode - they won't benefit from compaction @@ -88,13 +88,18 @@ impl TwcsPicker { ); } - let sorted_runs = find_sorted_runs(&mut files_to_merge); + let sorted_runs = if files_to_merge.len() < 1024 { + find_sorted_runs(&mut files_to_merge) + } else { + find_sorted_runs_by_time_range(&mut files_to_merge) + }; let found_runs = sorted_runs.len(); // We only remove deletion markers if we found less than 2 runs and not in append mode. // because after compaction there will be no overlapping files. - let filter_deleted = !files.overlapping && found_runs <= 2 && !self.append_mode; + let filter_deleted = + found_runs <= 2 && !self.append_mode && !window_has_overlap(files, windows); if found_runs == 0 { - continue; + return (vec![], filter_deleted); } let mut inputs = if found_runs > 1 { @@ -102,7 +107,7 @@ impl TwcsPicker { } else { let run = sorted_runs.last().unwrap(); if run.items().len() < self.trigger_file_num { - continue; + return (vec![], filter_deleted); } // no overlapping files, try merge small files merge_seq_files(run.items(), self.max_output_file_size) @@ -144,6 +149,26 @@ impl TwcsPicker { filter_deleted, &inputs, ); + } + (inputs, filter_deleted) + }; + + let mut output = vec![]; + let windows = time_windows + .values() + .filter(|w| !w.files.is_empty()) + .collect::>(); + let chunk_size = self.max_background_tasks.unwrap_or(windows.len()).max(1); + 'chunks: for chunk in windows.chunks(chunk_size) { + for (inputs, filter_deleted) in chunk + .par_iter() // parallelly calculate the inputs + .map(|window| find_inputs(window, time_windows)) + .collect::>() + { + if inputs.is_empty() { + continue; + } + output.push(CompactionOutput { output_level: LEVEL_COMPACTED, // always compact to l1 inputs: inputs.into_iter().flat_map(|fg| fg.into_files()).collect(), @@ -158,7 +183,7 @@ impl TwcsPicker { "Region ({:?}) compaction task size larger than max background tasks({}), remaining tasks discarded", region_id, max_background_tasks ); - break; + break 'chunks; } } } @@ -268,7 +293,6 @@ struct Window { // created from the same compaction task. files: HashMap, FileGroup>, time_window: i64, - overlapping: bool, primary_key_range: Option<(bytes::Bytes, bytes::Bytes)>, } @@ -283,7 +307,6 @@ impl Window { end, files, time_window: 0, - overlapping: false, primary_key_range, } } @@ -346,37 +369,21 @@ fn assign_to_windows<'a>( } } } - if windows.is_empty() { - return BTreeMap::new(); - } + windows.into_iter().collect() +} - let mut windows = windows.into_values().collect::>(); - windows.sort_unstable_by(|l, r| l.start.cmp(&r.start).then(l.end.cmp(&r.end).reverse())); - - for idx in 0..windows.len() { - let lhs_range = windows[idx].range(); - for next_idx in idx + 1..windows.len() { - let rhs_range = windows[next_idx].range(); - if rhs_range.0 > lhs_range.1 { - break; - } - - let windows_overlap = overlaps(&lhs_range, &rhs_range) - && match ( - &windows[idx].primary_key_range, - &windows[next_idx].primary_key_range, - ) { - (Some(lhs), Some(rhs)) => primary_key_ranges_overlap(lhs, rhs), +fn window_has_overlap(this: &Window, windows: &BTreeMap) -> bool { + windows + .values() + .filter(|that| this.time_window != that.time_window) + .any(|that| { + overlaps(&this.range(), &that.range()) && { + match (&this.primary_key_range, &that.primary_key_range) { + (Some(l), Some(r)) => primary_key_ranges_overlap(l, r), _ => true, - }; - if windows_overlap { - windows[idx].overlapping = true; - windows[next_idx].overlapping = true; + } } - } - } - - windows.into_iter().map(|w| (w.time_window, w)).collect() + }) } /// Finds the latest active writing window among all files. @@ -606,7 +613,8 @@ mod tests { for (expected_window, overlapping, window_files) in expected_files { let actual_window = windows.get(expected_window).unwrap(); - assert_eq!(*overlapping, actual_window.overlapping); + let actual_overlapping = window_has_overlap(actual_window, &windows); + assert_eq!(*overlapping, actual_overlapping); let mut file_ranges = actual_window .files .values() @@ -744,7 +752,8 @@ mod tests { let windows = assign_to_windows(files.iter(), 2); - assert!(!windows.get(&2).unwrap().overlapping); + let overlapping = window_has_overlap(windows.get(&2).unwrap(), &windows); + assert!(!overlapping); } #[test] @@ -773,7 +782,8 @@ mod tests { let windows = assign_to_windows(files.iter(), 2); - assert!(!windows.get(&4).unwrap().overlapping); + let overlapping = window_has_overlap(windows.get(&4).unwrap(), &windows); + assert!(!overlapping); } struct CompactionPickerTestCase { diff --git a/src/mito2/src/lib.rs b/src/mito2/src/lib.rs index 7d43685ded..a452e106c7 100644 --- a/src/mito2/src/lib.rs +++ b/src/mito2/src/lib.rs @@ -18,6 +18,7 @@ #![feature(debug_closure_helpers)] #![feature(duration_constructors)] +#![feature(binary_heap_pop_if)] #[cfg(any(test, feature = "test"))] #[cfg_attr(feature = "test", allow(unused))] From c1b041837791612789cdd1c407820b720f9a58da Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Mon, 1 Jun 2026 21:11:41 -0700 Subject: [PATCH 13/29] refactor: give instance to start plugin functions (#8208) * refactor: give instance to start plugin functions * refactor: remove start_standalone_plugins because it's never used * chore: make sure code is nicely formatted --- src/cmd/src/datanode.rs | 2 +- src/cmd/src/flownode.rs | 2 +- src/cmd/src/frontend.rs | 3 +-- src/cmd/src/metasrv.rs | 2 +- src/cmd/src/standalone.rs | 2 +- src/plugins/src/datanode.rs | 3 ++- src/plugins/src/flownode.rs | 4 ++-- src/plugins/src/frontend.rs | 3 ++- src/plugins/src/lib.rs | 2 +- src/plugins/src/meta_srv.rs | 3 ++- src/plugins/src/standalone.rs | 4 ---- 11 files changed, 14 insertions(+), 16 deletions(-) diff --git a/src/cmd/src/datanode.rs b/src/cmd/src/datanode.rs index 3c106bd43f..7f2057bc97 100644 --- a/src/cmd/src/datanode.rs +++ b/src/cmd/src/datanode.rs @@ -79,7 +79,7 @@ impl App for Instance { } async fn start(&mut self) -> Result<()> { - plugins::start_datanode_plugins(self.datanode.plugins()) + plugins::start_datanode_plugins(&self.datanode) .await .context(StartDatanodeSnafu)?; diff --git a/src/cmd/src/flownode.rs b/src/cmd/src/flownode.rs index 6228cbd3f3..32d9070ec6 100644 --- a/src/cmd/src/flownode.rs +++ b/src/cmd/src/flownode.rs @@ -90,7 +90,7 @@ impl App for Instance { } async fn start(&mut self) -> Result<()> { - plugins::start_flownode_plugins(self.flownode.flow_engine().plugins().clone()) + plugins::start_flownode_plugins(&self.flownode) .await .context(StartFlownodeSnafu)?; diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs index da2c111e7c..cbc07d10e9 100644 --- a/src/cmd/src/frontend.rs +++ b/src/cmd/src/frontend.rs @@ -95,8 +95,7 @@ impl App for Instance { } async fn start(&mut self) -> Result<()> { - let plugins = self.frontend.instance.plugins().clone(); - plugins::start_frontend_plugins(plugins) + plugins::start_frontend_plugins(&self.frontend.instance) .await .context(error::StartFrontendSnafu)?; diff --git a/src/cmd/src/metasrv.rs b/src/cmd/src/metasrv.rs index bf3cb2f5e7..e30b115ada 100644 --- a/src/cmd/src/metasrv.rs +++ b/src/cmd/src/metasrv.rs @@ -68,7 +68,7 @@ impl App for Instance { } async fn start(&mut self) -> Result<()> { - plugins::start_metasrv_plugins(self.instance.plugins()) + plugins::start_metasrv_plugins(&self.instance) .await .context(StartMetaServerSnafu)?; diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index e0f2c673ff..7d99e99554 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -164,7 +164,7 @@ impl App for Instance { .start(self.leader_services_context.clone()) .await?; - plugins::start_frontend_plugins(self.frontend.instance.plugins().clone()) + plugins::start_frontend_plugins(&self.frontend.instance) .await .context(error::StartFrontendSnafu)?; diff --git a/src/plugins/src/datanode.rs b/src/plugins/src/datanode.rs index 60640f05f1..321265a35d 100644 --- a/src/plugins/src/datanode.rs +++ b/src/plugins/src/datanode.rs @@ -14,6 +14,7 @@ use common_base::Plugins; use datanode::config::DatanodeOptions; +use datanode::datanode::Datanode; use datanode::error::Result; use crate::options::PluginOptions; @@ -28,6 +29,6 @@ pub async fn setup_datanode_plugins( Ok(()) } -pub async fn start_datanode_plugins(_plugins: Plugins) -> Result<()> { +pub async fn start_datanode_plugins(_instance: &Datanode) -> Result<()> { Ok(()) } diff --git a/src/plugins/src/flownode.rs b/src/plugins/src/flownode.rs index 9fbb018030..566051d84d 100644 --- a/src/plugins/src/flownode.rs +++ b/src/plugins/src/flownode.rs @@ -13,8 +13,8 @@ // limitations under the License. use common_base::Plugins; -use flow::FlownodeOptions; use flow::error::Result; +use flow::{FlownodeInstance, FlownodeOptions}; use crate::options::PluginOptions; @@ -27,7 +27,7 @@ pub async fn setup_flownode_plugins( Ok(()) } -pub async fn start_flownode_plugins(_plugins: Plugins) -> Result<()> { +pub async fn start_flownode_plugins(_instance: &FlownodeInstance) -> Result<()> { Ok(()) } diff --git a/src/plugins/src/frontend.rs b/src/plugins/src/frontend.rs index df7ec4fcb9..9986132d63 100644 --- a/src/plugins/src/frontend.rs +++ b/src/plugins/src/frontend.rs @@ -17,6 +17,7 @@ use common_base::Plugins; use common_meta::cache::CacheRegistryBuilder; use frontend::error::{IllegalAuthConfigSnafu, Result}; use frontend::frontend::FrontendOptions; +use frontend::instance::Instance; use snafu::ResultExt; use crate::options::PluginOptions; @@ -51,7 +52,7 @@ pub async fn setup_frontend_dynamic_plugins( Ok(()) } -pub async fn start_frontend_plugins(_plugins: Plugins) -> Result<()> { +pub async fn start_frontend_plugins(_instance: &Instance) -> Result<()> { Ok(()) } diff --git a/src/plugins/src/lib.rs b/src/plugins/src/lib.rs index ba33cb9825..a1b9b5e889 100644 --- a/src/plugins/src/lib.rs +++ b/src/plugins/src/lib.rs @@ -26,4 +26,4 @@ pub use flownode::{setup_flownode_plugins, start_flownode_plugins}; pub use frontend::{setup_frontend_plugins, start_frontend_plugins}; pub use meta_srv::{setup_metasrv_plugins, start_metasrv_plugins}; pub use options::PluginOptions; -pub use standalone::{setup_standalone_plugins, start_standalone_plugins}; +pub use standalone::setup_standalone_plugins; diff --git a/src/plugins/src/meta_srv.rs b/src/plugins/src/meta_srv.rs index 282ac241c5..6d862fdfbc 100644 --- a/src/plugins/src/meta_srv.rs +++ b/src/plugins/src/meta_srv.rs @@ -13,6 +13,7 @@ // limitations under the License. use common_base::Plugins; +use meta_srv::bootstrap::MetasrvInstance; use meta_srv::error::Result; use meta_srv::metasrv::MetasrvOptions; @@ -27,6 +28,6 @@ pub async fn setup_metasrv_plugins( Ok(()) } -pub async fn start_metasrv_plugins(_plugins: Plugins) -> Result<()> { +pub async fn start_metasrv_plugins(_instance: &MetasrvInstance) -> Result<()> { Ok(()) } diff --git a/src/plugins/src/standalone.rs b/src/plugins/src/standalone.rs index 510b84106b..a946fc7e91 100644 --- a/src/plugins/src/standalone.rs +++ b/src/plugins/src/standalone.rs @@ -31,10 +31,6 @@ pub async fn setup_standalone_plugins( Ok(()) } -pub async fn start_standalone_plugins(_plugins: Plugins) -> Result<()> { - Ok(()) -} - /// Allows standalone plugins to add cache invalidators to the layered registry. pub fn configure_cache_registry(_plugins: &Plugins) -> Option { None From c9f6f67ae988a09924b261917616ebc82f8f674c Mon Sep 17 00:00:00 2001 From: LFC <990479+MichaelScofield@users.noreply.github.com> Date: Tue, 2 Jun 2026 14:16:08 +0800 Subject: [PATCH 14/29] fix: nightly jsonbench test (#8212) * fix: nightly jsonbench test Signed-off-by: luofucong * Apply suggestions from code review Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --------- Signed-off-by: luofucong Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/nightly-jsonbench.yaml | 157 ++++++++++++++++------- 1 file changed, 109 insertions(+), 48 deletions(-) diff --git a/.github/workflows/nightly-jsonbench.yaml b/.github/workflows/nightly-jsonbench.yaml index 3667ee26a6..a9ce4dd363 100644 --- a/.github/workflows/nightly-jsonbench.yaml +++ b/.github/workflows/nightly-jsonbench.yaml @@ -1,19 +1,81 @@ name: Nightly JSONBench on: - schedule: - # Trigger at 00:00(Asia/Shanghai) on every weekday. - - cron: "0 16 * * 0-4" + workflow_run: + workflows: [ "GreptimeDB Nightly Build" ] + types: [ completed ] workflow_dispatch: + inputs: + run_id: + description: The nightly build workflow run id to download GreptimeDB artifacts from + required: true + type: string + +permissions: + actions: read + contents: read concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true jobs: + resolve-artifact: + name: Resolve GreptimeDB nightly artifact + if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }} + runs-on: ubuntu-latest + outputs: + artifact-name: ${{ steps.find-artifact.outputs.artifact-name }} + run-id: ${{ steps.resolve-run-id.outputs.run-id }} + steps: + - name: Resolve nightly build run id + id: resolve-run-id + shell: bash + env: + EVENT_NAME: ${{ github.event_name }} + WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }} + INPUT_RUN_ID: ${{ inputs.run_id }} + run: | + set -euo pipefail + + if [[ "${EVENT_NAME}" == "workflow_dispatch" ]]; then + run_id="${INPUT_RUN_ID}" + else + run_id="${WORKFLOW_RUN_ID}" + fi + + if [[ ! "${run_id}" =~ ^[0-9]+$ ]]; then + echo "Invalid workflow run id: ${run_id}" + exit 1 + fi + + echo "run-id=${run_id}" >> "${GITHUB_OUTPUT}" + + - name: Find GreptimeDB nightly artifact + id: find-artifact + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_ID: ${{ steps.resolve-run-id.outputs.run-id }} + run: | + set -euo pipefail + + artifact_name=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${RUN_ID}/artifacts" --paginate \ + --jq '.artifacts[] | select(.name | test("^greptime-linux-arm64-nightly-[0-9]{8}-[0-9a-f]+$")) | .name' \ + | head -n 1) + + if [[ -z "${artifact_name}" ]]; then + echo "Cannot find linux arm64 nightly artifact in workflow run ${RUN_ID}." + exit 1 + fi + + echo "Download GreptimeDB artifact: ${artifact_name}" + echo "artifact-name=${artifact_name}" >> "${GITHUB_OUTPUT}" + allocate-runner: name: Allocate runner - if: ${{ github.repository == 'GreptimeTeam/greptimedb' }} + if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }} + needs: [ resolve-artifact ] runs-on: ubuntu-latest outputs: linux-arm64-runner: ${{ steps.start-linux-arm64-runner.outputs.label }} @@ -43,55 +105,50 @@ jobs: jsonbench: name: Run JSONBench - if: ${{ github.repository == 'GreptimeTeam/greptimedb' }} - needs: [ allocate-runner ] + if: ${{ github.repository == 'GreptimeTeam/greptimedb' && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }} + needs: [ resolve-artifact, allocate-runner ] runs-on: ${{ needs.allocate-runner.outputs.linux-arm64-runner }} timeout-minutes: 120 env: - JSONBENCH_DATA_DIR: /home/runner/data/bluesky - JSONBENCH_OUTPUT_PREFIX: _ubuntu-latest + JSONBENCH_OUTPUT_PREFIX: _linux-arm64 steps: - - name: Checkout - uses: actions/checkout@v4 + - name: Download GreptimeDB nightly artifact + uses: actions/download-artifact@v4 with: - fetch-depth: 0 - persist-credentials: false + name: ${{ needs.resolve-artifact.outputs.artifact-name }} + path: greptimedb-artifact + github-token: ${{ secrets.GITHUB_TOKEN }} + run-id: ${{ needs.resolve-artifact.outputs.run-id }} - - uses: arduino/setup-protoc@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - uses: actions-rust-lang/setup-rust-toolchain@v1 - - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - with: - shared-key: "nightly-jsonbench" - cache-all-crates: "true" - save-if: ${{ github.ref == 'refs/heads/main' }} - - - name: Build GreptimeDB - run: cargo build --profile nightly --bin greptime - - - name: Reclaim disk space + - name: Prepare GreptimeDB binary shell: bash run: | set -euo pipefail - mkdir -p "${RUNNER_TEMP}/greptimedb-bin" - cp ./target/nightly/greptime "${RUNNER_TEMP}/greptimedb-bin/greptime" - chmod +x "${RUNNER_TEMP}/greptimedb-bin/greptime" - - rm -rf ./target + tar -xzf "greptimedb-artifact/${{ needs.resolve-artifact.outputs.artifact-name }}.tar.gz" + cp "${{ needs.resolve-artifact.outputs.artifact-name }}/greptime" ./greptime + chmod +x ./greptime + rm -rf greptimedb-artifact "${{ needs.resolve-artifact.outputs.artifact-name }}" - name: Run JSONBench + env: + # TODO(LFC): Change to "3" (100m) when JSON2 ingestion performance is optimized. + JSONBENCH_DATASET: 2 shell: bash run: | set -euo pipefail - cd "${RUNNER_TEMP}" - cp "${RUNNER_TEMP}/greptimedb-bin/greptime" ./greptime - chmod +x ./greptime + export JSONBENCH_DATA_DIR="/root/data/bluesky" + echo "Use JSONBench data directory ${JSONBENCH_DATA_DIR}" + + echo "Cloning JSONBench" + git clone --branch greptimedb-new-json --depth 1 https://github.com/GreptimeTeam/JSONBench.git JSONBench + + echo "Downloading JSONBench dataset choice ${JSONBENCH_DATASET} to ${JSONBENCH_DATA_DIR}" + mkdir -p "${JSONBENCH_DATA_DIR}" + printf "${JSONBENCH_DATASET}\n" | ./JSONBench/download_data.sh + downloaded_files=$(find "${JSONBENCH_DATA_DIR}" -type f | wc -l) + echo "Downloaded JSONBench dataset files: ${downloaded_files}" export GREPTIMEDB_STANDALONE__WAL__DIR=greptimedb_data/wal export GREPTIMEDB_STANDALONE__STORAGE__DATA_HOME=greptimedb_data @@ -100,10 +157,12 @@ jobs: export GREPTIMEDB_STANDALONE__HTTP__BODY_LIMIT=1GB export GREPTIMEDB_STANDALONE__HTTP__TIMEOUT=500s + echo "Starting GreptimeDB standalone" ./greptime standalone start > greptimedb.log 2>&1 & greptime_pid=$! trap 'kill "${greptime_pid}" 2>/dev/null || true' EXIT + echo "Waiting for GreptimeDB health check" until curl -s --fail -o /dev/null http://localhost:4000/health; do if ! kill -0 "${greptime_pid}" 2>/dev/null; then cat greptimedb.log @@ -111,12 +170,14 @@ jobs: fi sleep 1 done + echo "GreptimeDB is ready" - git clone --branch greptimedb-new-json --depth 1 https://github.com/GreptimeTeam/JSONBench.git JSONBench cp ./greptime JSONBench/greptimedb/greptime cd JSONBench/greptimedb - ./main.sh 3 "${JSONBENCH_DATA_DIR}" success.log error.log "${JSONBENCH_OUTPUT_PREFIX}" false + echo "Running JSONBench main.sh with dataset choice ${JSONBENCH_DATASET} and install=false" + ./main.sh ${JSONBENCH_DATASET} "${JSONBENCH_DATA_DIR}" success.log error.log "${JSONBENCH_OUTPUT_PREFIX}" false + echo "JSONBench finished" - name: Upload JSONBench results if: always() @@ -124,21 +185,21 @@ jobs: with: name: jsonbench-results path: | - ${{ runner.temp }}/greptimedb.log - ${{ runner.temp }}/JSONBench/greptimedb/*.log - ${{ runner.temp }}/JSONBench/greptimedb/*.total_size - ${{ runner.temp }}/JSONBench/greptimedb/*.data_size - ${{ runner.temp }}/JSONBench/greptimedb/*.index_size - ${{ runner.temp }}/JSONBench/greptimedb/*.count - ${{ runner.temp }}/JSONBench/greptimedb/*.results_runtime - ${{ runner.temp }}/JSONBench/greptimedb/*.query_results + ./greptimedb.log + ./JSONBench/greptimedb/*.log + ./JSONBench/greptimedb/*.total_size + ./JSONBench/greptimedb/*.data_size + ./JSONBench/greptimedb/*.index_size + ./JSONBench/greptimedb/*.count + ./JSONBench/greptimedb/*.results_runtime + ./JSONBench/greptimedb/*.query_results if-no-files-found: ignore retention-days: 7 stop-linux-arm64-runner: name: Stop Linux ARM64 runner # It's always run as the last job in the workflow to make sure that the runner is released. - if: ${{ always() }} + if: ${{ always() && needs.allocate-runner.outputs.linux-arm64-ec2-runner-instance-id != '' }} runs-on: ubuntu-latest needs: [ allocate-runner, From 935ef9a3613ada227e39b5ebe32d67cadf36cd17 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Tue, 2 Jun 2026 14:43:23 +0800 Subject: [PATCH 15/29] feat: check open region requirements (#8194) * feat: check open region capabilities Signed-off-by: WenyXu * test: allow file region migration tests Signed-off-by: WenyXu * refactor: refine open region requirements Signed-off-by: WenyXu * refactor: use fs scheme constant Signed-off-by: WenyXu * test: cover open region requirement predicate Signed-off-by: WenyXu * fix: check file engine open requirements Signed-off-by: WenyXu --------- Signed-off-by: WenyXu --- src/cmd/src/datanode/scanbench.rs | 1 + src/common/meta/src/instruction.rs | 63 +++++++++++- src/datanode/src/heartbeat/handler.rs | 4 +- .../src/heartbeat/handler/open_region.rs | 35 ++++--- src/datanode/src/region_server.rs | 5 + src/datanode/src/utils.rs | 2 + src/file-engine/src/engine.rs | 72 +++++++++++++- src/file-engine/src/region.rs | 2 + .../region_migration/open_candidate_region.rs | 80 ++++++++++++++-- src/metric-engine/src/engine.rs | 5 + src/metric-engine/src/engine/open.rs | 2 + src/metric-engine/src/engine/sync/region.rs | 1 + src/metric-engine/src/test_util.rs | 1 + src/mito2/Cargo.toml | 1 + src/mito2/src/engine/alter_test.rs | 10 ++ src/mito2/src/engine/append_mode_test.rs | 1 + src/mito2/src/engine/basic_test.rs | 1 + src/mito2/src/engine/batch_catchup_test.rs | 1 + src/mito2/src/engine/batch_open_test.rs | 3 + .../engine/bump_committed_sequence_test.rs | 2 + src/mito2/src/engine/catchup_test.rs | 6 ++ src/mito2/src/engine/compaction_test.rs | 2 + src/mito2/src/engine/open_test.rs | 11 +++ src/mito2/src/engine/parallel_test.rs | 1 + src/mito2/src/engine/skip_wal_test.rs | 3 + src/mito2/src/engine/sync_test.rs | 2 + src/mito2/src/engine/truncate_test.rs | 2 + src/mito2/src/error.rs | 15 +++ src/mito2/src/region/opener.rs | 96 +++++++++++++++++-- src/mito2/src/test_util.rs | 1 + src/mito2/src/worker/handle_open.rs | 17 +++- src/object-store/src/util.rs | 20 ++++ src/store-api/src/region_request.rs | 25 +++++ tests-integration/Cargo.toml | 2 +- 34 files changed, 456 insertions(+), 39 deletions(-) diff --git a/src/cmd/src/datanode/scanbench.rs b/src/cmd/src/datanode/scanbench.rs index b26705991c..b2a715ad31 100644 --- a/src/cmd/src/datanode/scanbench.rs +++ b/src/cmd/src/datanode/scanbench.rs @@ -524,6 +524,7 @@ impl ScanbenchCommand { options: HashMap::default(), skip_wal_replay: !self.enable_wal, checkpoint: None, + requirements: Default::default(), }; engine diff --git a/src/common/meta/src/instruction.rs b/src/common/meta/src/instruction.rs index 3fa6b1bad0..6872b9ad55 100644 --- a/src/common/meta/src/instruction.rs +++ b/src/common/meta/src/instruction.rs @@ -18,7 +18,7 @@ use std::time::Duration; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use store_api::region_engine::SyncRegionFromRequest; -use store_api::region_request::RegionFlushReason; +use store_api::region_request::{RegionFlushReason, RegionRequirements}; use store_api::storage::{FileRefsManifest, GcReport, RegionId, RegionNumber}; use strum::Display; use table::metadata::TableId; @@ -179,12 +179,24 @@ impl Display for OpenRegion { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!( f, - "OpenRegion(region_ident={}, region_storage_path={})", - self.region_ident, self.region_storage_path + "OpenRegion(region_ident={}, region_storage_path={}, reason={:?})", + self.region_ident, self.region_storage_path, self.reason ) } } +/// The reason why an open region instruction is triggered. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum OpenRegionReason { + /// Open triggered before region migration. + RegionMigration, + /// Open triggered by region failover. + RegionFailover, + /// Open triggered when adding a follower region. + #[cfg(feature = "enterprise")] + RegionFollower, +} + #[serde_with::serde_as] #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct OpenRegion { @@ -196,6 +208,10 @@ pub struct OpenRegion { pub region_wal_options: HashMap, #[serde(default)] pub skip_wal_replay: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, + #[serde(default)] + pub requirements: RegionRequirements, } impl OpenRegion { @@ -205,6 +221,8 @@ impl OpenRegion { region_options: HashMap, region_wal_options: HashMap, skip_wal_replay: bool, + reason: Option, + requirements: RegionRequirements, ) -> Self { Self { region_ident, @@ -212,6 +230,8 @@ impl OpenRegion { region_options, region_wal_options, skip_wal_replay, + reason, + requirements, } } } @@ -1126,11 +1146,13 @@ mod tests { HashMap::new(), HashMap::new(), false, + None, + RegionRequirements::empty(), )]); let serialized = serde_json::to_string(&open_region).unwrap(); assert_eq!( - r#"{"OpenRegions":[{"region_ident":{"datanode_id":2,"table_id":1024,"region_number":1,"engine":"mito2"},"region_storage_path":"test/foo","region_options":{},"region_wal_options":{},"skip_wal_replay":false}]}"#, + r#"{"OpenRegions":[{"region_ident":{"datanode_id":2,"table_id":1024,"region_number":1,"engine":"mito2"},"region_storage_path":"test/foo","region_options":{},"region_wal_options":{},"skip_wal_replay":false,"requirements":{"object_storage":false}}]}"#, serialized ); @@ -1213,6 +1235,8 @@ mod tests { HashMap::new(), HashMap::new(), false, + None, + RegionRequirements::empty(), )]); assert_eq!(open_region_instruction, open_region); @@ -1368,10 +1392,41 @@ mod tests { region_options, region_wal_options: HashMap::new(), skip_wal_replay: false, + reason: None, + requirements: RegionRequirements::empty(), }; assert_eq!(expected, deserialized); } + #[test] + fn test_serialize_open_region_with_reason_and_requirements() { + let open_region = OpenRegion::new( + RegionIdent { + datanode_id: 2, + table_id: 1024, + region_number: 1, + engine: "mito2".to_string(), + }, + "test/foo", + HashMap::new(), + HashMap::new(), + false, + Some(OpenRegionReason::RegionMigration), + RegionRequirements::object_storage(), + ); + + let serialized = serde_json::to_string(&open_region).unwrap(); + assert!(serialized.contains(r#""reason":"RegionMigration""#)); + assert!(serialized.contains(r#""object_storage":true"#)); + + let deserialized: OpenRegion = serde_json::from_str(&serialized).unwrap(); + assert_eq!(Some(OpenRegionReason::RegionMigration), deserialized.reason); + assert_eq!( + RegionRequirements::object_storage(), + deserialized.requirements + ); + } + #[test] fn test_flush_regions_creation() { let region_id = RegionId::new(1024, 1); diff --git a/src/datanode/src/heartbeat/handler.rs b/src/datanode/src/heartbeat/handler.rs index 10948a3e7c..79e0baaef3 100644 --- a/src/datanode/src/heartbeat/handler.rs +++ b/src/datanode/src/heartbeat/handler.rs @@ -313,7 +313,7 @@ mod tests { use mito2::test_util::{CreateRequestBuilder, TestEnv}; use store_api::path_utils::table_dir; use store_api::region_engine::RegionRole; - use store_api::region_request::{RegionCloseRequest, RegionRequest}; + use store_api::region_request::{RegionCloseRequest, RegionRequest, RegionRequirements}; use store_api::storage::RegionId; use tokio::sync::mpsc::{self, Receiver}; @@ -442,6 +442,8 @@ mod tests { HashMap::new(), HashMap::new(), false, + None, + RegionRequirements::empty(), )]) } diff --git a/src/datanode/src/heartbeat/handler/open_region.rs b/src/datanode/src/heartbeat/handler/open_region.rs index 56c07a3efe..9c483e588d 100644 --- a/src/datanode/src/heartbeat/handler/open_region.rs +++ b/src/datanode/src/heartbeat/handler/open_region.rs @@ -14,6 +14,7 @@ use common_meta::instruction::{InstructionReply, OpenRegion, SimpleReply}; use common_meta::wal_provider::prepare_wal_options; +use common_telemetry::info; use store_api::path_utils::table_dir; use store_api::region_request::{PathType, RegionOpenRequest}; use store_api::storage::RegionId; @@ -41,8 +42,13 @@ impl InstructionHandler for OpenRegionsHandler { mut region_options, region_wal_options, skip_wal_replay, + reason, + requirements, } = open_region; let region_id = RegionId::new(region_ident.table_id, region_ident.region_number); + info!( + "Received open region instruction, region_id: {region_id}, reason: {reason:?}" + ); prepare_wal_options(&mut region_options, region_id, ®ion_wal_options); let request = RegionOpenRequest { engine: region_ident.engine, @@ -51,6 +57,7 @@ impl InstructionHandler for OpenRegionsHandler { options: region_options, skip_wal_replay, checkpoint: None, + requirements, }; (region_id, request) }) @@ -85,7 +92,7 @@ mod tests { use mito2::engine::MITO_ENGINE_NAME; use mito2::test_util::{CreateRequestBuilder, TestEnv}; use store_api::path_utils::table_dir; - use store_api::region_request::{RegionCloseRequest, RegionRequest}; + use store_api::region_request::{RegionCloseRequest, RegionRequest, RegionRequirements}; use store_api::storage::RegionId; use crate::heartbeat::handler::RegionHeartbeatResponseHandler; @@ -98,17 +105,21 @@ mod tests { ) -> Instruction { let region_idents = region_ids .into_iter() - .map(|region_id| OpenRegion { - region_ident: RegionIdent { - datanode_id: 0, - table_id: region_id.table_id(), - region_number: region_id.region_number(), - engine: MITO_ENGINE_NAME.to_string(), - }, - region_storage_path: storage_path.to_string(), - region_options: HashMap::new(), - region_wal_options: HashMap::new(), - skip_wal_replay: false, + .map(|region_id| { + OpenRegion::new( + RegionIdent { + datanode_id: 0, + table_id: region_id.table_id(), + region_number: region_id.region_number(), + engine: MITO_ENGINE_NAME.to_string(), + }, + storage_path, + HashMap::new(), + HashMap::new(), + false, + None, + RegionRequirements::empty(), + ) }) .collect(); diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs index d5711e1761..e2c19a318e 100644 --- a/src/datanode/src/region_server.rs +++ b/src/datanode/src/region_server.rs @@ -2057,6 +2057,7 @@ mod tests { options: Default::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -2235,6 +2236,7 @@ mod tests { options: Default::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }, ), ( @@ -2246,6 +2248,7 @@ mod tests { options: Default::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }, ), ], @@ -2268,6 +2271,7 @@ mod tests { options: Default::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }, ), ( @@ -2279,6 +2283,7 @@ mod tests { options: Default::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }, ), ], diff --git a/src/datanode/src/utils.rs b/src/datanode/src/utils.rs index c5cd008c28..816ae021ba 100644 --- a/src/datanode/src/utils.rs +++ b/src/datanode/src/utils.rs @@ -175,6 +175,7 @@ pub async fn build_region_open_requests( options, skip_wal_replay: false, checkpoint, + requirements: Default::default(), }, )); } @@ -193,6 +194,7 @@ pub async fn build_region_open_requests( options, skip_wal_replay: true, checkpoint: None, + requirements: Default::default(), }, )); } diff --git a/src/file-engine/src/engine.rs b/src/file-engine/src/engine.rs index 175ebef237..2ddbb6c414 100644 --- a/src/file-engine/src/engine.rs +++ b/src/file-engine/src/engine.rs @@ -32,7 +32,7 @@ use store_api::region_engine::{ }; use store_api::region_request::{ AffectedRows, RegionCloseRequest, RegionCreateRequest, RegionDropRequest, RegionOpenRequest, - RegionRequest, + RegionRequest, RegionRequirements, }; use store_api::storage::{RegionId, ScanRequest, SequenceNumber}; use tokio::sync::Mutex; @@ -186,6 +186,24 @@ struct EngineInner { type EngineInnerRef = Arc; +fn ensure_open_requirements( + requirements: RegionRequirements, + object_store: &ObjectStore, +) -> EngineResult<()> { + if !requirements.object_storage { + return Ok(()); + } + + ensure!( + object_store::util::is_object_storage(object_store), + UnsupportedSnafu { + operation: "open region with object storage requirement on non-object storage" + } + ); + + Ok(()) +} + impl EngineInner { fn new(object_store: ObjectStore) -> Self { Self { @@ -289,6 +307,8 @@ impl EngineInner { return Ok(0); } + ensure_open_requirements(request.requirements, &self.object_store)?; + let res = FileRegion::open(region_id, request, &self.object_store).await; let region = res.inspect_err(|err| { error!( @@ -356,3 +376,53 @@ impl EngineInner { self.regions.read().unwrap().contains_key(®ion_id) } } + +#[cfg(test)] +mod tests { + use object_store::services::{Fs, S3}; + + use super::*; + use crate::error::Error; + + fn build_fs_object_store() -> ObjectStore { + ObjectStore::new(Fs::default().root("/tmp")) + .unwrap() + .finish() + } + + fn build_s3_object_store() -> ObjectStore { + ObjectStore::new( + S3::default() + .bucket("test-bucket") + .region("us-east-1") + .disable_ec2_metadata(), + ) + .unwrap() + .finish() + } + + #[test] + fn test_empty_open_requirements_are_supported() { + ensure_open_requirements(RegionRequirements::empty(), &build_fs_object_store()).unwrap(); + } + + #[test] + fn test_object_storage_open_requirement_rejects_fs_object_store() { + let err = ensure_open_requirements( + RegionRequirements::object_storage(), + &build_fs_object_store(), + ) + .unwrap_err(); + + assert!(matches!(err, Error::Unsupported { .. })); + } + + #[test] + fn test_object_storage_open_requirement_accepts_s3_object_store() { + ensure_open_requirements( + RegionRequirements::object_storage(), + &build_s3_object_store(), + ) + .unwrap(); + } +} diff --git a/src/file-engine/src/region.rs b/src/file-engine/src/region.rs index 3808b33a67..aceec21aa5 100644 --- a/src/file-engine/src/region.rs +++ b/src/file-engine/src/region.rs @@ -181,6 +181,7 @@ mod tests { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }; let region = FileRegion::open(region_id, request, &object_store) @@ -238,6 +239,7 @@ mod tests { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }; let err = FileRegion::open(region_id, request, &object_store) .await diff --git a/src/meta-srv/src/procedure/region_migration/open_candidate_region.rs b/src/meta-srv/src/procedure/region_migration/open_candidate_region.rs index 0c0e5de5d7..792c66bdc9 100644 --- a/src/meta-srv/src/procedure/region_migration/open_candidate_region.rs +++ b/src/meta-srv/src/procedure/region_migration/open_candidate_region.rs @@ -18,7 +18,9 @@ use std::ops::Div; use api::v1::meta::MailboxMessage; use common_meta::RegionIdent; use common_meta::distributed_time_constants::default_distributed_time_constants; -use common_meta::instruction::{Instruction, InstructionReply, OpenRegion, SimpleReply}; +use common_meta::instruction::{ + Instruction, InstructionReply, OpenRegion, OpenRegionReason, SimpleReply, +}; use common_meta::key::datanode_table::RegionInfo; use common_procedure::{Context as ProcedureContext, Status}; use common_telemetry::info; @@ -26,12 +28,13 @@ use common_telemetry::tracing_context::TracingContext; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt}; use store_api::region_engine::RegionRole; +use store_api::region_request::RegionRequirements; use tokio::time::Instant; use crate::error::{self, Result}; use crate::handler::HeartbeatMailbox; use crate::procedure::region_migration::flush_leader_region::PreFlushRegion; -use crate::procedure::region_migration::{Context, State}; +use crate::procedure::region_migration::{Context, RegionMigrationTriggerReason, State}; use crate::service::mailbox::Channel; #[derive(Debug, Serialize, Deserialize)] @@ -67,6 +70,10 @@ impl OpenCandidateRegion { let region_ids = ctx.persistent_ctx.region_ids.clone(); let from_peer_id = ctx.persistent_ctx.from_peer.id; let to_peer_id = ctx.persistent_ctx.to_peer.id; + let reason = match ctx.persistent_ctx.trigger_reason { + RegionMigrationTriggerReason::Failover => OpenRegionReason::RegionFailover, + _ => OpenRegionReason::RegionMigration, + }; let datanode_table_values = ctx.get_from_peer_datanode_table_values().await?; let mut open_regions = Vec::with_capacity(region_ids.len()); @@ -97,6 +104,8 @@ impl OpenCandidateRegion { region_options, region_wal_options, true, + Some(reason), + RegionRequirements::object_storage(), )); } @@ -233,18 +242,20 @@ mod tests { } fn new_mock_open_instruction(datanode_id: DatanodeId, region_id: RegionId) -> Instruction { - Instruction::OpenRegions(vec![OpenRegion { - region_ident: RegionIdent { + Instruction::OpenRegions(vec![OpenRegion::new( + RegionIdent { datanode_id, table_id: region_id.table_id(), region_number: region_id.region_number(), engine: MITO2_ENGINE.to_string(), }, - region_storage_path: "/bar/foo/region/".to_string(), - region_options: Default::default(), - region_wal_options: Default::default(), - skip_wal_replay: true, - }]) + "/bar/foo/region/", + Default::default(), + Default::default(), + true, + Some(OpenRegionReason::RegionMigration), + RegionRequirements::object_storage(), + )]) } #[tokio::test] @@ -263,6 +274,57 @@ mod tests { assert!(!err.is_retryable()); } + #[tokio::test] + async fn test_build_open_region_instruction_reason() { + let state = OpenCandidateRegion; + let mut persistent_context = new_persistent_context(); + let from_peer_id = persistent_context.from_peer.id; + let region_id = persistent_context.region_ids[0]; + let env = TestingEnv::new(); + + let table_info = new_test_table_info(1024); + let region_routes = vec![RegionRoute { + region: Region::new_test(region_id), + leader_peer: Some(Peer::empty(from_peer_id)), + ..Default::default() + }]; + env.table_metadata_manager() + .create_table_metadata( + table_info, + TableRouteValue::physical(region_routes), + HashMap::default(), + ) + .await + .unwrap(); + + let mut ctx = env + .context_factory() + .new_context(persistent_context.clone()); + let instruction = state.build_open_region_instruction(&mut ctx).await.unwrap(); + let open_regions = instruction.into_open_regions().unwrap(); + assert_eq!( + Some(OpenRegionReason::RegionMigration), + open_regions[0].reason + ); + assert_eq!( + RegionRequirements::object_storage(), + open_regions[0].requirements + ); + + persistent_context.trigger_reason = RegionMigrationTriggerReason::Failover; + let mut ctx = env.context_factory().new_context(persistent_context); + let instruction = state.build_open_region_instruction(&mut ctx).await.unwrap(); + let open_regions = instruction.into_open_regions().unwrap(); + assert_eq!( + Some(OpenRegionReason::RegionFailover), + open_regions[0].reason + ); + assert_eq!( + RegionRequirements::object_storage(), + open_regions[0].requirements + ); + } + #[tokio::test] async fn test_datanode_is_unreachable() { let state = OpenCandidateRegion; diff --git a/src/metric-engine/src/engine.rs b/src/metric-engine/src/engine.rs index ef4d802cfc..fa9ef804cc 100644 --- a/src/metric-engine/src/engine.rs +++ b/src/metric-engine/src/engine.rs @@ -620,6 +620,7 @@ mod test { options: physical_region_option, skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }; engine .handle_request(physical_region_id, RegionRequest::Open(open_request)) @@ -644,6 +645,7 @@ mod test { options: HashMap::new(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }; engine .handle_request( @@ -721,6 +723,7 @@ mod test { options: physical_region_option, skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }; // Opening an already opened region should succeed. // Since the region is already open, no metadata recovery operations will be performed. @@ -749,6 +752,7 @@ mod test { options: physical_region_option, skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }; let err = metric_engine .handle_request(physical_region_id, RegionRequest::Open(open_request)) @@ -854,6 +858,7 @@ mod test { options: options.clone(), skip_wal_replay: true, checkpoint: None, + requirements: Default::default(), }, ) }) diff --git a/src/metric-engine/src/engine/open.rs b/src/metric-engine/src/engine/open.rs index 59b1cfd928..8fcdfcd821 100644 --- a/src/metric-engine/src/engine/open.rs +++ b/src/metric-engine/src/engine/open.rs @@ -222,6 +222,7 @@ impl MetricEngineInner { entry_id: checkpoint.metadata_entry_id.unwrap_or_default(), metadata_entry_id: None, }), + requirements: request.requirements, }; let mut data_region_options = request.options; @@ -239,6 +240,7 @@ impl MetricEngineInner { entry_id: checkpoint.entry_id, metadata_entry_id: None, }), + requirements: request.requirements, }; (open_metadata_region_request, open_data_region_request) diff --git a/src/metric-engine/src/engine/sync/region.rs b/src/metric-engine/src/engine/sync/region.rs index cbe6515a19..d1f92bef64 100644 --- a/src/metric-engine/src/engine/sync/region.rs +++ b/src/metric-engine/src/engine/sync/region.rs @@ -321,6 +321,7 @@ mod tests { options: physical_region_option, skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await diff --git a/src/metric-engine/src/test_util.rs b/src/metric-engine/src/test_util.rs index ec55a01903..8d4a822b6b 100644 --- a/src/metric-engine/src/test_util.rs +++ b/src/metric-engine/src/test_util.rs @@ -144,6 +144,7 @@ impl TestEnv { options: physical_region_option, skip_wal_replay: true, checkpoint: None, + requirements: Default::default(), }), ) .await diff --git a/src/mito2/Cargo.toml b/src/mito2/Cargo.toml index 99e3439879..ea281f2c32 100644 --- a/src/mito2/Cargo.toml +++ b/src/mito2/Cargo.toml @@ -8,6 +8,7 @@ license.workspace = true default = [] test = ["common-test-util", "rstest", "rstest_reuse", "rskafka"] testing = ["test"] +test-shared-fs-region-migration = [] enterprise = [] vector_index = ["dep:roaring", "index/vector_index"] diff --git a/src/mito2/src/engine/alter_test.rs b/src/mito2/src/engine/alter_test.rs index a7798a7678..b43f057ea6 100644 --- a/src/mito2/src/engine/alter_test.rs +++ b/src/mito2/src/engine/alter_test.rs @@ -277,6 +277,7 @@ async fn test_alter_region_with_format(flat_format: bool) { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -481,6 +482,7 @@ async fn test_put_after_alter_with_format(flat_format: bool) { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -844,6 +846,7 @@ async fn test_alter_column_fulltext_options_with_format(flat_format: bool) { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -979,6 +982,7 @@ async fn test_alter_column_set_inverted_index_with_format(flat_format: bool) { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -1248,6 +1252,7 @@ async fn test_alter_region_sst_format_with_flush() { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -1366,6 +1371,7 @@ async fn test_alter_region_sst_format_without_flush() { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -1492,6 +1498,7 @@ async fn test_alter_region_sst_format_flat_to_pk_with_flush() { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -1610,6 +1617,7 @@ async fn test_alter_region_sst_format_flat_to_pk_without_flush() { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -1725,6 +1733,7 @@ async fn test_alter_region_append_mode_with_flush() { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -1843,6 +1852,7 @@ async fn test_alter_region_append_mode_without_flush() { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await diff --git a/src/mito2/src/engine/append_mode_test.rs b/src/mito2/src/engine/append_mode_test.rs index de58e04e46..188e28ccf5 100644 --- a/src/mito2/src/engine/append_mode_test.rs +++ b/src/mito2/src/engine/append_mode_test.rs @@ -348,6 +348,7 @@ async fn test_alter_append_mode_clears_merge_mode_with_format(flat_format: bool) options, skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await diff --git a/src/mito2/src/engine/basic_test.rs b/src/mito2/src/engine/basic_test.rs index e1e462f692..0cc122573e 100644 --- a/src/mito2/src/engine/basic_test.rs +++ b/src/mito2/src/engine/basic_test.rs @@ -196,6 +196,7 @@ async fn test_region_replay_with_format(factory: Option, flat_f options, skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await diff --git a/src/mito2/src/engine/batch_catchup_test.rs b/src/mito2/src/engine/batch_catchup_test.rs index dc0b552adc..a3808b1999 100644 --- a/src/mito2/src/engine/batch_catchup_test.rs +++ b/src/mito2/src/engine/batch_catchup_test.rs @@ -160,6 +160,7 @@ async fn test_batch_catchup_with_format(factory: Option, flat_f skip_wal_replay: true, path_type: PathType::Bare, checkpoint: None, + requirements: Default::default(), }, ) }) diff --git a/src/mito2/src/engine/batch_open_test.rs b/src/mito2/src/engine/batch_open_test.rs index 6b16b3c120..2522cf2f84 100644 --- a/src/mito2/src/engine/batch_open_test.rs +++ b/src/mito2/src/engine/batch_open_test.rs @@ -136,6 +136,7 @@ async fn test_batch_open_with_format(factory: Option, flat_form skip_wal_replay: false, path_type: PathType::Bare, checkpoint: None, + requirements: Default::default(), }, ) }) @@ -149,6 +150,7 @@ async fn test_batch_open_with_format(factory: Option, flat_form skip_wal_replay: false, path_type: PathType::Bare, checkpoint: None, + requirements: Default::default(), }, )); @@ -221,6 +223,7 @@ async fn test_batch_open_err_with_format(factory: Option, flat_ skip_wal_replay: false, path_type: PathType::Bare, checkpoint: None, + requirements: Default::default(), }, ) }) diff --git a/src/mito2/src/engine/bump_committed_sequence_test.rs b/src/mito2/src/engine/bump_committed_sequence_test.rs index 12db0044c5..23a5af8865 100644 --- a/src/mito2/src/engine/bump_committed_sequence_test.rs +++ b/src/mito2/src/engine/bump_committed_sequence_test.rs @@ -112,6 +112,7 @@ async fn test_bump_committed_sequence_with_format(flat_format: bool) { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -151,6 +152,7 @@ async fn test_bump_committed_sequence_with_format(flat_format: bool) { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await diff --git a/src/mito2/src/engine/catchup_test.rs b/src/mito2/src/engine/catchup_test.rs index e10e91b51b..b79a2b0625 100644 --- a/src/mito2/src/engine/catchup_test.rs +++ b/src/mito2/src/engine/catchup_test.rs @@ -97,6 +97,7 @@ async fn test_catchup_with_last_entry_id(factory: Option) { options, skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -218,6 +219,7 @@ async fn test_catchup_with_incorrect_last_entry_id(factory: Option) { options, skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -423,6 +426,7 @@ async fn test_catchup_with_manifest_update(factory: Option) { options, skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -527,6 +531,7 @@ async fn open_region( skip_wal_replay, path_type: PathType::Bare, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -622,6 +627,7 @@ async fn test_local_catchup(factory: Option) { skip_wal_replay: true, path_type: PathType::Bare, checkpoint: None, + requirements: Default::default(), }), ) .await diff --git a/src/mito2/src/engine/compaction_test.rs b/src/mito2/src/engine/compaction_test.rs index f76e9f8bf9..fd0982b7e5 100644 --- a/src/mito2/src/engine/compaction_test.rs +++ b/src/mito2/src/engine/compaction_test.rs @@ -1023,6 +1023,7 @@ async fn test_change_region_compaction_window_with_format(flat_format: bool) { options: Default::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -1125,6 +1126,7 @@ async fn test_open_overwrite_compaction_window_with_format(flat_format: bool) { options, skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await diff --git a/src/mito2/src/engine/open_test.rs b/src/mito2/src/engine/open_test.rs index 28ad1de71e..11279954a9 100644 --- a/src/mito2/src/engine/open_test.rs +++ b/src/mito2/src/engine/open_test.rs @@ -64,6 +64,7 @@ async fn test_engine_open_empty_with_format(flat_format: bool) { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -110,6 +111,7 @@ async fn test_engine_open_existing_with_format(flat_format: bool) { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -237,6 +239,7 @@ async fn test_engine_region_open_with_options_with_format(flat_format: bool) { options: HashMap::from([("ttl".to_string(), "4d".to_string())]), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -297,6 +300,7 @@ async fn test_engine_region_open_with_custom_store_with_format(flat_format: bool options: HashMap::from([("storage".to_string(), "Gcs".to_string())]), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -392,6 +396,7 @@ async fn test_open_region_skip_wal_replay_with_format(flat_format: bool) { options: Default::default(), skip_wal_replay: true, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -431,6 +436,7 @@ async fn test_open_region_skip_wal_replay_with_format(flat_format: bool) { options: Default::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -484,6 +490,7 @@ async fn test_open_region_wait_for_opening_region_ok_with_format(flat_format: bo options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -535,6 +542,7 @@ async fn test_open_region_wait_for_opening_region_err_with_format(flat_format: b options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -691,6 +699,7 @@ async fn test_open_backfills_partition_expr_with_fetcher() { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -725,6 +734,7 @@ async fn test_open_backfills_partition_expr_with_fetcher() { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -766,6 +776,7 @@ async fn test_open_keeps_none_without_fetcher() { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await diff --git a/src/mito2/src/engine/parallel_test.rs b/src/mito2/src/engine/parallel_test.rs index b88a60739b..5a1354ec15 100644 --- a/src/mito2/src/engine/parallel_test.rs +++ b/src/mito2/src/engine/parallel_test.rs @@ -52,6 +52,7 @@ async fn scan_in_parallel( skip_wal_replay: false, path_type: PathType::Bare, checkpoint: None, + requirements: Default::default(), }), ) .await diff --git a/src/mito2/src/engine/skip_wal_test.rs b/src/mito2/src/engine/skip_wal_test.rs index 97f159b8ac..3b6cf89f07 100644 --- a/src/mito2/src/engine/skip_wal_test.rs +++ b/src/mito2/src/engine/skip_wal_test.rs @@ -87,6 +87,7 @@ async fn test_close_region_skip_wal(insert: bool) { options: request.options.clone(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -154,6 +155,7 @@ async fn test_close_follower_region_skip_wal() { options: request.options.clone(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -271,6 +273,7 @@ async fn test_close_region_after_truncate_skip_wal() { options: request.options, skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await diff --git a/src/mito2/src/engine/sync_test.rs b/src/mito2/src/engine/sync_test.rs index 17d73b1848..657ee868ce 100644 --- a/src/mito2/src/engine/sync_test.rs +++ b/src/mito2/src/engine/sync_test.rs @@ -127,6 +127,7 @@ async fn test_sync_after_flush_region_with_format(flat_format: bool) { // Ensure the region is not replayed from the WAL. skip_wal_replay: true, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -239,6 +240,7 @@ async fn test_sync_after_alter_region_with_format(flat_format: bool) { // Ensure the region is not replayed from the WAL. skip_wal_replay: true, checkpoint: None, + requirements: Default::default(), }), ) .await diff --git a/src/mito2/src/engine/truncate_test.rs b/src/mito2/src/engine/truncate_test.rs index 8c3fdad75d..8c6dd023f0 100644 --- a/src/mito2/src/engine/truncate_test.rs +++ b/src/mito2/src/engine/truncate_test.rs @@ -323,6 +323,7 @@ async fn test_engine_truncate_reopen_with_format(flat_format: bool) { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await @@ -447,6 +448,7 @@ async fn test_engine_truncate_during_flush_with_format(flat_format: bool) { options: HashMap::default(), skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), ) .await diff --git a/src/mito2/src/error.rs b/src/mito2/src/error.rs index 3571f7c0c4..2278b61669 100644 --- a/src/mito2/src/error.rs +++ b/src/mito2/src/error.rs @@ -916,6 +916,20 @@ pub enum Error { source: Arc, }, + #[snafu(display( + "Region {} does not satisfy open requirement '{}': {}", + region_id, + requirement, + reason + ))] + OpenRegionRequirement { + region_id: RegionId, + requirement: &'static str, + reason: &'static str, + #[snafu(implicit)] + location: Location, + }, + #[snafu(display("Failed to parse job id"))] ParseJobId { #[snafu(implicit)] @@ -1376,6 +1390,7 @@ impl ErrorExt for Error { PrimaryKeyLengthMismatch { .. } => StatusCode::InvalidArguments, InvalidSender { .. } => StatusCode::InvalidArguments, InvalidSchedulerState { .. } => StatusCode::InvalidArguments, + OpenRegionRequirement { .. } => StatusCode::InvalidArguments, DeleteSsts { .. } | DeleteIndex { .. } | DeleteIndexes { .. } => { StatusCode::StorageUnavailable } diff --git a/src/mito2/src/region/opener.rs b/src/mito2/src/region/opener.rs index 3142a87c38..412172aead 100644 --- a/src/mito2/src/region/opener.rs +++ b/src/mito2/src/region/opener.rs @@ -27,8 +27,9 @@ use futures::future::BoxFuture; use log_store::kafka::log_store::KafkaLogStore; use log_store::noop::log_store::NoopLogStore; use log_store::raft_engine::log_store::RaftEngineLogStore; +use object_store::ObjectStore; use object_store::manager::ObjectStoreManagerRef; -use object_store::util::normalize_dir; +use object_store::util::{is_object_storage, normalize_dir}; use snafu::{OptionExt, ResultExt, ensure}; use store_api::logstore::LogStore; use store_api::logstore::provider::Provider; @@ -36,7 +37,7 @@ use store_api::metadata::{ ColumnMetadata, RegionMetadata, RegionMetadataBuilder, RegionMetadataRef, }; use store_api::region_engine::RegionRole; -use store_api::region_request::PathType; +use store_api::region_request::{PathType, RegionRequirements}; use store_api::storage::{ColumnId, RegionId}; use tokio::sync::Semaphore; @@ -46,8 +47,8 @@ use crate::cache::file_cache::{FileCache, FileType, IndexKey}; use crate::config::MitoConfig; use crate::error; use crate::error::{ - EmptyRegionDirSnafu, InvalidMetadataSnafu, ObjectStoreNotFoundSnafu, RegionCorruptedSnafu, - Result, StaleLogEntrySnafu, + EmptyRegionDirSnafu, InvalidMetadataSnafu, InvalidRegionOptionsSnafu, ObjectStoreNotFoundSnafu, + RegionCorruptedSnafu, Result, StaleLogEntrySnafu, }; use crate::manifest::action::RegionManifest; use crate::manifest::manager::{RegionManifestManager, RegionManifestOptions}; @@ -206,6 +207,29 @@ impl RegionOpener { Ok(self) } + /// Ensures the current region open request satisfies its requirements. + pub(crate) fn ensure_open_requirements(&self, requirements: RegionRequirements) -> Result<()> { + if !requirements.object_storage { + return Ok(()); + } + + let options = self.options.as_ref().context(InvalidRegionOptionsSnafu { + reason: "missing region options before requirement check".to_string(), + })?; + let object_store = get_object_store(&options.storage, &self.object_store_manager)?; + + ensure!( + supports_open_region_object_storage_requirement(&object_store), + error::OpenRegionRequirementSnafu { + region_id: self.region_id, + requirement: "object storage", + reason: "region data must be accessible from another datanode", + } + ); + + Ok(()) + } + /// Sets the cache manager for the region. pub(crate) fn cache(mut self, cache_manager: Option) -> Self { self.cache_manager = cache_manager; @@ -597,6 +621,21 @@ impl RegionOpener { } } +#[cfg(not(feature = "test-shared-fs-region-migration"))] +fn supports_open_region_object_storage_requirement(object_store: &ObjectStore) -> bool { + is_object_storage(object_store) +} + +#[cfg(feature = "test-shared-fs-region-migration")] +fn supports_open_region_object_storage_requirement(object_store: &ObjectStore) -> bool { + // Integration tests can configure multiple datanodes to share the same + // temporary home dir. That makes file storage accessible to all test + // datanodes, but production file storage still does not satisfy this + // requirement. + is_object_storage(object_store) + || object_store.info().scheme() == object_store::services::FS_SCHEME +} + /// Creates a version builder from a region manifest. pub(crate) fn version_builder_from_manifest( manifest: &RegionManifest, @@ -1172,14 +1211,17 @@ mod tests { use datatypes::arrow::array::{ArrayRef, BinaryArray, Int64Array}; use datatypes::arrow::record_batch::RecordBatch; use object_store::ObjectStore; - use object_store::services::{Fs, Memory}; + use object_store::services::{Fs, Memory, S3}; use parquet::arrow::ArrowWriter; use parquet::file::metadata::KeyValue; use parquet::file::properties::WriterProperties; use store_api::region_request::PathType; use store_api::storage::{FileId, RegionId}; - use super::{preload_parquet_meta_cache_for_files, sanitize_region_options}; + use super::{ + preload_parquet_meta_cache_for_files, sanitize_region_options, + supports_open_region_object_storage_requirement, + }; use crate::cache::CacheManager; use crate::cache::file_cache::{FileType, IndexKey}; use crate::manifest::action::{RegionManifest, RemovedFilesRecord}; @@ -1207,6 +1249,48 @@ mod tests { } } + fn build_fs_object_store() -> ObjectStore { + ObjectStore::new(Fs::default().root("/tmp")) + .unwrap() + .finish() + } + + #[test] + #[cfg(not(feature = "test-shared-fs-region-migration"))] + fn test_open_requirement_rejects_fs_object_store() { + let object_store = build_fs_object_store(); + + assert!(!supports_open_region_object_storage_requirement( + &object_store + )); + } + + #[test] + #[cfg(feature = "test-shared-fs-region-migration")] + fn test_open_requirement_accepts_shared_fs_object_store_for_tests() { + let object_store = build_fs_object_store(); + + assert!(supports_open_region_object_storage_requirement( + &object_store + )); + } + + #[test] + fn test_open_requirement_accepts_s3_object_store() { + let object_store = ObjectStore::new( + S3::default() + .bucket("test-bucket") + .region("us-east-1") + .disable_ec2_metadata(), + ) + .unwrap() + .finish(); + + assert!(supports_open_region_object_storage_requirement( + &object_store + )); + } + #[test] fn test_sanitize_region_options_options_format_wins() { // Manifest persisted PrimaryKey, but the re-parsed options now request Flat diff --git a/src/mito2/src/test_util.rs b/src/mito2/src/test_util.rs index c4ca7cc7b3..711fd0a969 100644 --- a/src/mito2/src/test_util.rs +++ b/src/mito2/src/test_util.rs @@ -1282,6 +1282,7 @@ pub async fn reopen_region( skip_wal_replay: false, path_type: PathType::Bare, checkpoint: None, + requirements: Default::default(), }), ) .await diff --git a/src/mito2/src/worker/handle_open.rs b/src/mito2/src/worker/handle_open.rs index 73bdca775c..a154140d98 100644 --- a/src/mito2/src/worker/handle_open.rs +++ b/src/mito2/src/worker/handle_open.rs @@ -87,14 +87,11 @@ impl RegionWorkerLoop { else { return; }; - if let Err(err) = self.check_and_cleanup_region(region_id, &request).await { - sender.send(Err(err)); - return; - } info!("Try to open region {}, worker: {}", region_id, self.id); sanitize_open_request_options(&mut request.options); // Open region from specific region dir. + let requirements = request.requirements; let opener = match RegionOpener::new( region_id, &request.table_dir, @@ -112,7 +109,7 @@ impl RegionWorkerLoop { .cache(Some(self.cache_manager.clone())) .wal_entry_reader(wal_entry_receiver.map(|receiver| Box::new(receiver) as _)) .replay_checkpoint(request.checkpoint.map(|checkpoint| checkpoint.entry_id)) - .parse_options(request.options) + .parse_options(request.options.clone()) { Ok(opener) => opener, Err(err) => { @@ -121,6 +118,16 @@ impl RegionWorkerLoop { } }; + if let Err(err) = opener.ensure_open_requirements(requirements) { + sender.send(Err(err)); + return; + } + + if let Err(err) = self.check_and_cleanup_region(region_id, &request).await { + sender.send(Err(err)); + return; + } + let now = Instant::now(); let regions = self.regions.clone(); let wal = self.wal.clone(); diff --git a/src/object-store/src/util.rs b/src/object-store/src/util.rs index 849f91b729..92f0bd7299 100644 --- a/src/object-store/src/util.rs +++ b/src/object-store/src/util.rs @@ -22,11 +22,17 @@ use opendal::layers::{ LoggingInterceptor, LoggingLayer, RetryEvent, RetryInterceptor, RetryLayer, TracingLayer, }; use opendal::raw::{AccessorInfo, HttpClient, Operation}; +use opendal::services::FS_SCHEME; use snafu::ResultExt; use crate::config::HttpClientConfig; use crate::{ObjectStore, error}; +/// Returns true if the object store is not backed by local filesystem. +pub fn is_object_storage(object_store: &ObjectStore) -> bool { + object_store.info().scheme() != FS_SCHEME +} + /// Join two paths and normalize the output dir. /// /// The output dir is always ends with `/`. e.g. @@ -249,7 +255,11 @@ impl RetryInterceptor for PrintDetailedError { #[cfg(test)] mod tests { + use opendal::services::Fs; + use super::*; + use crate::ObjectStore; + use crate::util::is_object_storage; #[test] fn test_normalize_dir() { @@ -289,4 +299,14 @@ mod tests { assert_eq!("/abc", join_path("//", "/abc")); assert_eq!("abc/def", join_path("abc/", "//def")); } + + #[test] + fn test_fs_is_not_object_storage() { + let object_store = ObjectStore::new(Fs::default().root("/tmp")) + .unwrap() + .finish(); + + assert_eq!(FS_SCHEME, object_store.info().scheme()); + assert!(!is_object_storage(&object_store)); + } } diff --git a/src/store-api/src/region_request.rs b/src/store-api/src/region_request.rs index 951abca1be..abf96736ec 100644 --- a/src/store-api/src/region_request.rs +++ b/src/store-api/src/region_request.rs @@ -315,6 +315,7 @@ fn make_region_open(open: OpenRequest) -> Result> options: open.options, skip_wal_replay: false, checkpoint: None, + requirements: Default::default(), }), )]) } @@ -566,6 +567,28 @@ pub struct RegionDropRequest { pub partial_drop: bool, } +/// Requirements for a region request. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(default)] +pub struct RegionRequirements { + /// Whether the region data must be backed by object storage. + pub object_storage: bool, +} + +impl RegionRequirements { + /// Returns empty requirements. + pub fn empty() -> Self { + Self::default() + } + + /// Returns requirements for object storage. + pub fn object_storage() -> Self { + Self { + object_storage: true, + } + } +} + /// Open region request. #[derive(Debug, Clone)] pub struct RegionOpenRequest { @@ -581,6 +604,8 @@ pub struct RegionOpenRequest { pub skip_wal_replay: bool, /// Replay checkpoint. pub checkpoint: Option, + /// Requirements for opening the region. + pub requirements: RegionRequirements, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/tests-integration/Cargo.toml b/tests-integration/Cargo.toml index 43850e4ed3..f51badc8d6 100644 --- a/tests-integration/Cargo.toml +++ b/tests-integration/Cargo.toml @@ -63,7 +63,7 @@ log-query = { workspace = true } loki-proto.workspace = true meta-client.workspace = true meta-srv = { workspace = true, features = ["mock"] } -mito2.workspace = true +mito2 = { workspace = true, features = ["test-shared-fs-region-migration"] } object-store.workspace = true operator = { workspace = true, features = ["testing"] } plugins.workspace = true From 50b1a07232a624635c365798a6d21875f1316a3f Mon Sep 17 00:00:00 2001 From: Yingwen Date: Tue, 2 Jun 2026 16:00:01 +0800 Subject: [PATCH 16/29] feat(datanode): hold query permit for stream and expose limiter timeout (#8215) * feat(datanode): hold query permit for stream and expose limiter timeout Signed-off-by: evenyag * style(datanode): use as_mut() to poll inner stream in PermitGuardedStream Signed-off-by: evenyag --------- Signed-off-by: evenyag --- config/config.md | 1 + config/datanode.example.toml | 3 ++ src/datanode/src/config.rs | 7 +++ src/datanode/src/datanode.rs | 3 +- src/datanode/src/region_server.rs | 74 +++++++++++++++++++++++++------ 5 files changed, 73 insertions(+), 15 deletions(-) diff --git a/config/config.md b/config/config.md index d9cffaf122..df06d2153c 100644 --- a/config/config.md +++ b/config/config.md @@ -451,6 +451,7 @@ | `init_regions_in_background` | Bool | `false` | Initialize all regions in the background during the startup.
By default, it provides services after all regions have been initialized. | | `init_regions_parallelism` | Integer | `16` | Parallelism of initializing regions. | | `max_concurrent_queries` | Integer | `0` | The maximum concurrent queries allowed to be executed. Zero means unlimited. | +| `concurrent_query_limiter_timeout` | String | `100ms` | Timeout to acquire a permit from the concurrent query limiter when `max_concurrent_queries` is reached. | | `enable_telemetry` | Bool | `true` | Enable telemetry to collect anonymous usage data. Enabled by default. | | `http` | -- | -- | The HTTP server options. | | `http.addr` | String | `127.0.0.1:4000` | The address to bind the HTTP server. | diff --git a/config/datanode.example.toml b/config/datanode.example.toml index d558918daf..9351c4e85d 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -20,6 +20,9 @@ init_regions_parallelism = 16 ## The maximum concurrent queries allowed to be executed. Zero means unlimited. max_concurrent_queries = 0 +## Timeout to acquire a permit from the concurrent query limiter when `max_concurrent_queries` is reached. +concurrent_query_limiter_timeout = "100ms" + ## Enable telemetry to collect anonymous usage data. Enabled by default. #+ enable_telemetry = true diff --git a/src/datanode/src/config.rs b/src/datanode/src/config.rs index 2ce306006b..b757c95121 100644 --- a/src/datanode/src/config.rs +++ b/src/datanode/src/config.rs @@ -14,6 +14,8 @@ //! Datanode configurations +use std::time::Duration; + use common_base::readable_size::ReadableSize; use common_config::{Configurable, DEFAULT_DATA_HOME}; use common_options::memory::MemoryOptions; @@ -75,6 +77,10 @@ pub struct DatanodeOptions { pub wal: DatanodeWalConfig, pub storage: StorageConfig, pub max_concurrent_queries: usize, + /// Timeout to acquire a permit from the concurrent query limiter when + /// `max_concurrent_queries` is reached. Only effective when the limiter is enabled. + #[serde(with = "humantime_serde")] + pub concurrent_query_limiter_timeout: Duration, /// Options for different store engines. pub region_engine: Vec, pub logging: LoggingOptions, @@ -131,6 +137,7 @@ impl Default for DatanodeOptions { wal: DatanodeWalConfig::default(), storage: StorageConfig::default(), max_concurrent_queries: 0, + concurrent_query_limiter_timeout: Duration::from_millis(100), region_engine: vec![ RegionEngineConfig::Mito(MitoConfig::default()), RegionEngineConfig::File(FileEngineConfig::default()), diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs index 9a2fe3d982..12d7c5109c 100644 --- a/src/datanode/src/datanode.rs +++ b/src/datanode/src/datanode.rs @@ -445,8 +445,7 @@ impl DatanodeBuilder { event_listener, table_provider_factory, opts.max_concurrent_queries, - //TODO: revaluate the hardcoded timeout on the next version of datanode concurrency limiter. - Duration::from_millis(100), + opts.concurrent_query_limiter_timeout, opts.grpc.flight_compression, ); diff --git a/src/datanode/src/region_server.rs b/src/datanode/src/region_server.rs index e2c19a318e..ce831353d1 100644 --- a/src/datanode/src/region_server.rs +++ b/src/datanode/src/region_server.rs @@ -49,6 +49,7 @@ use common_telemetry::{debug, error, info, warn}; use dashmap::DashMap; use datafusion::datasource::TableProvider; use datafusion_common::tree_node::TreeNode; +use datatypes::schema::SchemaRef; use either::Either; use futures_util::Stream; use futures_util::future::try_join_all; @@ -82,7 +83,7 @@ use store_api::region_request::{ RegionOpenRequest, RegionRequest, }; use store_api::storage::RegionId; -use tokio::sync::{Semaphore, SemaphorePermit}; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; use tokio::time::timeout; use tonic::{Request, Response, Result as TonicResult}; @@ -257,7 +258,7 @@ impl RegionServer { request: api::v1::region::QueryRequest, query_ctx: QueryContextRef, ) -> Result { - let _permit = if let Some(p) = &self.inner.parallelism { + let permit = if let Some(p) = &self.inner.parallelism { Some(p.acquire().await?) } else { None @@ -298,14 +299,13 @@ impl RegionServer { ) .await?; - Ok(wrap_flow_region_watermark_stream( - stream, region_id, &query_ctx, - )) + let stream = wrap_flow_region_watermark_stream(stream, region_id, &query_ctx); + Ok(maybe_guard_stream(stream, permit)) } #[tracing::instrument(skip_all)] pub async fn handle_read(&self, request: QueryRequest) -> Result { - let _permit = if let Some(p) = &self.inner.parallelism { + let permit = if let Some(p) = &self.inner.parallelism { Some(p.acquire().await?) } else { None @@ -332,9 +332,8 @@ impl RegionServer { .handle_read(QueryRequest { plan, ..request }, query_ctx.clone()) .await?; - Ok(wrap_flow_region_watermark_stream( - stream, region_id, &query_ctx, - )) + let stream = wrap_flow_region_watermark_stream(stream, region_id, &query_ctx); + Ok(maybe_guard_stream(stream, permit)) } /// Returns all opened and reportable regions. @@ -1058,7 +1057,7 @@ struct RegionServerInner { } struct RegionServerParallelism { - semaphore: Semaphore, + semaphore: Arc, timeout: Duration, } @@ -1071,19 +1070,68 @@ impl RegionServerParallelism { return None; } Some(RegionServerParallelism { - semaphore: Semaphore::new(max_concurrent_queries), + semaphore: Arc::new(Semaphore::new(max_concurrent_queries)), timeout: concurrent_query_limiter_timeout, }) } - pub async fn acquire(&self) -> Result> { - timeout(self.timeout, self.semaphore.acquire()) + pub async fn acquire(&self) -> Result { + timeout(self.timeout, self.semaphore.clone().acquire_owned()) .await .context(ConcurrentQueryLimiterTimeoutSnafu)? .context(ConcurrentQueryLimiterClosedSnafu) } } +/// Wraps a record batch stream and holds a concurrency permit until the stream is +/// fully consumed (dropped), so `max_concurrent_queries` bounds the number of +/// in-flight read streams, not just query planning. +struct PermitGuardedStream { + inner: SendableRecordBatchStream, + _permit: OwnedSemaphorePermit, +} + +impl RecordBatchStream for PermitGuardedStream { + fn name(&self) -> &str { + self.inner.name() + } + + fn schema(&self) -> SchemaRef { + self.inner.schema() + } + + fn output_ordering(&self) -> Option<&[OrderOption]> { + self.inner.output_ordering() + } + + fn metrics(&self) -> Option { + self.inner.metrics() + } +} + +impl Stream for PermitGuardedStream { + type Item = common_recordbatch::error::Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.inner.as_mut().poll_next(cx) + } +} + +/// Wraps `stream` so it holds `permit` until fully consumed. Returns `stream` +/// unchanged when no permit was acquired (limiter disabled). +fn maybe_guard_stream( + stream: SendableRecordBatchStream, + permit: Option, +) -> SendableRecordBatchStream { + match permit { + Some(permit) => Box::pin(PermitGuardedStream { + inner: stream, + _permit: permit, + }), + None => stream, + } +} + enum CurrentEngine { Engine(RegionEngineRef), EarlyReturn(AffectedRows), From 7f76ad5439ba985a57f8eda092df10aa85a6d92a Mon Sep 17 00:00:00 2001 From: discord9 Date: Tue, 2 Jun 2026 16:24:50 +0800 Subject: [PATCH 17/29] feat: validate batching flow sink schema on create (#8176) * feat: check schema on create Signed-off-by: discord9 * chore: update sqlness Signed-off-by: discord9 * fix(flow): avoid duplicate fields when matching sink schema Signed-off-by: discord9 * fix: null handling Signed-off-by: discord9 * chore: per review Signed-off-by: discord9 * chore: debug log Signed-off-by: discord9 --------- Signed-off-by: discord9 --- src/flow/src/batching_mode/engine.rs | 5 +- src/flow/src/batching_mode/task.rs | 30 ++ src/flow/src/batching_mode/utils.rs | 243 +++++++-- src/flow/src/batching_mode/utils/test.rs | 499 +++++++++++++++++- .../common/flow/flow_last_non_null.result | 21 +- .../common/flow/flow_last_non_null.sql | 13 +- .../flow/flow_sink_schema_mismatch.result | 123 +++++ .../common/flow/flow_sink_schema_mismatch.sql | 89 ++++ .../flow_tql_missing_value_sink_schema.result | 90 ++++ .../flow_tql_missing_value_sink_schema.sql | 55 ++ 10 files changed, 1093 insertions(+), 75 deletions(-) create mode 100644 tests/cases/standalone/common/flow/flow_sink_schema_mismatch.result create mode 100644 tests/cases/standalone/common/flow/flow_sink_schema_mismatch.sql create mode 100644 tests/cases/standalone/flow-tql/flow_tql_missing_value_sink_schema.result create mode 100644 tests/cases/standalone/flow-tql/flow_tql_missing_value_sink_schema.sql diff --git a/src/flow/src/batching_mode/engine.rs b/src/flow/src/batching_mode/engine.rs index 68fb3793e4..319ddcf2e7 100644 --- a/src/flow/src/batching_mode/engine.rs +++ b/src/flow/src/batching_mode/engine.rs @@ -630,8 +630,11 @@ impl BatchingEngine { let engine = self.query_engine.clone(); let frontend = self.frontend_client.clone(); - // check execute once first to detect any error early + // Create sink table if needed, then validate an existing/created sink schema before + // spawning the background task. This catches user-created sink schema mismatches at + // CREATE FLOW time instead of surfacing them later in the execution loop. task.check_or_create_sink_table(&engine, &frontend).await?; + task.validate_sink_table_schema(&engine).await?; let (start_tx, start_rx) = oneshot::channel(); diff --git a/src/flow/src/batching_mode/task.rs b/src/flow/src/batching_mode/task.rs index cbd6a05cc2..5bd328ad9a 100644 --- a/src/flow/src/batching_mode/task.rs +++ b/src/flow/src/batching_mode/task.rs @@ -265,6 +265,36 @@ impl BatchingTask { Ok(None) } + /// Validates that the sink table schema can accept this flow's output. + /// + /// This is a dry-run of the same schema matching logic used by runtime insert-plan + /// generation, but without adding dirty-window filters or executing the query. It is used + /// during CREATE FLOW to catch existing sink table mismatches early. + pub async fn validate_sink_table_schema(&self, engine: &QueryEngineRef) -> Result<(), Error> { + let (table, _) = get_table_info_df_schema( + self.config.catalog_manager.clone(), + self.config.sink_table_name.clone(), + ) + .await?; + + let table_meta = &table.table_info().meta; + let merge_mode_last_non_null = + is_merge_mode_last_non_null(&table_meta.options.extra_options); + let primary_key_indices = table_meta.primary_key_indices.clone(); + let query_ctx = self.state.read().unwrap().query_ctx.clone(); + + gen_plan_with_matching_schema( + &self.config.query, + query_ctx, + engine.clone(), + table_meta.schema.clone(), + &primary_key_indices, + merge_mode_last_non_null, + ) + .await + .map(|_| ()) + } + async fn is_table_exist(&self, table_name: &[String; 3]) -> Result { self.config .catalog_manager diff --git a/src/flow/src/batching_mode/utils.rs b/src/flow/src/batching_mode/utils.rs index e86b1ee3be..5e033c6ae7 100644 --- a/src/flow/src/batching_mode/utils.rs +++ b/src/flow/src/batching_mode/utils.rs @@ -33,9 +33,10 @@ use datafusion_common::{ }; use datafusion_expr::logical_plan::{Aggregate, TableScan}; use datafusion_expr::{ - Distinct, JoinType, LogicalPlan, LogicalPlanBuilder, Operator, Projection, and, binary_expr, - bitwise_and, bitwise_or, bitwise_xor, is_null, or, when, + Distinct, ExprSchemable, JoinType, LogicalPlan, LogicalPlanBuilder, Operator, Projection, and, + binary_expr, bitwise_and, bitwise_or, bitwise_xor, is_null, or, when, }; +use datatypes::prelude::ConcreteDataType; use datatypes::schema::{ColumnSchema, SchemaRef}; use query::QueryEngineRef; use query::parser::{DEFAULT_LOOKBACK_STRING, PromQuery, QueryLanguageParser, QueryStatement}; @@ -955,7 +956,7 @@ pub(crate) async fn gen_plan_with_matching_schema( .clone() .rewrite(&mut add_auto_column) .with_context(|_| DatafusionSnafu { - context: format!("Failed to rewrite plan:\n {}\n", plan), + context: "Failed to rewrite plan".to_string(), })? .data; Ok(plan) @@ -1090,33 +1091,23 @@ impl ColumnMatcherRewriter { } /// modify the exprs in place so that it matches the schema and some auto columns are added - fn modify_project_exprs(&mut self, mut exprs: Vec) -> DfResult> { + fn modify_project_exprs( + &mut self, + mut exprs: Vec, + input_schema: &DFSchema, + ) -> DfResult> { if self.allow_partial { return self.modify_project_exprs_with_partial(exprs); } + let original_exprs = exprs.clone(); + let all_names = self .schema .column_schemas() .iter() .map(|c| c.name.clone()) .collect::>(); - // first match by position - for (idx, expr) in exprs.iter_mut().enumerate() { - if !all_names.contains(&expr.qualified_name().1) - && let Some(col_name) = self - .schema - .column_schemas() - .get(idx) - .map(|c| c.name.clone()) - { - // if the data type mismatched, later check_execute will error out - // hence no need to check it here, beside, optimize pass might be able to cast it - // so checking here is not necessary - *expr = expr.clone().alias(col_name); - } - } - // add columns if have different column count let query_col_cnt = exprs.len(); let table_col_cnt = self.schema.column_schemas().len(); @@ -1140,10 +1131,9 @@ impl ColumnMatcherRewriter { // is the update at column exprs.push(datafusion::prelude::now().alias(&last_col_schema.name)); } else { - // helpful error message - return Err(DataFusionError::Plan(format!( - "Expect the last column in table to be timestamp column, found column {} with type {:?}", - last_col_schema.name, last_col_schema.data_type + return Err(DataFusionError::Plan(format_flow_sink_schema_mismatch( + &original_exprs, + self.schema.as_ref(), ))); } } else if query_col_cnt + 2 == table_col_cnt { @@ -1170,14 +1160,110 @@ impl ColumnMatcherRewriter { ))); } } else { - return Err(DataFusionError::Plan(format!( - "Expect table have 0,1 or 2 columns more than query columns, found {} query columns {:?}, {} table columns {:?}", - query_col_cnt, - exprs, - table_col_cnt, - self.schema.column_schemas() + return Err(DataFusionError::Plan(format_flow_sink_schema_mismatch( + &original_exprs, + self.schema.as_ref(), ))); } + + self.match_extra_output_columns(exprs, input_schema, &original_exprs, &all_names) + } + + /// Match flow output columns whose names are not in the sink schema by the same position only. + /// + /// This keeps the legacy "omit output aliases and map by position" behavior, but only when the + /// sink column at the same index is actually missing from the flow output. If the extra output + /// would be aliased to a sink column that already exists elsewhere, report a schema mismatch + /// instead of guessing another sink column by type. + /// + /// In particular, this intentionally rejects cross-position remaps like + /// `record_time_window2 -> record_time_window`: they are easy to confuse with real schema + /// mismatches and should be fixed by giving the flow output the sink column name explicitly. + fn match_extra_output_columns( + &self, + mut exprs: Vec, + input_schema: &DFSchema, + original_exprs: &[Expr], + all_names: &BTreeSet, + ) -> DfResult> { + let mut output_names = exprs + .iter() + .map(|expr| expr.qualified_name().1) + .collect::>(); + let output_name_set = output_names.iter().cloned().collect::>(); + let extra_expr_indices = output_names + .iter() + .enumerate() + .filter_map(|(idx, name)| (!all_names.contains(name)).then_some(idx)) + .collect::>(); + let missing_sink_indices = self + .schema + .column_schemas() + .iter() + .enumerate() + .filter_map(|(idx, column)| (!output_name_set.contains(&column.name)).then_some(idx)) + .collect::>(); + + if extra_expr_indices.is_empty() && missing_sink_indices.is_empty() { + return Ok(exprs); + } + + if extra_expr_indices.len() != missing_sink_indices.len() { + return Err(DataFusionError::Plan(format_flow_sink_schema_mismatch( + original_exprs, + self.schema.as_ref(), + ))); + } + + let mut positional_matches = Vec::new(); + for expr_idx in extra_expr_indices { + if !missing_sink_indices.contains(&expr_idx) { + return Err(DataFusionError::Plan(format_flow_sink_schema_mismatch( + original_exprs, + self.schema.as_ref(), + ))); + } + + let target_col_schema = &self.schema.column_schemas()[expr_idx]; + let expr_type = + ConcreteDataType::from_arrow_type(&exprs[expr_idx].get_type(input_schema)?); + if is_obviously_incompatible_positional_match(&expr_type, &target_col_schema.data_type) + { + return Err(DataFusionError::Plan(format!( + "Cannot match flow output column '{}' to sink column '{}' by position: incompatible data types, flow output type is {:?}, sink column type is {:?}. {}", + output_names[expr_idx], + target_col_schema.name, + expr_type, + target_col_schema.data_type, + format_flow_sink_schema_mismatch(original_exprs, self.schema.as_ref()) + ))); + } + + let target_name = target_col_schema.name.clone(); + positional_matches.push(format!( + "{} -> {} (flow output type: {:?}, sink column type: {:?})", + output_names[expr_idx], target_name, expr_type, target_col_schema.data_type + )); + exprs[expr_idx] = exprs[expr_idx].clone().alias(target_name.clone()); + output_names[expr_idx] = target_name; + } + + if !positional_matches.is_empty() { + debug!( + "Matched flow output columns to sink columns by position: {:?}", + positional_matches + ); + } + + let duplicated_output_names = duplicate_names(&output_names); + if !duplicated_output_names.is_empty() { + return Err(DataFusionError::Plan(format!( + "Flow output schema contains duplicate column(s) after schema matching {:?}. {}", + duplicated_output_names, + format_flow_sink_schema_mismatch(&exprs, self.schema.as_ref()) + ))); + } + Ok(exprs) } @@ -1186,12 +1272,9 @@ impl ColumnMatcherRewriter { let query_col_cnt = exprs.len(); if query_col_cnt > table_col_cnt { - return Err(DataFusionError::Plan(format!( - "Expect query column count <= table column count, found {} query columns {:?}, {} table columns {:?}", - query_col_cnt, - exprs, - table_col_cnt, - self.schema.column_schemas() + return Err(DataFusionError::Plan(format_flow_sink_schema_mismatch( + &exprs, + self.schema.as_ref(), ))); } @@ -1209,8 +1292,9 @@ impl ColumnMatcherRewriter { .collect(); if !missing.is_empty() { return Err(DataFusionError::Plan(format!( - "Column(s) {:?} required by sink table are missing from flow output when merge_mode=last_non_null", - missing + "Column(s) {:?} required by sink table are missing from flow output when merge_mode=last_non_null. {}", + missing, + format_flow_sink_schema_mismatch(&exprs, self.schema.as_ref()) ))); } @@ -1250,8 +1334,9 @@ impl ColumnMatcherRewriter { if !remap.is_empty() { let extra: Vec<_> = remap.keys().cloned().collect(); return Err(DataFusionError::Plan(format!( - "Flow output has extra column(s) {:?} not found in sink schema when merge_mode=last_non_null", - extra + "Flow output has extra column(s) {:?} not found in sink schema when merge_mode=last_non_null. {}", + extra, + format_flow_sink_schema_mismatch(&exprs, self.schema.as_ref()) ))); } @@ -1281,6 +1366,80 @@ impl ColumnMatcherRewriter { } } +fn is_obviously_incompatible_positional_match( + expr_type: &ConcreteDataType, + sink_type: &ConcreteDataType, +) -> bool { + // This is a coarse type-family guard for legacy positional aliasing, not a strict type equality + // check. For example, numeric width/sign differences are allowed here and left to downstream + // coercion, and untyped NULL can be coerced to any target type. Clearly different families such + // as timestamp vs string are rejected early. + if expr_type.is_null() || expr_type == sink_type { + return false; + } + + expr_type.is_timestamp() != sink_type.is_timestamp() + || expr_type.is_string() != sink_type.is_string() + || expr_type.is_boolean() != sink_type.is_boolean() + || expr_type.is_json() != sink_type.is_json() + || expr_type.is_vector() != sink_type.is_vector() +} + +fn duplicate_names(names: &[String]) -> Vec { + let mut seen = HashSet::new(); + let mut duplicated = BTreeSet::new(); + for name in names { + if !seen.insert(name.as_str()) { + duplicated.insert(name.as_str()); + } + } + duplicated.into_iter().map(str::to_string).collect() +} + +fn format_flow_sink_schema_mismatch( + query_exprs: &[Expr], + sink_schema: &datatypes::schema::Schema, +) -> String { + let flow_output_columns = query_exprs + .iter() + .map(|expr| expr.qualified_name().1) + .collect::>(); + let sink_table_columns = sink_schema + .column_schemas() + .iter() + .map(|col| col.name.clone()) + .collect::>(); + + let flow_output_set = flow_output_columns.iter().cloned().collect::>(); + let sink_table_set = sink_table_columns.iter().cloned().collect::>(); + + let mut extra_flow_columns = flow_output_columns + .iter() + .filter(|name| !sink_table_set.contains(*name)) + .cloned() + .collect::>(); + extra_flow_columns.sort(); + extra_flow_columns.dedup(); + + let mut missing_sink_columns = sink_table_columns + .iter() + .filter(|name| !flow_output_set.contains(*name)) + .cloned() + .collect::>(); + missing_sink_columns.sort(); + missing_sink_columns.dedup(); + + format!( + "Flow output schema does not match sink table schema: found {} flow output columns and {} sink table columns. flow output columns: {:?}, sink table columns: {:?}, extra flow columns not in sink: {:?}, missing sink columns from flow output: {:?}", + flow_output_columns.len(), + sink_table_columns.len(), + flow_output_columns, + sink_table_columns, + extra_flow_columns, + missing_sink_columns + ) +} + impl TreeNodeRewriter for ColumnMatcherRewriter { type Node = LogicalPlan; fn f_down(&mut self, mut node: Self::Node) -> DfResult> { @@ -1327,7 +1486,7 @@ impl TreeNodeRewriter for ColumnMatcherRewriter { // if not, wrap it in a projection if let LogicalPlan::Projection(project) = &node { let exprs = project.expr.clone(); - let exprs = self.modify_project_exprs(exprs)?; + let exprs = self.modify_project_exprs(exprs, project.input.schema())?; self.is_rewritten = true; let new_plan = @@ -1341,7 +1500,7 @@ impl TreeNodeRewriter for ColumnMatcherRewriter { field.name(), ))); } - let exprs = self.modify_project_exprs(exprs)?; + let exprs = self.modify_project_exprs(exprs, node.schema())?; self.is_rewritten = true; let new_plan = LogicalPlan::Projection(Projection::try_new(exprs, Arc::new(node.clone()))?); diff --git a/src/flow/src/batching_mode/utils/test.rs b/src/flow/src/batching_mode/utils/test.rs index 317b0a5475..9ca1186fb6 100644 --- a/src/flow/src/batching_mode/utils/test.rs +++ b/src/flow/src/batching_mode/utils/test.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use catalog::RegisterTableRequest; use common_recordbatch::RecordBatch; use common_time::Timestamp; use datafusion_common::tree_node::TreeNode as _; @@ -29,7 +30,9 @@ use substrait::{DFLogicalSubstraitConvertor, SubstraitPlan}; use table::test_util::MemTable; use super::*; +use crate::batching_mode::BatchingModeOptions; use crate::batching_mode::state::FilterExprInfo; +use crate::batching_mode::task::{BatchingTask, TaskArgs}; use crate::test_utils::create_test_query_engine; fn u32_table(table_name: &str, columns: Vec<&str>, rows: usize) -> TableRef { @@ -432,9 +435,7 @@ async fn test_add_auto_column_rewriter() { // error datatype mismatch ( "SELECT number, ts FROM numbers_with_ts", - Err( - "Expect the last column in table to be timestamp column, found column atat with type Int8", - ), + Err("missing sink columns from flow output: [\"atat\"]"), vec![ ColumnSchema::new("number", ConcreteDataType::int32_datatype(), true), ColumnSchema::new( @@ -498,6 +499,383 @@ async fn test_add_auto_column_rewriter() { } } +#[tokio::test] +async fn test_gen_plan_with_matching_schema_reports_extra_flow_columns_before_positional_alias() { + let query_engine = create_test_query_engine(); + let ctx = QueryContext::arc(); + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true), + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ColumnSchema::new( + "max(numbers_with_ts.number)", + ConcreteDataType::uint32_datatype(), + true, + ), + ])); + + let err = gen_plan_with_matching_schema( + "SELECT number, number AS extra, ts, max(number) FROM numbers_with_ts GROUP BY number, ts", + ctx, + query_engine, + sink_schema, + &[], + false, + ) + .await + .unwrap_err() + .to_string(); + + assert!( + err.contains("Flow output schema does not match sink table schema"), + "{err}" + ); + assert!(err.contains("flow output columns"), "{err}"); + assert!(err.contains("sink table columns"), "{err}"); + assert!(err.contains("extra flow columns not in sink"), "{err}"); + assert!(err.contains("extra"), "{err}"); + assert!( + !err.contains("extra AS ts"), + "schema error should not primarily expose positional alias: {err}" + ); +} + +#[tokio::test] +async fn test_gen_plan_with_matching_schema_rejects_positional_alias_type_mismatch() { + let query_engine = create_test_query_engine(); + let ctx = QueryContext::arc(); + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true), + ColumnSchema::new( + "event_time", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ColumnSchema::new( + "max(numbers_with_ts.number)", + ConcreteDataType::uint32_datatype(), + true, + ), + ])); + + let err = gen_plan_with_matching_schema( + "SELECT number, number AS not_time, max(number) FROM numbers_with_ts GROUP BY number", + ctx, + query_engine, + sink_schema, + &[], + false, + ) + .await + .unwrap_err() + .to_string(); + + assert!( + err.contains( + "Cannot match flow output column 'not_time' to sink column 'event_time' by position" + ), + "{err}" + ); + assert!(err.contains("incompatible data types"), "{err}"); + assert!( + !err.contains("not_time AS event_time"), + "schema error should not expose an incompatible positional alias: {err}" + ); +} + +#[tokio::test] +async fn test_gen_plan_with_matching_schema_rejects_cross_position_extra_column_match() { + let query_engine = create_test_query_engine(); + let ctx = QueryContext::arc(); + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true), + ColumnSchema::new( + "time_window", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + true, + ), + ])); + + let err = gen_plan_with_matching_schema( + "SELECT number, ts, date_bin('5 minutes', ts) AS time_window2 FROM numbers_with_ts GROUP BY number, ts, time_window2", + ctx, + query_engine, + sink_schema, + &[], + false, + ) + .await + .unwrap_err() + .to_string(); + + assert!( + err.contains("Flow output schema does not match sink table schema"), + "{err}" + ); + assert!(err.contains("time_window2"), "{err}"); + assert!(err.contains("time_window"), "{err}"); + assert!(!err.contains("DuplicateUnqualifiedField"), "{err}"); +} + +#[tokio::test] +async fn test_gen_plan_with_matching_schema_accepts_out_of_order_matching_names() { + let query_engine = create_test_query_engine(); + let ctx = QueryContext::arc(); + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true), + ColumnSchema::new( + "time_window", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + true, + ), + ])); + + let plan = gen_plan_with_matching_schema( + "SELECT number, ts, date_bin('5 minutes', ts) AS time_window FROM numbers_with_ts GROUP BY number, ts, time_window", + ctx, + query_engine, + sink_schema, + &[], + false, + ) + .await + .unwrap(); + let output_names = plan + .schema() + .fields() + .iter() + .map(|field| field.name().clone()) + .collect::>(); + + assert_eq!( + output_names, + vec![ + "number".to_string(), + "ts".to_string(), + "time_window".to_string() + ] + ); + assert!(duplicate_names(&output_names).is_empty()); +} + +#[tokio::test] +async fn test_gen_plan_with_matching_schema_allows_numeric_positional_alias() { + let query_engine = create_test_query_engine(); + let ctx = QueryContext::arc(); + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("renamed_number", ConcreteDataType::int64_datatype(), true), + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ])); + + let plan = gen_plan_with_matching_schema( + "SELECT number, ts FROM numbers_with_ts", + ctx, + query_engine, + sink_schema, + &[], + false, + ) + .await + .unwrap(); + let sql = df_plan_to_sql(&plan).unwrap(); + + assert_eq!( + "SELECT numbers_with_ts.number AS renamed_number, numbers_with_ts.ts FROM numbers_with_ts", + sql + ); +} + +#[tokio::test] +async fn test_gen_plan_with_matching_schema_allows_null_positional_alias() { + let query_engine = create_test_query_engine(); + let ctx = QueryContext::arc(); + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true), + ColumnSchema::new("label", ConcreteDataType::string_datatype(), true), + ])); + + let plan = gen_plan_with_matching_schema( + "SELECT number, NULL AS label_placeholder FROM numbers_with_ts", + ctx, + query_engine, + sink_schema, + &[], + false, + ) + .await + .unwrap(); + let output_names = plan + .schema() + .fields() + .iter() + .map(|field| field.name().clone()) + .collect::>(); + let sql = df_plan_to_sql(&plan).unwrap(); + + assert_eq!( + output_names, + vec!["number".to_string(), "label".to_string()] + ); + assert!(sql.contains("NULL AS label"), "{sql}"); +} + +#[tokio::test] +async fn test_gen_plan_with_matching_schema_accepts_matching_flow_schema() { + let query_engine = create_test_query_engine(); + let ctx = QueryContext::arc(); + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true), + ColumnSchema::new("extra", ConcreteDataType::uint32_datatype(), true), + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ColumnSchema::new( + "max(numbers_with_ts.number)", + ConcreteDataType::uint32_datatype(), + true, + ), + ])); + + let plan = gen_plan_with_matching_schema( + "SELECT number, number AS extra, ts, max(number) FROM numbers_with_ts GROUP BY number, ts", + ctx, + query_engine, + sink_schema, + &[], + false, + ) + .await + .unwrap(); + let sql = df_plan_to_sql(&plan).unwrap(); + + assert_eq!( + "SELECT numbers_with_ts.number, numbers_with_ts.number AS extra, numbers_with_ts.ts, max(numbers_with_ts.number) FROM numbers_with_ts GROUP BY numbers_with_ts.number, numbers_with_ts.ts", + sql + ); +} + +#[tokio::test] +async fn test_validate_sink_table_schema_rejects_existing_sink_missing_flow_column() { + let query_engine = create_test_query_engine(); + let query_ctx = QueryContext::arc(); + let sql = "SELECT number, number AS extra, max(number) FROM numbers_with_ts GROUP BY number"; + let plan = sql_to_df_plan(query_ctx.clone(), query_engine.clone(), sql, true) + .await + .unwrap(); + + let catalog_manager = catalog::memory::new_memory_catalog_manager().unwrap(); + let sink_table_name = [ + "greptime".to_string(), + "public".to_string(), + "existing_sink".to_string(), + ]; + let sink_table = u32_table( + "existing_sink", + vec!["number", "max(numbers_with_ts.number)"], + 0, + ); + catalog_manager + .register_table_sync(RegisterTableRequest { + catalog: sink_table_name[0].clone(), + schema: sink_table_name[1].clone(), + table_name: sink_table_name[2].clone(), + table_id: 4096, + table: sink_table, + }) + .unwrap(); + + let (_shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let task = BatchingTask::try_new(TaskArgs { + flow_id: 1, + query: sql, + plan, + time_window_expr: None, + expire_after: None, + sink_table_name, + source_table_names: vec![[ + "greptime".to_string(), + "public".to_string(), + "numbers_with_ts".to_string(), + ]], + query_ctx, + catalog_manager, + shutdown_rx, + batch_opts: Arc::new(BatchingModeOptions::default()), + flow_eval_interval: None, + }) + .unwrap(); + + let err = task + .validate_sink_table_schema(&query_engine) + .await + .unwrap_err() + .to_string(); + + assert!( + err.contains("Flow output schema does not match sink table schema"), + "{err}" + ); + assert!(err.contains("extra"), "{err}"); +} + +#[tokio::test] +async fn test_gen_plan_with_matching_schema_allow_partial_fills_nullable_columns() { + let query_engine = create_test_query_engine(); + let ctx = QueryContext::arc(); + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), false), + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ColumnSchema::new("optional_value", ConcreteDataType::uint32_datatype(), true), + ])); + + let plan = gen_plan_with_matching_schema( + "SELECT number, ts FROM numbers_with_ts", + ctx, + query_engine, + sink_schema, + &[0], + true, + ) + .await + .unwrap(); + let sql = df_plan_to_sql(&plan).unwrap(); + + assert_eq!( + "SELECT numbers_with_ts.number, numbers_with_ts.ts, NULL AS optional_value FROM numbers_with_ts", + sql + ); +} + #[tokio::test] async fn test_find_group_by_exprs() { let testcases = vec![ @@ -1491,3 +1869,118 @@ async fn test_analyze_incremental_aggregate_plan_rejects_cast_wrapped_alias() { ); } } + +#[tokio::test] +async fn test_gen_plan_with_matching_schema_last_non_null_rejects_missing_primary_key_column() { + let query_engine = create_test_query_engine(); + let ctx = QueryContext::arc(); + // Sink table with primary_key_indices=[0] ("number"), time_index="ts", and merge_mode=last_non_null. + // The flow query omits "number", which is a required primary-key column. + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true), + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ColumnSchema::new("optional_value", ConcreteDataType::uint32_datatype(), true), + ])); + + let err = gen_plan_with_matching_schema( + "SELECT ts FROM numbers_with_ts", + ctx, + query_engine, + sink_schema, + &[0], + true, + ) + .await + .unwrap_err() + .to_string(); + + assert!( + err.contains( + "required by sink table are missing from flow output when merge_mode=last_non_null" + ), + "{err}" + ); + assert!(err.contains("number"), "{err}"); +} + +#[tokio::test] +async fn test_gen_plan_with_matching_schema_last_non_null_rejects_missing_time_index_column() { + let query_engine = create_test_query_engine(); + let ctx = QueryContext::arc(); + // Sink table with primary_key_indices=[0] ("number"), time_index="ts", and merge_mode=last_non_null. + // The flow query omits "ts", which is a required time-index column. + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true), + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ColumnSchema::new("optional_value", ConcreteDataType::uint32_datatype(), true), + ])); + + let err = gen_plan_with_matching_schema( + "SELECT number FROM numbers_with_ts", + ctx, + query_engine, + sink_schema, + &[0], + true, + ) + .await + .unwrap_err() + .to_string(); + + assert!( + err.contains( + "required by sink table are missing from flow output when merge_mode=last_non_null" + ), + "{err}" + ); + assert!(err.contains("ts"), "{err}"); +} + +#[tokio::test] +async fn test_gen_plan_with_matching_schema_last_non_null_rejects_extra_flow_column() { + let query_engine = create_test_query_engine(); + let ctx = QueryContext::arc(); + // Sink table with merge_mode=last_non_null. + // Sink has 3 columns: number (pk), ts (time_index), optional_value (nullable). + // Flow outputs: number, number AS extra, ts → "extra" is not in sink schema. + // query_col_cnt(3) <= table_col_cnt(3), so the extra branch is reached. + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", ConcreteDataType::uint32_datatype(), true), + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ColumnSchema::new("optional_value", ConcreteDataType::uint32_datatype(), true), + ])); + + let err = gen_plan_with_matching_schema( + "SELECT number, number AS extra, ts FROM numbers_with_ts", + ctx, + query_engine, + sink_schema, + &[0], + true, + ) + .await + .unwrap_err() + .to_string(); + + assert!(err.contains("extra column(s)"), "{err}"); + assert!(err.contains("extra"), "{err}"); + assert!( + err.contains("Flow output schema does not match sink table schema"), + "{err}" + ); +} diff --git a/tests/cases/standalone/common/flow/flow_last_non_null.result b/tests/cases/standalone/common/flow/flow_last_non_null.result index 50cb46faa3..0c03c19399 100644 --- a/tests/cases/standalone/common/flow/flow_last_non_null.result +++ b/tests/cases/standalone/common/flow/flow_last_non_null.result @@ -162,6 +162,8 @@ CREATE TABLE approx_rate ( Affected Rows: 0 +-- Without merge_mode=last_non_null, this partial output is rejected at CREATE FLOW time. +-- SQLNESS REPLACE (in\scontext:\sFailed\sto\srewrite\splan:\sError\sduring\splanning:.*) in context: Failed to rewrite plan CREATE FLOW find_approx_rate SINK TO approx_rate AS SELECT (max(byte) - min(byte)) / 30.0 as rate, @@ -172,24 +174,7 @@ from GROUP BY time_window; -Affected Rows: 0 - -INSERT INTO - bytes_log -VALUES - (NULL, '2023-01-01 00:00:01'), - (300, '2023-01-01 00:00:31'); - -Affected Rows: 2 - --- should return error -ADMIN FLUSH_FLOW('find_approx_rate'); - -Error: 1002(Unexpected), Failed to execute admin function flush_flow: Execution error: Internal error: 1003 - -DROP FLOW find_approx_rate; - -Affected Rows: 0 +Error: 3001(EngineExecuteQuery), Datafusion error: Plan("Flow output schema does not match sink table schema: found 3 flow output columns and 4 sink table columns. flow output columns: [\"rate\", \"time_window\", \"update_at\"], sink table columns: [\"rate\", \"time_window\", \"update_at\", \"bb\"], extra flow columns not in sink: [], missing sink columns from flow output: [\"bb\"]") in context: Failed to rewrite plan DROP TABLE bytes_log; diff --git a/tests/cases/standalone/common/flow/flow_last_non_null.sql b/tests/cases/standalone/common/flow/flow_last_non_null.sql index 95ebe4aaa6..29c5444f95 100644 --- a/tests/cases/standalone/common/flow/flow_last_non_null.sql +++ b/tests/cases/standalone/common/flow/flow_last_non_null.sql @@ -84,6 +84,8 @@ CREATE TABLE approx_rate ( TIME INDEX(time_window) ); +-- Without merge_mode=last_non_null, this partial output is rejected at CREATE FLOW time. +-- SQLNESS REPLACE (in\scontext:\sFailed\sto\srewrite\splan:\sError\sduring\splanning:.*) in context: Failed to rewrite plan CREATE FLOW find_approx_rate SINK TO approx_rate AS SELECT (max(byte) - min(byte)) / 30.0 as rate, @@ -93,16 +95,5 @@ from bytes_log GROUP BY time_window; - -INSERT INTO - bytes_log -VALUES - (NULL, '2023-01-01 00:00:01'), - (300, '2023-01-01 00:00:31'); - --- should return error -ADMIN FLUSH_FLOW('find_approx_rate'); - -DROP FLOW find_approx_rate; DROP TABLE bytes_log; DROP TABLE approx_rate; diff --git a/tests/cases/standalone/common/flow/flow_sink_schema_mismatch.result b/tests/cases/standalone/common/flow/flow_sink_schema_mismatch.result new file mode 100644 index 0000000000..54fcba2285 --- /dev/null +++ b/tests/cases/standalone/common/flow/flow_sink_schema_mismatch.result @@ -0,0 +1,123 @@ +-- Verify that batching flow rejects CREATE FLOW when the pre-existing sink +-- table schema does not match the flow output (create-time validation, not runtime). +CREATE TABLE source_mm ( + "number" INT, + extra STRING, + ts TIMESTAMP TIME INDEX +); + +Affected Rows: 0 + +-- Pre-create a sink table that is intentionally missing the "extra" column. +-- This case validates batching mode at CREATE FLOW time, before any INSERT/FLUSH. +CREATE TABLE sink_mm ( + "number" INT, + time_window TIMESTAMP TIME INDEX, + cnt BIGINT +); + +Affected Rows: 0 + +-- This CREATE FLOW should fail immediately: the flow outputs (number, extra, time_window, cnt) +-- but sink_mm has only (number, time_window, cnt). +-- SQLNESS REPLACE (in\scontext:\sFailed\sto\srewrite\splan:\sError\sduring\splanning:.*) in context: Failed to rewrite plan +CREATE FLOW mismatch_flow SINK TO sink_mm AS +SELECT + "number", + extra, + date_bin(INTERVAL '1 second', ts) as time_window, + count(*) as cnt +FROM + source_mm +GROUP BY + "number", extra, time_window; + +Error: 3001(EngineExecuteQuery), Datafusion error: Plan("Flow output schema does not match sink table schema: found 4 flow output columns and 3 sink table columns. flow output columns: [\"number\", \"extra\", \"time_window\", \"cnt\"], sink table columns: [\"number\", \"time_window\", \"cnt\"], extra flow columns not in sink: [\"extra\"], missing sink columns from flow output: []") in context: Failed to rewrite plan + +DROP TABLE source_mm; + +Affected Rows: 0 + +DROP TABLE sink_mm; + +Affected Rows: 0 + +-- TQL/PromQL flows use the same create-time sink schema validation path. +CREATE TABLE tql_source_mm ( + `value` DOUBLE, + ts TIMESTAMP TIME INDEX, + sensor STRING, + loc STRING, + PRIMARY KEY (sensor, loc) +); + +Affected Rows: 0 + +-- Pre-create a TQL sink table that is intentionally missing the "sensor" tag column. +CREATE TABLE tql_sink_mm ( + `value` DOUBLE, + ts TIMESTAMP TIME INDEX +); + +Affected Rows: 0 + +-- This CREATE FLOW should fail immediately: the TQL output has (value, sensor, ts), +-- but tql_sink_mm has only (value, ts). +-- SQLNESS REPLACE (in\scontext:\sFailed\sto\srewrite\splan:\sError\sduring\splanning:.*) in context: Failed to rewrite plan +CREATE FLOW tql_mismatch_flow +SINK TO tql_sink_mm +EVAL INTERVAL '1m' AS +TQL EVAL (now() - '1m'::interval, now(), '1m') +avg by(sensor) (tql_source_mm) AS value; + +Error: 3001(EngineExecuteQuery), Datafusion error: Plan("Flow output schema does not match sink table schema: found 3 flow output columns and 2 sink table columns. flow output columns: [\"value\", \"sensor\", \"ts\"], sink table columns: [\"value\", \"ts\"], extra flow columns not in sink: [\"sensor\"], missing sink columns from flow output: []") in context: Failed to rewrite plan + +DROP TABLE tql_source_mm; + +Affected Rows: 0 + +DROP TABLE tql_sink_mm; + +Affected Rows: 0 + +-- Real merge_mode=last_non_null sink options should enable partial schema validation. +CREATE TABLE lnn_source_mm ( + device STRING, + val DOUBLE, + ts TIMESTAMP TIME INDEX +); + +Affected Rows: 0 + +CREATE TABLE lnn_sink_mm ( + device STRING, + time_window TIMESTAMP TIME INDEX, + cnt BIGINT, + PRIMARY KEY (device) +) WITH('merge_mode'='last_non_null'); + +Affected Rows: 0 + +-- This CREATE FLOW should fail through the last_non_null partial validator: the +-- sink primary key "device" is required but absent from the flow output. +-- SQLNESS REPLACE (in\scontext:\sFailed\sto\srewrite\splan:\sError\sduring\splanning:.*) in context: Failed to rewrite plan +CREATE FLOW lnn_missing_pk_flow +SINK TO lnn_sink_mm AS +SELECT + date_bin(INTERVAL '1 second', ts) as time_window, + count(*) as cnt +FROM + lnn_source_mm +GROUP BY + time_window; + +Error: 3001(EngineExecuteQuery), Datafusion error: Plan("Column(s) [\"device\"] required by sink table are missing from flow output when merge_mode=last_non_null. Flow output schema does not match sink table schema: found 2 flow output columns and 3 sink table columns. flow output columns: [\"time_window\", \"cnt\"], sink table columns: [\"device\", \"time_window\", \"cnt\"], extra flow columns not in sink: [], missing sink columns from flow output: [\"device\"]") in context: Failed to rewrite plan + +DROP TABLE lnn_source_mm; + +Affected Rows: 0 + +DROP TABLE lnn_sink_mm; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/flow/flow_sink_schema_mismatch.sql b/tests/cases/standalone/common/flow/flow_sink_schema_mismatch.sql new file mode 100644 index 0000000000..2d00799817 --- /dev/null +++ b/tests/cases/standalone/common/flow/flow_sink_schema_mismatch.sql @@ -0,0 +1,89 @@ +-- Verify that batching flow rejects CREATE FLOW when the pre-existing sink +-- table schema does not match the flow output (create-time validation, not runtime). +CREATE TABLE source_mm ( + "number" INT, + extra STRING, + ts TIMESTAMP TIME INDEX +); + +-- Pre-create a sink table that is intentionally missing the "extra" column. +-- This case validates batching mode at CREATE FLOW time, before any INSERT/FLUSH. +CREATE TABLE sink_mm ( + "number" INT, + time_window TIMESTAMP TIME INDEX, + cnt BIGINT +); + +-- This CREATE FLOW should fail immediately: the flow outputs (number, extra, time_window, cnt) +-- but sink_mm has only (number, time_window, cnt). +-- SQLNESS REPLACE (in\scontext:\sFailed\sto\srewrite\splan:\sError\sduring\splanning:.*) in context: Failed to rewrite plan +CREATE FLOW mismatch_flow SINK TO sink_mm AS +SELECT + "number", + extra, + date_bin(INTERVAL '1 second', ts) as time_window, + count(*) as cnt +FROM + source_mm +GROUP BY + "number", extra, time_window; + +DROP TABLE source_mm; +DROP TABLE sink_mm; + +-- TQL/PromQL flows use the same create-time sink schema validation path. +CREATE TABLE tql_source_mm ( + `value` DOUBLE, + ts TIMESTAMP TIME INDEX, + sensor STRING, + loc STRING, + PRIMARY KEY (sensor, loc) +); + +-- Pre-create a TQL sink table that is intentionally missing the "sensor" tag column. +CREATE TABLE tql_sink_mm ( + `value` DOUBLE, + ts TIMESTAMP TIME INDEX +); + +-- This CREATE FLOW should fail immediately: the TQL output has (value, sensor, ts), +-- but tql_sink_mm has only (value, ts). +-- SQLNESS REPLACE (in\scontext:\sFailed\sto\srewrite\splan:\sError\sduring\splanning:.*) in context: Failed to rewrite plan +CREATE FLOW tql_mismatch_flow +SINK TO tql_sink_mm +EVAL INTERVAL '1m' AS +TQL EVAL (now() - '1m'::interval, now(), '1m') +avg by(sensor) (tql_source_mm) AS value; + +DROP TABLE tql_source_mm; +DROP TABLE tql_sink_mm; + +-- Real merge_mode=last_non_null sink options should enable partial schema validation. +CREATE TABLE lnn_source_mm ( + device STRING, + val DOUBLE, + ts TIMESTAMP TIME INDEX +); + +CREATE TABLE lnn_sink_mm ( + device STRING, + time_window TIMESTAMP TIME INDEX, + cnt BIGINT, + PRIMARY KEY (device) +) WITH('merge_mode'='last_non_null'); + +-- This CREATE FLOW should fail through the last_non_null partial validator: the +-- sink primary key "device" is required but absent from the flow output. +-- SQLNESS REPLACE (in\scontext:\sFailed\sto\srewrite\splan:\sError\sduring\splanning:.*) in context: Failed to rewrite plan +CREATE FLOW lnn_missing_pk_flow +SINK TO lnn_sink_mm AS +SELECT + date_bin(INTERVAL '1 second', ts) as time_window, + count(*) as cnt +FROM + lnn_source_mm +GROUP BY + time_window; + +DROP TABLE lnn_source_mm; +DROP TABLE lnn_sink_mm; diff --git a/tests/cases/standalone/flow-tql/flow_tql_missing_value_sink_schema.result b/tests/cases/standalone/flow-tql/flow_tql_missing_value_sink_schema.result new file mode 100644 index 0000000000..53df353078 --- /dev/null +++ b/tests/cases/standalone/flow-tql/flow_tql_missing_value_sink_schema.result @@ -0,0 +1,90 @@ +-- Regression for a TQL flow whose pre-created sink table is missing the value +-- output column. The labels are intentionally minimal and anonymous. +CREATE DATABASE source_schema; + +Affected Rows: 1 + +CREATE DATABASE sink_schema; + +Affected Rows: 1 + +USE source_schema; + +Affected Rows: 0 + +CREATE TABLE metric_input ( + namespace STRING NULL, + app STRING NULL, + greptime_timestamp TIMESTAMP(3) NOT NULL, + greptime_value DOUBLE NULL, + TIME INDEX (greptime_timestamp), + PRIMARY KEY (namespace, app) +); + +Affected Rows: 0 + +INSERT INTO metric_input VALUES + ('ns', 'app-a', '2026-01-23T03:40:00Z', 10.0), + ('ns', 'app-a', '2026-01-23T03:50:00Z', 20.0); + +Affected Rows: 2 + +USE sink_schema; + +Affected Rows: 0 + +-- Intentionally omit greptime_value DOUBLE from the pre-created sink table. +CREATE TABLE missing_value_sink ( + namespace STRING NULL, + app STRING NULL, + greptime_timestamp TIMESTAMP(3) NOT NULL, + TIME INDEX (greptime_timestamp), + PRIMARY KEY (namespace, app) +) +ENGINE=mito; + +Affected Rows: 0 + +-- SQLNESS REPLACE (in\scontext:\sFailed\sto\srewrite\splan:\sError\sduring\splanning:.*) in context: Failed to rewrite plan +CREATE FLOW missing_value_flow +SINK TO sink_schema.missing_value_sink +EVAL INTERVAL '3600 s' +AS TQL EVAL ( + date_bin('2m'::interval, now() - '2m'::interval), + date_bin('2m'::interval, now() - '2m'::interval), + '1h' +) + avg by (namespace, app) ( + avg_over_time(metric_input{__schema__="source_schema"}[1h]) + ); + +Error: 3001(EngineExecuteQuery), Datafusion error: Plan("Flow output schema does not match sink table schema: found 4 flow output columns and 3 sink table columns. flow output columns: [\"namespace\", \"app\", \"greptime_timestamp\", \"avg(prom_avg_over_time(greptime_timestamp_range,greptime_value))\"], sink table columns: [\"namespace\", \"app\", \"greptime_timestamp\"], extra flow columns not in sink: [\"avg(prom_avg_over_time(greptime_timestamp_range,greptime_value))\"], missing sink columns from flow output: []") in context: Failed to rewrite plan + +DROP FLOW IF EXISTS missing_value_flow; + +Affected Rows: 0 + +DROP TABLE missing_value_sink; + +Affected Rows: 0 + +USE source_schema; + +Affected Rows: 0 + +DROP TABLE metric_input; + +Affected Rows: 0 + +USE public; + +Affected Rows: 0 + +DROP DATABASE sink_schema; + +Affected Rows: 0 + +DROP DATABASE source_schema; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/flow-tql/flow_tql_missing_value_sink_schema.sql b/tests/cases/standalone/flow-tql/flow_tql_missing_value_sink_schema.sql new file mode 100644 index 0000000000..3693775800 --- /dev/null +++ b/tests/cases/standalone/flow-tql/flow_tql_missing_value_sink_schema.sql @@ -0,0 +1,55 @@ +-- Regression for a TQL flow whose pre-created sink table is missing the value +-- output column. The labels are intentionally minimal and anonymous. + +CREATE DATABASE source_schema; +CREATE DATABASE sink_schema; + +USE source_schema; + +CREATE TABLE metric_input ( + namespace STRING NULL, + app STRING NULL, + greptime_timestamp TIMESTAMP(3) NOT NULL, + greptime_value DOUBLE NULL, + TIME INDEX (greptime_timestamp), + PRIMARY KEY (namespace, app) +); + +INSERT INTO metric_input VALUES + ('ns', 'app-a', '2026-01-23T03:40:00Z', 10.0), + ('ns', 'app-a', '2026-01-23T03:50:00Z', 20.0); + +USE sink_schema; + +-- Intentionally omit greptime_value DOUBLE from the pre-created sink table. +CREATE TABLE missing_value_sink ( + namespace STRING NULL, + app STRING NULL, + greptime_timestamp TIMESTAMP(3) NOT NULL, + TIME INDEX (greptime_timestamp), + PRIMARY KEY (namespace, app) +) +ENGINE=mito; + +-- SQLNESS REPLACE (in\scontext:\sFailed\sto\srewrite\splan:\sError\sduring\splanning:.*) in context: Failed to rewrite plan +CREATE FLOW missing_value_flow +SINK TO sink_schema.missing_value_sink +EVAL INTERVAL '3600 s' +AS TQL EVAL ( + date_bin('2m'::interval, now() - '2m'::interval), + date_bin('2m'::interval, now() - '2m'::interval), + '1h' +) + avg by (namespace, app) ( + avg_over_time(metric_input{__schema__="source_schema"}[1h]) + ); + +DROP FLOW IF EXISTS missing_value_flow; +DROP TABLE missing_value_sink; + +USE source_schema; +DROP TABLE metric_input; + +USE public; +DROP DATABASE sink_schema; +DROP DATABASE source_schema; From f3919a3117798e0cdf95d316b3af7c45d639b395 Mon Sep 17 00:00:00 2001 From: shuiyisong <113876041+shuiyisong@users.noreply.github.com> Date: Tue, 2 Jun 2026 18:04:15 +0800 Subject: [PATCH 18/29] chore: update vrl to 0.33 (#8219) * chore: update vrl to 0.33 Signed-off-by: shuiyisong * chore: reuse thread local runtime Signed-off-by: shuiyisong --------- Signed-off-by: shuiyisong --- Cargo.lock | 683 +++++++++++++++--- Cargo.toml | 2 +- src/meta-client/src/client.rs | 2 +- src/pipeline/benches/processor.rs | 65 ++ .../src/etl/processor/vrl_processor.rs | 20 +- 5 files changed, 664 insertions(+), 108 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a65159d26a..2485a5ceec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -79,8 +79,9 @@ checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", "const-random", - "getrandom 0.3.3", + "getrandom 0.3.4", "once_cell", + "serde", "version_check", "zerocopy", ] @@ -771,7 +772,7 @@ version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef1e3e699d84ab1b0911a1010c5c106aa34ae89aeac103be5ce0c3859db1e891" dependencies = [ - "term", + "term 1.0.2", ] [[package]] @@ -1427,6 +1428,12 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + [[package]] name = "borsh" version = "1.5.7" @@ -1525,6 +1532,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + [[package]] name = "bytemuck" version = "1.23.1" @@ -1635,7 +1648,7 @@ dependencies = [ "paste", "prometheus 0.14.0", "promql-parser", - "rand 0.9.1", + "rand 0.9.4", "serde", "serde_json", "session", @@ -1973,7 +1986,7 @@ dependencies = [ "partition", "paste", "query", - "rand 0.9.1", + "rand 0.9.4", "reqwest 0.13.2", "serde", "serde_json", @@ -2020,7 +2033,7 @@ dependencies = [ "prometheus 0.14.0", "prost 0.14.1", "query", - "rand 0.9.1", + "rand 0.9.4", "serde_json", "snafu 0.8.6", "store-api", @@ -2031,6 +2044,15 @@ dependencies = [ "tracing", ] +[[package]] +name = "clipboard-win" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4" +dependencies = [ + "error-code", +] + [[package]] name = "clocksource" version = "0.8.1" @@ -2123,7 +2145,7 @@ dependencies = [ "prometheus 0.14.0", "prost 0.14.1", "query", - "rand 0.9.1", + "rand 0.9.4", "regex", "reqwest 0.13.2", "serde", @@ -2462,7 +2484,7 @@ dependencies = [ "hyper-util", "lazy_static", "prost 0.14.1", - "rand 0.9.1", + "rand 0.9.4", "serde", "serde_json", "snafu 0.8.6", @@ -2581,7 +2603,7 @@ dependencies = [ "prometheus 0.14.0", "prost 0.14.1", "prost-types 0.14.1", - "rand 0.9.1", + "rand 0.9.4", "regex", "rskafka", "rustls", @@ -2649,7 +2671,7 @@ dependencies = [ "futures-util", "humantime-serde", "object-store", - "rand 0.9.1", + "rand 0.9.4", "serde", "serde_json", "smallvec", @@ -2833,7 +2855,7 @@ dependencies = [ "common-query", "common-recordbatch", "once_cell", - "rand 0.9.1", + "rand 0.9.4", "tempfile", ] @@ -2849,7 +2871,7 @@ dependencies = [ "humantime", "humantime-serde", "once_cell", - "rand 0.9.1", + "rand 0.9.4", "serde", "serde_json", "snafu 0.8.6", @@ -3594,6 +3616,12 @@ dependencies = [ "parking_lot_core 0.9.11", ] +[[package]] +name = "data-encoding" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" + [[package]] name = "datafusion" version = "53.1.0" @@ -3638,7 +3666,7 @@ dependencies = [ "object_store", "parking_lot 0.12.4", "parquet", - "rand 0.9.1", + "rand 0.9.4", "regex", "sqlparser", "tempfile", @@ -3755,7 +3783,7 @@ dependencies = [ "liblzma", "log", "object_store", - "rand 0.9.1", + "rand 0.9.4", "tokio", "tokio-util", "url", @@ -3881,7 +3909,7 @@ dependencies = [ "log", "object_store", "parking_lot 0.12.4", - "rand 0.9.1", + "rand 0.9.4", "tempfile", "url", ] @@ -3944,7 +3972,7 @@ dependencies = [ "md-5 0.10.6", "memchr", "num-traits", - "rand 0.9.1", + "rand 0.9.4", "regex", "sha2 0.10.9", "unicode-segmentation", @@ -4234,7 +4262,7 @@ dependencies = [ "datafusion-proto-common", "object_store", "prost 0.14.1", - "rand 0.9.1", + "rand 0.9.4", ] [[package]] @@ -4685,6 +4713,27 @@ dependencies = [ "crypto-common 0.2.1", ] +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if", + "dirs-sys-next", +] + +[[package]] +name = "dirs-sys-next" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -4713,14 +4762,14 @@ dependencies = [ [[package]] name = "dns-lookup" -version = "2.0.4" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5766087c2235fec47fafa4cfecc81e494ee679d0fd4a59887ea0919bfb0e4fc" +checksum = "6e39034cee21a2f5bbb66ba0e3689819c4bb5d00382a282006e802a7ffa6c41d" dependencies = [ "cfg-if", "libc", - "socket2 0.5.10", - "windows-sys 0.48.0", + "socket2 0.6.0", + "windows-sys 0.60.2", ] [[package]] @@ -4740,31 +4789,30 @@ dependencies = [ [[package]] name = "domain" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a11dd7f04a6a6d2aea0153c6e31f5ea7af8b2efdf52cdaeea7a9a592c7fefef9" +checksum = "8c469892dddfeff64ecfdbc64cf059c77fb0decaeccd4d5d484394bdd6312bac" dependencies = [ "bumpalo", "bytes", "domain-macros", "futures-util", - "hashbrown 0.14.5", + "hashbrown 0.17.1", + "jiff", "log", - "moka", "octseq", - "rand 0.8.5", + "rand 0.10.1", "serde", "smallvec", - "time", "tokio", "tracing", ] [[package]] name = "domain-macros" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e197fdfd2cdb5fdeb7f8ddcf3aed5d5d04ecde2890d448b14ffb716f7376b70" +checksum = "6fef7ef74e413e36d5364db163ca577ccb56f2f74377705d5f920ee3e1544127" dependencies = [ "proc-macro2", "quote", @@ -4849,6 +4897,15 @@ dependencies = [ "serde", ] +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + [[package]] name = "ena" version = "0.14.3" @@ -4962,6 +5019,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "error-code" +version = "3.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea2df4cf52843e0452895c455a1a2cfbb842a1e7329671acf418fdc53ed4c59" + [[package]] name = "etcd-client" version = "0.17.0" @@ -5018,6 +5081,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "exitcode" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de853764b47027c2e862a995c34978ffa63c1501f2e15f987ba11bd4f9bba193" + [[package]] name = "fail" version = "0.5.1" @@ -5043,9 +5112,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fancy-regex" -version = "0.14.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e24cb5a94bcae1e5408b0effca5cd7172ea3c5755049c5f3af4cd283a165298" +checksum = "72cf461f865c862bb7dc573f643dd6a2b6842f7c30b07882b56bd148cc2761b8" dependencies = [ "bit-set", "regex-automata", @@ -5274,7 +5343,7 @@ dependencies = [ "prometheus 0.14.0", "prost 0.14.1", "query", - "rand 0.9.1", + "rand 0.9.4", "serde", "serde_json", "servers", @@ -5299,6 +5368,17 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + [[package]] name = "flume" version = "0.11.1" @@ -5337,6 +5417,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fraction" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872" +dependencies = [ + "lazy_static", + "num", +] + [[package]] name = "fragile" version = "2.0.1" @@ -5405,7 +5495,7 @@ dependencies = [ "promql-parser", "prost 0.14.1", "query", - "rand 0.9.1", + "rand 0.9.4", "reqwest 0.13.2", "serde", "serde_json", @@ -5756,21 +5846,21 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "js-sys", "libc", "r-efi", - "wasi 0.14.2+wasi-0.2.4", + "wasip2", "wasm-bindgen", ] @@ -5842,9 +5932,9 @@ dependencies = [ [[package]] name = "grok" -version = "2.1.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c52724b609896f661a3f4641dd3a44dc602958ef615857c12d00756b4e9355b" +checksum = "6ddab6a9c8bb998cb2fc3101fde8ef561b7c4970db3957be7a8eee1e168f666b" dependencies = [ "glob", "onig", @@ -5984,6 +6074,9 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", +] [[package]] name = "hashlink" @@ -6650,7 +6743,7 @@ dependencies = [ "pin-project", "prost 0.14.1", "puffin", - "rand 0.9.1", + "rand 0.9.4", "rand_chacha 0.9.0", "regex", "regex-automata", @@ -6841,6 +6934,15 @@ dependencies = [ "derive_utils", ] +[[package]] +name = "ipcrypt-rs" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96e4f67dbfc0f75d7b65953ecf0be3fd84ee0cb1ae72a00a4aa9a2f5518a2c80" +dependencies = [ + "aes", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -7016,6 +7118,36 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.17", + "walkdir", + "windows-link 0.2.1", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + [[package]] name = "jni-sys" version = "0.3.1" @@ -7050,7 +7182,7 @@ version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "libc", ] @@ -7139,11 +7271,38 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c6e529149475ca0b2820835d3dce8fcc41c6b943ca608d32f35b449255e4627" dependencies = [ - "fluent-uri", + "fluent-uri 0.1.4", "serde", "serde_json", ] +[[package]] +name = "jsonschema" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89f50532ce4a0ba3ae930212908d8ec50e7806065c059fe9c75da2ece6132294" +dependencies = [ + "ahash 0.8.12", + "bytecount", + "data-encoding", + "email_address", + "fancy-regex", + "fraction", + "getrandom 0.3.4", + "idna", + "itoa", + "num-cmp", + "num-traits", + "percent-encoding", + "referencing", + "regex", + "regex-syntax", + "serde", + "serde_json", + "unicode-general-category", + "uuid-simd", +] + [[package]] name = "jsonwebtoken" version = "10.3.0" @@ -7337,7 +7496,7 @@ dependencies = [ "regex-syntax", "sha3", "string_cache", - "term", + "term 1.0.2", "unicode-xid", "walkdir", ] @@ -7736,7 +7895,7 @@ dependencies = [ "protobuf 2.28.0", "protobuf-build", "raft-engine", - "rand 0.9.1", + "rand 0.9.4", "rskafka", "serde", "serde_json", @@ -8043,7 +8202,7 @@ dependencies = [ "futures-util", "humantime-serde", "meta-srv", - "rand 0.9.1", + "rand 0.9.4", "serde", "serde_json", "session", @@ -8111,7 +8270,7 @@ dependencies = [ "partition", "prometheus 0.14.0", "prost 0.14.1", - "rand 0.9.1", + "rand 0.9.4", "regex", "rskafka", "serde", @@ -8253,7 +8412,7 @@ checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" dependencies = [ "libc", "log", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", "windows-sys 0.59.0", ] @@ -8343,7 +8502,7 @@ dependencies = [ "prometheus 0.14.0", "prost 0.14.1", "puffin", - "rand 0.9.1", + "rand 0.9.4", "rayon", "regex", "roaring", @@ -8656,6 +8815,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "neli" version = "0.6.5" @@ -8758,6 +8923,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "nom-language" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2de2bc5b451bfedaef92c90b8939a8fff5770bdcc1fafd6239d086aab8fa6b29" +dependencies = [ + "nom 8.0.0", +] + [[package]] name = "notify" version = "8.0.0" @@ -8841,6 +9015,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-cmp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + [[package]] name = "num-complex" version = "0.4.6" @@ -9001,6 +9181,31 @@ dependencies = [ "libc", ] +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.11.1", + "objc2", +] + [[package]] name = "object" version = "0.36.7" @@ -9040,7 +9245,7 @@ dependencies = [ "object_store_opendal", "opendal", "prometheus 0.14.0", - "rand 0.9.1", + "rand 0.9.4", "reqwest 0.13.2", "serde", "snafu 0.8.6", @@ -9094,9 +9299,9 @@ dependencies = [ [[package]] name = "octseq" -version = "0.5.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "126c3ca37c9c44cec575247f43a3e4374d8927684f129d2beeb0d2cef262fe12" +checksum = "182eab3e1cd9cdc0ecf1ce3342d9844f3dc7d098f0694569bfdf327b612d69fd" dependencies = [ "bytes", "serde", @@ -9552,7 +9757,7 @@ dependencies = [ "futures-util", "opentelemetry 0.30.0", "percent-encoding", - "rand 0.9.1", + "rand 0.9.4", "serde_json", "thiserror 2.0.17", "tokio", @@ -9570,7 +9775,7 @@ dependencies = [ "futures-util", "opentelemetry 0.31.0", "percent-encoding", - "rand 0.9.1", + "rand 0.9.4", "thiserror 2.0.17", ] @@ -9748,7 +9953,7 @@ dependencies = [ "paste", "prost 0.14.1", "prost-build 0.14.1", - "rand 0.9.1", + "rand 0.9.4", "replace_with", "serde", "smallvec", @@ -10154,7 +10359,7 @@ dependencies = [ "md5", "pg_interval_2", "postgres-types", - "rand 0.10.0", + "rand 0.10.1", "rust_decimal", "rustls-pki-types", "ryu", @@ -10531,7 +10736,7 @@ dependencies = [ "hmac", "md-5 0.10.6", "memchr", - "rand 0.9.1", + "rand 0.9.4", "sha2 0.10.9", "stringprep", ] @@ -10680,6 +10885,19 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "prettytable-rs" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eea25e07510aa6ab6547308ebe3c036016d162b8da920dbb079e3ba8acf3d95a" +dependencies = [ + "encode_unicode", + "is-terminal", + "lazy_static", + "term 0.7.0", + "unicode-width 0.1.14", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -10845,7 +11063,7 @@ checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40" dependencies = [ "bitflags 2.11.1", "num-traits", - "rand 0.9.1", + "rand 0.9.4", "rand_chacha 0.9.0", "rand_xorshift", "regex-syntax", @@ -11272,7 +11490,7 @@ dependencies = [ "promql", "promql-parser", "prost 0.14.1", - "rand 0.9.1", + "rand 0.9.4", "regex", "serde", "serde_json", @@ -11345,9 +11563,9 @@ checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098" dependencies = [ "aws-lc-rs", "bytes", - "getrandom 0.3.3", + "getrandom 0.3.4", "lru-slab", - "rand 0.9.1", + "rand 0.9.4", "ring", "rustc-hash 2.1.1", "rustls", @@ -11453,9 +11671,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.1" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", @@ -11463,9 +11681,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc266eb313df6c5c09c1c7b1fbe2510961e5bcd3add930c1e31f7ed9da0feff8" +checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20 0.10.0", "getrandom 0.4.1", @@ -11508,7 +11726,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", ] [[package]] @@ -11601,6 +11819,17 @@ dependencies = [ "bitflags 2.11.1", ] +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.16", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "ref-cast" version = "1.0.24" @@ -11621,6 +11850,21 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "referencing" +version = "0.38.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15a8af0c6bb8eaf8b07cb06fc31ff30ca6fe19fb99afa476c276d8b24f365b0b" +dependencies = [ + "ahash 0.8.12", + "fluent-uri 0.4.1", + "getrandom 0.3.4", + "hashbrown 0.16.1", + "parking_lot 0.12.4", + "percent-encoding", + "serde_json", +] + [[package]] name = "regex" version = "1.12.2" @@ -11686,6 +11930,15 @@ version = "1.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" +[[package]] +name = "relative-path" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bca40a312222d8ba74837cb474edef44b37f561da5f773981007a10bbaa992b0" +dependencies = [ + "serde", +] + [[package]] name = "rend" version = "0.4.2" @@ -11825,6 +12078,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", + "h2 0.4.11", "http 1.3.1", "http-body 1.0.1", "http-body-util", @@ -11837,6 +12091,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls", + "rustls-native-certs 0.8.1", "rustls-pki-types", "serde", "serde_json", @@ -11898,6 +12153,50 @@ dependencies = [ "web-sys", ] +[[package]] +name = "reqwest-middleware" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57f17d28a6e6acfe1733fe24bcd30774d13bffa4b8a22535b4c8c98423088d4e" +dependencies = [ + "anyhow", + "async-trait", + "http 1.3.1", + "reqwest 0.12.28", + "serde", + "thiserror 1.0.69", + "tower-service", +] + +[[package]] +name = "reqwest-retry" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "105747e3a037fe5bf17458d794de91149e575b6183fc72c85623a44abb9683f5" +dependencies = [ + "anyhow", + "async-trait", + "futures", + "getrandom 0.2.16", + "http 1.3.1", + "hyper 1.6.0", + "reqwest 0.12.28", + "reqwest-middleware", + "retry-policies", + "thiserror 2.0.17", + "tokio", + "wasmtimer", +] + +[[package]] +name = "retry-policies" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc05fbf560421a0357a750cbe78c7ca19d4923918490daabba313d5dbc871e47" +dependencies = [ + "rand 0.10.1", +] + [[package]] name = "rgb" version = "0.8.50" @@ -11985,9 +12284,12 @@ dependencies = [ [[package]] name = "roxmltree" -version = "0.20.0" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c20b6793b5c2fa6553b250154b78d6d0db37e72700ae35fad9387a46f487c97" +checksum = "f1964b10c76125c36f8afe190065a4bf9a87bf324842c05701330bba9f1cacbb" +dependencies = [ + "memchr", +] [[package]] name = "rsa" @@ -12042,7 +12344,7 @@ dependencies = [ "integer-encoding 4.0.2", "lz4", "parking_lot 0.12.4", - "rand 0.9.1", + "rand 0.9.4", "rsasl", "rustls", "snap", @@ -12088,7 +12390,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "relative-path", + "relative-path 1.9.3", "rustc_version", "syn 2.0.117", "unicode-ident", @@ -12309,7 +12611,7 @@ checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", - "jni", + "jni 0.21.1", "log", "once_cell", "rustls", @@ -12346,6 +12648,25 @@ version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" +[[package]] +name = "rustyline" +version = "17.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" +dependencies = [ + "bitflags 2.11.1", + "cfg-if", + "clipboard-win", + "libc", + "log", + "memchr", + "nix 0.30.1", + "unicode-segmentation", + "unicode-width 0.2.1", + "utf8parse", + "windows-sys 0.60.2", +] + [[package]] name = "ryu" version = "1.0.20" @@ -12724,6 +13045,19 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "serde_yaml_ng" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b4db627b98b36d4203a7b458cf3573730f2bb591b28871d916dfa9efabfd41f" +dependencies = [ + "indexmap 2.13.0", + "itoa", + "ryu", + "serde", + "unsafe-libyaml", +] + [[package]] name = "servers" version = "1.1.0" @@ -12820,7 +13154,7 @@ dependencies = [ "prost 0.14.1", "query", "quoted-string", - "rand 0.9.1", + "rand 0.9.4", "regex", "reqwest 0.13.2", "rust-embed", @@ -12998,7 +13332,7 @@ version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c962f626b54771990066e5435ec8331d1462576cd2d1e62f24076ae014f92112" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "halfbrown", "ref-cast", "serde", @@ -13007,6 +13341,16 @@ dependencies = [ "value-trait", ] +[[package]] +name = "simd_cesu8" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -13898,12 +14242,12 @@ dependencies = [ [[package]] name = "syslog_loose" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "161028c00842709450114c39db3b29f44c898055ed8833bb9b535aba7facf30e" +checksum = "d6ec4df26907adce53e94eac201a9ba38744baea3bc97f34ffd591d5646231a6" dependencies = [ "chrono", - "nom 7.1.3", + "nom 8.0.0", ] [[package]] @@ -14147,12 +14491,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ "fastrand", - "getrandom 0.3.3", + "getrandom 0.3.4", "once_cell", "rustix 1.0.7", "windows-sys 0.61.2", ] +[[package]] +name = "term" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f" +dependencies = [ + "dirs-next", + "rustversion", + "winapi", +] + [[package]] name = "term" version = "1.0.2" @@ -14205,7 +14560,7 @@ dependencies = [ "nix 0.28.0", "partition", "paste", - "rand 0.9.1", + "rand 0.9.4", "rand_chacha 0.9.0", "reqwest 0.13.2", "rustls", @@ -14292,7 +14647,7 @@ dependencies = [ "plugins", "prost 0.14.1", "query", - "rand 0.9.1", + "rand 0.9.4", "rstest", "rstest_reuse", "sea-query", @@ -14582,7 +14937,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "rand 0.9.1", + "rand 0.9.4", "socket2 0.5.10", "tokio", "tokio-util", @@ -15233,6 +15588,12 @@ version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c1cb5db39152898a79168971543b1cb5020dff7fe43c8dc468b0885f5e29df5" +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + [[package]] name = "unicode-ident" version = "1.0.22" @@ -15366,11 +15727,21 @@ checksum = "b672338555252d43fd2240c714dc444b8c6fb0a5c5335e65a07bba7742735ddb" dependencies = [ "getrandom 0.4.1", "js-sys", - "rand 0.9.1", + "rand 0.9.4", "serde_core", "wasm-bindgen", ] +[[package]] +name = "uuid-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "valuable" version = "0.1.1" @@ -15453,9 +15824,9 @@ dependencies = [ [[package]] name = "vrl" -version = "0.25.0" +version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f49394b948406ea1564aa00152e011d87a38ad35d277ebddda257a9ee39c419" +checksum = "925a4d3321b18a200c82c3ec02ee2be2b4bf16db07a5ce7e2a9a888b795ea862" dependencies = [ "aes", "aes-siv", @@ -15485,8 +15856,10 @@ dependencies = [ "domain", "dyn-clone", "encoding_rs", + "exitcode", "fancy-regex", "flate2", + "getrandom 0.3.4", "grok", "hex", "hmac", @@ -15496,12 +15869,15 @@ dependencies = [ "indexmap 2.13.0", "indoc", "influxdb-line-protocol", + "ipcrypt-rs", "itertools 0.14.0", + "jsonschema", "lalrpop", "lalrpop-util", "lz4_flex 0.11.6", "md-5 0.10.6", - "nom 7.1.3", + "nom 8.0.0", + "nom-language", "ofb", "onig", "ordered-float 4.6.0", @@ -15510,20 +15886,27 @@ dependencies = [ "percent-encoding", "pest", "pest_derive", + "prettytable-rs", "prost 0.13.5", "prost-reflect", "psl", "psl-types", "publicsuffix", "quoted_printable", - "rand 0.8.5", + "rand 0.9.4", "regex", + "relative-path 2.0.1", + "reqwest 0.12.28", + "reqwest-middleware", + "reqwest-retry", "roxmltree", "rust_decimal", + "rustyline", "seahash", "serde", "serde_json", "serde_yaml", + "serde_yaml_ng", "sha-1", "sha2 0.10.9", "sha3", @@ -15531,6 +15914,8 @@ dependencies = [ "snafu 0.8.6", "snap", "strip-ansi-escapes", + "strum 0.26.3", + "strum_macros 0.26.4", "syslog_loose", "termcolor", "thiserror 2.0.17", @@ -15541,7 +15926,9 @@ dependencies = [ "url", "utf8-width", "uuid", + "webbrowser", "woothee", + "xxhash-rust", "zstd", ] @@ -15585,15 +15972,6 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" -[[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" -dependencies = [ - "wit-bindgen-rt", -] - [[package]] name = "wasip2" version = "1.0.2+wasi-0.2.9" @@ -15734,6 +16112,20 @@ dependencies = [ "semver", ] +[[package]] +name = "wasmtimer" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c598d6b99ea013e35844697fc4670d08339d5cda15588f193c6beedd12f644b" +dependencies = [ + "futures", + "js-sys", + "parking_lot 0.12.4", + "pin-utils", + "slab", + "wasm-bindgen", +] + [[package]] name = "web-sys" version = "0.3.95" @@ -15754,6 +16146,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webbrowser" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fc95580916af1e68ff6a7be07446fc5db73ebf71cf092de939bbf5f7e189f72" +dependencies = [ + "core-foundation 0.10.1", + "jni 0.22.4", + "log", + "ndk-context", + "objc2", + "objc2-foundation", + "url", + "web-sys", +] + [[package]] name = "webpki" version = "0.22.4" @@ -16040,6 +16448,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -16088,13 +16505,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link 0.2.1", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + [[package]] name = "windows-threading" version = "0.1.0" @@ -16122,6 +16556,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -16140,6 +16580,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -16158,12 +16604,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -16182,6 +16640,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -16200,6 +16664,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -16218,6 +16688,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -16236,6 +16712,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "winnow" version = "0.5.40" @@ -16283,15 +16765,6 @@ dependencies = [ "wit-parser", ] -[[package]] -name = "wit-bindgen-rt" -version = "0.39.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags 2.11.1", -] - [[package]] name = "wit-bindgen-rust" version = "0.51.0" @@ -16445,6 +16918,12 @@ dependencies = [ "rustix 1.0.7", ] +[[package]] +name = "xxhash-rust" +version = "0.8.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/Cargo.toml b/Cargo.toml index 56200a24d6..ee8d4dcf11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -259,7 +259,7 @@ tracing-opentelemetry = "0.31.0" tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "fmt"] } typetag = "0.2" uuid = { version = "1.17", features = ["serde", "v4", "v7", "fast-rng"] } -vrl = "0.25" +vrl = "0.33" zstd = "0.13" # DO_NOT_REMOVE_THIS: END_OF_EXTERNAL_DEPENDENCIES diff --git a/src/meta-client/src/client.rs b/src/meta-client/src/client.rs index cbd9b43151..de41caf19e 100644 --- a/src/meta-client/src/client.rs +++ b/src/meta-client/src/client.rs @@ -1344,7 +1344,7 @@ mod tests { // Generates rough 10MB data, which is larger than the default grpc message size limit. for i in 0..10 { - let data: Vec = (0..1024 * 1024).map(|_| rng.random()).collect(); + let data: Vec = (0..1024 * 1024).map(|_| rng.random::()).collect(); in_memory .put( PutRequest::new() diff --git a/src/pipeline/benches/processor.rs b/src/pipeline/benches/processor.rs index 83a8e53225..e088c89c6b 100644 --- a/src/pipeline/benches/processor.rs +++ b/src/pipeline/benches/processor.rs @@ -233,6 +233,36 @@ transform: parse(&Content::Yaml(pipeline_yaml)).unwrap() } +fn prepare_vrl_pipeline() -> Pipeline { + let pipeline_yaml = r#" +--- +description: Minimal VRL processor benchmark + +processors: + - vrl: + source: | + .service_alias = .service + .host_alias = .host + del(.unused) + .processed = true + . + +transform: + - field: service + type: string + - field: host + type: string + - field: service_alias + type: string + - field: host_alias + type: string + - field: processed + type: boolean +"#; + + parse(&Content::Yaml(pipeline_yaml)).unwrap() +} + fn criterion_benchmark(c: &mut Criterion) { let input_value_str = include_str!("./data.log"); let input_value = Deserializer::from_str(input_value_str) @@ -262,6 +292,41 @@ fn criterion_benchmark(c: &mut Criterion) { }) }); group.finish(); + + let vrl_input_value = (0..128) + .map(|i| { + serde_json::json!({ + "service": "frontend", + "host": format!("host-{i}"), + "unused": "drop-me" + }) + .into() + }) + .collect::>(); + let vrl_pipeline = prepare_vrl_pipeline(); + + let (vrl_pipeline, mut vrl_schema_info, vrl_pipeline_def, vrl_pipeline_param) = + setup_pipeline!(vrl_pipeline); + let vrl_pipeline_ctx = PipelineContext::new( + &vrl_pipeline_def, + &vrl_pipeline_param, + session::context::Channel::Unknown, + ); + + let mut group = c.benchmark_group("vrl processor"); + group.sample_size(50); + group.bench_function("processor mut", |b| { + b.iter(|| { + processor_mut( + black_box(vrl_pipeline.clone()), + black_box(&vrl_pipeline_ctx), + black_box(&mut vrl_schema_info), + black_box(vrl_input_value.clone()), + ) + .unwrap(); + }) + }); + group.finish(); } // Testing the pipeline's performance in converting Json to Rows diff --git a/src/pipeline/src/etl/processor/vrl_processor.rs b/src/pipeline/src/etl/processor/vrl_processor.rs index 20258a0427..ee3452523d 100644 --- a/src/pipeline/src/etl/processor/vrl_processor.rs +++ b/src/pipeline/src/etl/processor/vrl_processor.rs @@ -12,9 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::cell::RefCell; use std::collections::BTreeMap; use chrono_tz::Tz; +use once_cell::sync::Lazy; use snafu::{OptionExt, ensure}; use vrl::compiler::runtime::Runtime; use vrl::compiler::{Program, TargetValue, compile}; @@ -31,6 +33,12 @@ use crate::etl::processor::yaml_string; pub(crate) const PROCESSOR_VRL: &str = "vrl"; const SOURCE: &str = "source"; +static UTC_TIMEZONE: Lazy = Lazy::new(|| TimeZone::Named(Tz::UTC)); + +thread_local! { + static VRL_RUNTIME: RefCell = RefCell::new(Runtime::default()); +} + #[derive(Debug)] pub struct VrlProcessor { source: String, @@ -74,10 +82,14 @@ impl VrlProcessor { secrets: Secrets::default(), }; - let timezone = TimeZone::Named(Tz::UTC); - let mut runtime = Runtime::default(); - let re = runtime - .resolve(&mut target, &self.program, &timezone) + let re = VRL_RUNTIME + .with(|runtime| { + let mut runtime = runtime.borrow_mut(); + runtime.clear(); + let result = runtime.resolve(&mut target, &self.program, &UTC_TIMEZONE); + runtime.clear(); + result + }) .map_err(|e| { ExecuteVrlSnafu { msg: e.get_expression_error().to_string(), From aaffe1a048cab04ccec62f9fba3b1a837643e60e Mon Sep 17 00:00:00 2001 From: jeremyhi Date: Tue, 2 Jun 2026 20:10:10 -0700 Subject: [PATCH 19/29] refactor(cli): split export-v2 verify into plan/scan/reconcile (#8206) * refactor(cli): split export-v2 verify into plan/scan/reconcile Signed-off-by: jeremyhi * fix: by AI comments Signed-off-by: jeremyhi --------- Signed-off-by: jeremyhi --- src/cli/src/data/export_v2/command.rs | 248 +++++++++++++++++++++----- 1 file changed, 206 insertions(+), 42 deletions(-) diff --git a/src/cli/src/data/export_v2/command.rs b/src/cli/src/data/export_v2/command.rs index db0f576a4e..bb027bbef1 100644 --- a/src/cli/src/data/export_v2/command.rs +++ b/src/cli/src/data/export_v2/command.rs @@ -1077,7 +1077,9 @@ async fn verify_snapshot(storage: &OpenDalStorage) -> Result { )); } let data_files = storage.list_files_recursive("data/").await?; - if let Some(path) = data_files.first() { + // Report the lexicographically smallest path so the message is stable + // regardless of listing order across backends. + if let Some(path) = data_files.iter().min() { report.push_error(format!( "Schema-only snapshot should not contain data files (found '{}')", path @@ -1103,75 +1105,113 @@ fn summarize_chunks(manifest: &Manifest) -> VerifyChunkSummary { } } +/// A data file declared by a completed chunk that is expected to exist in storage. +#[derive(Debug)] +struct ChunkFile { + chunk_id: u32, + path: String, +} + +/// Expected snapshot contents derived purely from the manifest (no object-store IO). +/// +/// Separating planning from scanning makes it obvious which problems come from +/// the manifest alone and which require comparing against actual storage. +#[derive(Debug, Default)] +struct VerifyPlan { + /// Valid data files declared by completed chunks; each must exist in storage. + files_to_check: Vec, + /// All syntactically-safe data paths declared by any chunk, regardless of + /// status. Used as the orphan-detection baseline so a listed-but-invalid + /// file is not also reported as unexpected. + claimed_data_files: HashSet, + /// Total data-file references in completed chunks (valid + invalid). + data_files_total: usize, + /// Problems detectable from the manifest alone. + problems: Vec, +} + +/// Actual data files discovered under `data/` (the only object-store IO in +/// chunk/data-file verification). +#[derive(Debug)] +struct VerifyDataScan { + existing_data_files: HashSet, +} + +/// Result of reconciling the manifest plan against the storage scan. +#[derive(Debug, Default)] +struct VerifyOutcome { + data_files_total: usize, + data_files_verified: usize, + problems: Vec, +} + async fn verify_chunks_and_data_files( storage: &OpenDalStorage, report: &mut VerifyReport, ) -> Result<()> { - let existing_files: HashSet<_> = storage - .list_files_recursive("data/") - .await? - .into_iter() - .collect(); - let mut data_files_total = 0; - let mut data_files_verified = 0; - let mut problems = Vec::new(); - let mut seen_chunk_ids = HashSet::new(); - let mut claimed_data_files = HashSet::new(); + let plan = build_verify_plan(&report.manifest); + let scan = scan_data_files(storage).await?; + let outcome = reconcile_plan_with_scan(plan, &scan); - for chunk in &report.manifest.chunks { + report.data_files_total = outcome.data_files_total; + report.data_files_verified = outcome.data_files_verified; + report.problems.extend(outcome.problems); + + Ok(()) +} + +/// Builds the expected-state plan from the manifest. Pure; performs no IO. +fn build_verify_plan(manifest: &Manifest) -> VerifyPlan { + let mut plan = VerifyPlan::default(); + let mut seen_chunk_ids = HashSet::new(); + + for chunk in &manifest.chunks { if !seen_chunk_ids.insert(chunk.id) { - problems.push(VerifyProblem { + plan.problems.push(VerifyProblem { severity: VerifySeverity::Error, message: format!("Chunk {}: duplicate chunk id", chunk.id), }); } for file in &chunk.files { if let Some(path) = safe_manifest_data_file_path(file) { - claimed_data_files.insert(path.to_string()); + plan.claimed_data_files.insert(path.to_string()); } } match chunk.status { ChunkStatus::Completed => { if chunk.files.is_empty() { - problems.push(VerifyProblem { + plan.problems.push(VerifyProblem { severity: VerifySeverity::Error, message: format!("Chunk {}: completed chunk has no data files", chunk.id), }); continue; } - let allowed_prefixes = report - .manifest + let allowed_prefixes = manifest .schemas .iter() .map(|schema| data_dir_for_schema_chunk(schema, chunk.id)) .collect::>(); for file in &chunk.files { - data_files_total += 1; - let Some(path) = valid_manifest_data_file_path(file, &allowed_prefixes) else { - problems.push(VerifyProblem { + plan.data_files_total += 1; + match valid_manifest_data_file_path(file, &allowed_prefixes) { + Some(path) => plan.files_to_check.push(ChunkFile { + chunk_id: chunk.id, + path: path.to_string(), + }), + None => plan.problems.push(VerifyProblem { severity: VerifySeverity::Error, message: format!( "Chunk {}: invalid data file path '{}'", chunk.id, file ), - }); - continue; - }; - - if existing_files.contains(path) { - data_files_verified += 1; - } else { - problems.push(VerifyProblem { - severity: VerifySeverity::Error, - message: format!("Chunk {}: missing file '{}'", chunk.id, path), - }); + }), } } } ChunkStatus::Skipped => { if !chunk.files.is_empty() { - problems.push(VerifyProblem { + plan.problems.push(VerifyProblem { severity: VerifySeverity::Error, message: format!( "Chunk {}: skipped chunk should not list data files", @@ -1181,20 +1221,20 @@ async fn verify_chunks_and_data_files( } } ChunkStatus::Pending => { - problems.push(VerifyProblem { + plan.problems.push(VerifyProblem { severity: VerifySeverity::Error, message: format!("Chunk {}: status is 'pending'", chunk.id), }); } ChunkStatus::InProgress => { - problems.push(VerifyProblem { + plan.problems.push(VerifyProblem { severity: VerifySeverity::Error, message: format!("Chunk {}: status is 'in_progress'", chunk.id), }); } ChunkStatus::Failed => { let reason = chunk.error.as_deref().unwrap_or("unknown error"); - problems.push(VerifyProblem { + plan.problems.push(VerifyProblem { severity: VerifySeverity::Error, message: format!("Chunk {}: status is 'failed' (error: {})", chunk.id, reason), }); @@ -1202,20 +1242,60 @@ async fn verify_chunks_and_data_files( } } - for path in &existing_files { - if !claimed_data_files.contains(path) { + plan +} + +/// Lists all data files under `data/`. This is the only object-store IO in +/// chunk/data-file verification. +async fn scan_data_files(storage: &OpenDalStorage) -> Result { + let existing_data_files = storage + .list_files_recursive("data/") + .await? + .into_iter() + .collect(); + Ok(VerifyDataScan { + existing_data_files, + }) +} + +/// Reconciles the manifest plan against the storage scan. Pure; performs no IO. +/// +/// Emits missing-file problems for expected files absent from storage and +/// unexpected-file problems for storage files no chunk claims. Unexpected files +/// are sorted by path so output is deterministic regardless of listing order. +fn reconcile_plan_with_scan(plan: VerifyPlan, scan: &VerifyDataScan) -> VerifyOutcome { + let mut problems = plan.problems; + let mut data_files_verified = 0; + + for file in &plan.files_to_check { + if scan.existing_data_files.contains(&file.path) { + data_files_verified += 1; + } else { problems.push(VerifyProblem { severity: VerifySeverity::Error, - message: format!("Unexpected data file '{}' is not listed in manifest", path), + message: format!("Chunk {}: missing file '{}'", file.chunk_id, file.path), }); } } - report.data_files_total = data_files_total; - report.data_files_verified = data_files_verified; - report.problems.extend(problems); + let mut orphans: Vec<&String> = scan + .existing_data_files + .iter() + .filter(|path| !plan.claimed_data_files.contains(*path)) + .collect(); + orphans.sort(); + for path in orphans { + problems.push(VerifyProblem { + severity: VerifySeverity::Error, + message: format!("Unexpected data file '{}' is not listed in manifest", path), + }); + } - Ok(()) + VerifyOutcome { + data_files_total: plan.data_files_total, + data_files_verified, + problems, + } } fn valid_manifest_data_file_path<'a>( @@ -2294,6 +2374,90 @@ mod tests { ); } + #[test] + fn test_build_verify_plan_classifies_chunks_without_io() { + let mut manifest = test_manifest( + chrono::Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(), + false, + true, + ); + // test_manifest(complete) gives: chunk 1 completed (1 file), chunk 2 skipped. + let mut failed = ChunkMeta::new(3, TimeRange::unbounded()); + failed.mark_failed("boom".to_string()); + manifest.chunks.push(failed); + manifest + .chunks + .push(ChunkMeta::new(4, TimeRange::unbounded())); + + let plan = build_verify_plan(&manifest); + + assert_eq!(plan.files_to_check.len(), 1); + assert_eq!(plan.files_to_check[0].chunk_id, 1); + assert_eq!(plan.files_to_check[0].path, "data/public/1/file.parquet"); + assert_eq!(plan.data_files_total, 1); + assert!( + plan.claimed_data_files + .contains("data/public/1/file.parquet") + ); + assert_eq!(plan.problems.len(), 2); + assert!( + plan.problems + .iter() + .any(|problem| problem.message.contains("status is 'failed'")) + ); + assert!( + plan.problems + .iter() + .any(|problem| problem.message.contains("status is 'pending'")) + ); + } + + #[tokio::test] + async fn test_verify_snapshot_produces_deterministic_problem_output() { + let dir = tempdir().unwrap(); + let manifest = test_manifest( + chrono::Utc.with_ymd_and_hms(2026, 1, 1, 0, 0, 0).unwrap(), + false, + true, + ); + write_root_manifest(dir.path(), manifest); + write_snapshot_file(dir.path(), "schema/schemas.json", b"[]"); + write_default_ddl_files(dir.path()); + write_snapshot_file(dir.path(), "data/public/1/file.parquet", b"data"); + // Many orphan files under a known chunk prefix to stress ordering. + for i in 0..50 { + write_snapshot_file( + dir.path(), + &format!("data/public/1/orphan_{:02}.parquet", i), + b"x", + ); + } + + let storage = file_storage_for_dir(dir.path()); + let messages = |report: &VerifyReport| { + report + .problems + .iter() + .map(|problem| problem.message.clone()) + .collect::>() + }; + let first = messages(&verify_snapshot(&storage).await.unwrap()); + let second = messages(&verify_snapshot(&storage).await.unwrap()); + + // Output is identical across runs despite HashSet-based scanning. + assert_eq!(first, second); + + let orphans = first + .iter() + .filter(|message| message.contains("Unexpected data file")) + .cloned() + .collect::>(); + assert_eq!(orphans.len(), 50); + let mut sorted = orphans.clone(); + sorted.sort(); + assert_eq!(orphans, sorted); + } + fn write_test_manifest(root: &std::path::Path, dir: &str, manifest: Manifest) { let snapshot_dir = root.join(dir); std::fs::create_dir_all(&snapshot_dir).unwrap(); From 11f06709cd5bc7d66ad90a0b633a64f93a06a540 Mon Sep 17 00:00:00 2001 From: shuiyisong <113876041+shuiyisong@users.noreply.github.com> Date: Wed, 3 Jun 2026 12:13:32 +0800 Subject: [PATCH 20/29] chore: remove `HttpConfigurator` (#8224) Signed-off-by: shuiyisong --- src/frontend/src/server.rs | 1 - src/servers/src/configurator.rs | 14 -------------- src/servers/src/http.rs | 24 ++---------------------- 3 files changed, 2 insertions(+), 37 deletions(-) diff --git a/src/frontend/src/server.rs b/src/frontend/src/server.rs index e66ae718ba..6b120ccba5 100644 --- a/src/frontend/src/server.rs +++ b/src/frontend/src/server.rs @@ -288,7 +288,6 @@ where let http_server = builder .with_metrics_handler(MetricsHandler) - .with_plugins(self.plugins.clone()) .with_greptime_config_options(toml) .build(); Ok(http_server) diff --git a/src/servers/src/configurator.rs b/src/servers/src/configurator.rs index e8ba8264bd..7116fe0ce8 100644 --- a/src/servers/src/configurator.rs +++ b/src/servers/src/configurator.rs @@ -14,25 +14,11 @@ use std::sync::Arc; -use axum::Router as HttpRouter; use common_error::ext::BoxedError; use tonic::transport::server::Router as GrpcRouter; use crate::grpc::builder::GrpcServerBuilder; -/// A configurator that customizes or enhances an HTTP router. -#[async_trait::async_trait] -pub trait HttpConfigurator: Send + Sync { - /// Configures the given HTTP router using the provided context. - async fn configure_http( - &self, - route: HttpRouter, - ctx: C, - ) -> std::result::Result; -} - -pub type HttpConfiguratorRef = Arc>; - /// A configurator that customizes or enhances a gRPC router. #[async_trait::async_trait] pub trait GrpcRouterConfigurator: Send + Sync { diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs index e5dc5380d1..6d3ab76ec1 100644 --- a/src/servers/src/http.rs +++ b/src/servers/src/http.rs @@ -27,7 +27,6 @@ use axum::response::{IntoResponse, Response}; use axum::routing::Route; use axum::serve::ListenerExt; use axum::{Router, middleware, routing}; -use common_base::Plugins; use common_base::readable_size::ReadableSize; use common_recordbatch::RecordBatch; use common_telemetry::{error, info}; @@ -52,11 +51,9 @@ use tower_http::trace::TraceLayer; use self::authorize::AuthState; use self::result::table_result::TableResponse; -use crate::configurator::HttpConfiguratorRef; use crate::elasticsearch; use crate::error::{ - AddressBindSnafu, AlreadyStartedSnafu, Error, InternalIoSnafu, InvalidHeaderValueSnafu, - OtherSnafu, Result, + AddressBindSnafu, AlreadyStartedSnafu, Error, InternalIoSnafu, InvalidHeaderValueSnafu, Result, }; use crate::http::influxdb::{influxdb_health, influxdb_ping, influxdb_write_v1, influxdb_write_v2}; use crate::http::otlp::OtlpState; @@ -139,9 +136,6 @@ pub struct HttpServer { user_provider: Option, memory_limiter: ServerMemoryLimiter, - // plugins - plugins: Plugins, - // server configs options: HttpOptions, bind_addr: Option, @@ -516,7 +510,6 @@ pub struct DashboardState { pub struct HttpServerBuilder { options: HttpOptions, - plugins: Plugins, user_provider: Option, router: Router, memory_limiter: ServerMemoryLimiter, @@ -526,7 +519,6 @@ impl HttpServerBuilder { pub fn new(options: HttpOptions) -> Self { Self { options, - plugins: Plugins::default(), user_provider: None, router: Router::new(), memory_limiter: ServerMemoryLimiter::default(), @@ -687,10 +679,6 @@ impl HttpServerBuilder { Self { router, ..self } } - pub fn with_plugins(self, plugins: Plugins) -> Self { - Self { plugins, ..self } - } - pub fn with_greptime_config_options(self, opts: String) -> Self { let config_router = HttpServer::route_config(GreptimeOptionsConfigState { greptime_config_options: opts, @@ -748,7 +736,6 @@ impl HttpServerBuilder { options: self.options, user_provider: self.user_provider, shutdown_tx: Mutex::new(None), - plugins: self.plugins, router: StdMutex::new(self.router), bind_addr: None, memory_limiter: self.memory_limiter, @@ -1237,14 +1224,7 @@ impl Server for HttpServer { AlreadyStartedSnafu { server: "HTTP" } ); - let mut app = self.make_app(); - if let Some(configurator) = self.plugins.get::>() { - app = configurator - .configure_http(app, ()) - .await - .context(OtherSnafu)?; - } - let app = self.build(app)?; + let app = self.build(self.make_app())?; let listener = tokio::net::TcpListener::bind(listening) .await .context(AddressBindSnafu { addr: listening })? From 91d7dc547b643e5bad96614ca9a40103efb21e24 Mon Sep 17 00:00:00 2001 From: liyang Date: Wed, 3 Jun 2026 14:42:49 +0800 Subject: [PATCH 21/29] ci: update arm64 runner (#8226) * ci: update arm64 runner Fixed: https://github.com/GreptimeTeam/greptimedb/actions/runs/26856891871/job/79202060647 * Update dev-build.yml --- .github/workflows/dev-build.yml | 2 +- .github/workflows/nightly-build.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index d03fbeff14..c3af006f54 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -30,7 +30,7 @@ on: linux_arm64_runner: type: choice description: The runner uses to build linux-arm64 artifacts - default: ec2-c6g.4xlarge-arm64 + default: ec2-c6g.8xlarge-arm64 options: - ec2-c6g.xlarge-arm64 # 4C8G - ec2-c6g.2xlarge-arm64 # 8C16G diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 14ebb6e715..54af32a94b 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -27,7 +27,7 @@ on: linux_arm64_runner: type: choice description: The runner uses to build linux-arm64 artifacts - default: ec2-c6g.4xlarge-arm64 + default: ec2-c6g.8xlarge-arm64 options: - ec2-c6g.xlarge-arm64 # 4C8G - ec2-c6g.2xlarge-arm64 # 8C16G From 31cdd864d217a759639445fb3b5751013f916f49 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Wed, 3 Jun 2026 20:05:50 +0800 Subject: [PATCH 22/29] fix: format detected IPv6 grpc address (#8221) Signed-off-by: WenyXu --- src/servers/src/grpc.rs | 58 ++++++++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 10 deletions(-) diff --git a/src/servers/src/grpc.rs b/src/servers/src/grpc.rs index 3adfd24945..50dd0b69c4 100644 --- a/src/servers/src/grpc.rs +++ b/src/servers/src/grpc.rs @@ -24,7 +24,7 @@ pub mod prom_query_gateway; pub mod region_server; use std::any::Any; -use std::net::SocketAddr; +use std::net::{IpAddr, SocketAddr}; use std::time::Duration; use api::v1::health_check_server::{HealthCheck, HealthCheckServer}; @@ -95,14 +95,8 @@ impl GrpcOptions { if self.server_addr.is_empty() { match local_ip_address::local_ip() { Ok(ip) => { - let detected_addr = format!( - "{}:{}", - ip, - self.bind_addr - .split(':') - .nth(1) - .unwrap_or(DEFAULT_GRPC_ADDR_PORT) - ); + let port = port_from_bind_addr(&self.bind_addr); + let detected_addr = format_server_addr(ip, port); info!("Using detected: {} as server address", detected_addr); self.server_addr = detected_addr; } @@ -131,7 +125,18 @@ impl GrpcOptions { } } -const DEFAULT_GRPC_ADDR_PORT: &str = "4001"; +const DEFAULT_GRPC_ADDR_PORT: u16 = 4001; + +fn port_from_bind_addr(bind_addr: &str) -> u16 { + bind_addr + .rsplit_once(':') + .and_then(|(_, port)| port.parse().ok()) + .unwrap_or(DEFAULT_GRPC_ADDR_PORT) +} + +fn format_server_addr(ip: IpAddr, port: u16) -> String { + SocketAddr::new(ip, port).to_string() +} const DEFAULT_INTERNAL_GRPC_ADDR_PORT: &str = "4010"; @@ -415,3 +420,36 @@ impl Server for GrpcServer { self } } + +#[cfg(test)] +mod tests { + use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; + + use super::{DEFAULT_GRPC_ADDR_PORT, format_server_addr, port_from_bind_addr}; + + #[test] + fn test_port_from_bind_addr() { + assert_eq!(3002, port_from_bind_addr("127.0.0.1:3002")); + assert_eq!(3002, port_from_bind_addr("[::]:3002")); + assert_eq!( + 3002, + port_from_bind_addr("greptimedb-metasrv.default.svc.cluster.local:3002") + ); + assert_eq!( + DEFAULT_GRPC_ADDR_PORT, + port_from_bind_addr("invalid-bind-addr") + ); + } + + #[test] + fn test_format_server_addr() { + assert_eq!( + "127.0.0.1:3002", + format_server_addr(IpAddr::V4(Ipv4Addr::LOCALHOST), 3002) + ); + assert_eq!( + "[::1]:3002", + format_server_addr(IpAddr::V6(Ipv6Addr::LOCALHOST), 3002) + ); + } +} From ca07a53debc3e64c03a03f2521c500c237453528 Mon Sep 17 00:00:00 2001 From: QuakeWang <45645138+QuakeWang@users.noreply.github.com> Date: Wed, 3 Jun 2026 20:11:02 +0800 Subject: [PATCH 23/29] feat: support CSV copy skip bad records (#8198) * feat: support CSV copy headers and skip bad records Signed-off-by: QuakeWang * refactor: reuse CSV skippable error helper Signed-off-by: QuakeWang * fix: address CSV skip bad records review Signed-off-by: QuakeWang * docs: clarify CSV skip bad records performance tradeoff Signed-off-by: QuakeWang --------- Signed-off-by: QuakeWang --- src/common/datasource/src/file_format.rs | 1 + src/common/datasource/src/file_format/csv.rs | 238 +++++++++++++++++- src/operator/src/statement/copy_table_from.rs | 182 +++++++++++++- src/sql/src/parsers/copy_parser.rs | 22 ++ .../common/copy/copy_from_fs_csv.result | 22 ++ .../common/copy/copy_from_fs_csv.sql | 9 + tests/data/csv/skip_bad_records.csv | 4 + 7 files changed, 457 insertions(+), 21 deletions(-) create mode 100644 tests/data/csv/skip_bad_records.csv diff --git a/src/common/datasource/src/file_format.rs b/src/common/datasource/src/file_format.rs index e36f94c0d2..d9d7b8b648 100644 --- a/src/common/datasource/src/file_format.rs +++ b/src/common/datasource/src/file_format.rs @@ -61,6 +61,7 @@ pub const FORMAT_COMPRESSION_TYPE: &str = "compression_type"; pub const FORMAT_DELIMITER: &str = "delimiter"; pub const FORMAT_SCHEMA_INFER_MAX_RECORD: &str = "schema_infer_max_record"; pub const FORMAT_HAS_HEADER: &str = "has_header"; +pub const FORMAT_SKIP_BAD_RECORDS: &str = "skip_bad_records"; pub const FORMAT_TYPE: &str = "format"; pub const FILE_PATTERN: &str = "pattern"; pub const TIMESTAMP_FORMAT: &str = "timestamp_format"; diff --git a/src/common/datasource/src/file_format/csv.rs b/src/common/datasource/src/file_format/csv.rs index 77ea553f35..2b39051b48 100644 --- a/src/common/datasource/src/file_format/csv.rs +++ b/src/common/datasource/src/file_format/csv.rs @@ -13,15 +13,24 @@ // limitations under the License. use std::collections::HashMap; +use std::io; use std::str::FromStr; +use std::sync::Arc; +use std::task::Poll; use arrow::csv::reader::Format; use arrow::csv::{self, WriterBuilder}; +use arrow::error::ArrowError; use arrow::record_batch::RecordBatch; -use arrow_schema::Schema; +use arrow_schema::{Schema, SchemaRef}; use async_trait::async_trait; +use bytes::{Buf, Bytes}; use common_runtime; +use common_telemetry::warn; use datafusion::physical_plan::SendableRecordBatchStream; +use datafusion::physical_plan::stream::RecordBatchStreamAdapter; +use futures::StreamExt; +use futures::stream::BoxStream; use object_store::ObjectStore; use snafu::ResultExt; use tokio_util::compat::FuturesAsyncReadCompatExt; @@ -34,9 +43,12 @@ use crate::file_format::{self, FileFormat, stream_to_file}; use crate::share_buffer::SharedBuffer; use crate::util::normalize_infer_schema; +const SKIP_BAD_RECORDS_BATCH_SIZE: usize = 1; + #[derive(Debug, Clone, PartialEq, Eq)] pub struct CsvFormat { pub has_header: bool, + pub skip_bad_records: bool, pub delimiter: u8, pub schema_infer_max_record: Option, pub compression_type: CompressionType, @@ -76,13 +88,11 @@ impl TryFrom<&HashMap> for CsvFormat { })?); }; if let Some(has_header) = value.get(file_format::FORMAT_HAS_HEADER) { - format.has_header = has_header.parse().map_err(|_| { - error::ParseFormatSnafu { - key: file_format::FORMAT_HAS_HEADER, - value: has_header, - } - .build() - })?; + format.has_header = parse_bool(file_format::FORMAT_HAS_HEADER, has_header)?; + }; + if let Some(skip_bad_records) = value.get(file_format::FORMAT_SKIP_BAD_RECORDS) { + format.skip_bad_records = + parse_bool(file_format::FORMAT_SKIP_BAD_RECORDS, skip_bad_records)?; }; if let Some(timestamp_format) = value.get(file_format::TIMESTAMP_FORMAT) { format.timestamp_format = Some(timestamp_format.clone()); @@ -97,10 +107,17 @@ impl TryFrom<&HashMap> for CsvFormat { } } +fn parse_bool(key: &'static str, value: &str) -> Result { + value + .parse() + .map_err(|_| error::ParseFormatSnafu { key, value }.build()) +} + impl Default for CsvFormat { fn default() -> Self { Self { has_header: true, + skip_bad_records: false, delimiter: b',', schema_infer_max_record: Some(file_format::DEFAULT_SCHEMA_INFER_MAX_RECORD), compression_type: CompressionType::Uncompressed, @@ -189,10 +206,136 @@ impl DfRecordBatchEncoder for csv::Writer { } } +/// Builds a CSV stream that can skip selected record-level parse/cast errors. +/// +/// This recovery path intentionally uses one-record batches. It is slower than +/// normal CSV scanning, but keeps each parse/cast failure isolated to a single +/// record. Arrow's CSV decoder clears buffered rows before type parsing, so a +/// failed multi-row flush cannot be safely retried row by row without replaying +/// input bytes. +pub async fn tolerant_csv_stream( + store: &ObjectStore, + path: &str, + schema: SchemaRef, + projection: Vec, + format: &CsvFormat, +) -> Result { + let meta = store + .stat(path) + .await + .context(error::ReadObjectSnafu { path })?; + + let reader = store + .reader(path) + .await + .context(error::ReadObjectSnafu { path })? + .into_bytes_stream(0..meta.content_length()) + .await + .context(error::ReadObjectSnafu { path })?; + + let reader = format.compression_type.convert_stream(reader).boxed(); + tolerant_csv_stream_from_reader( + reader, + path, + schema, + projection, + format.has_header, + format.delimiter, + ) +} + +fn tolerant_csv_stream_from_reader( + reader: BoxStream<'static, io::Result>, + path: &str, + schema: SchemaRef, + projection: Vec, + has_header: bool, + delimiter: u8, +) -> Result { + let projected_schema = Arc::new( + schema + .project(&projection) + .context(error::InferSchemaSnafu)?, + ); + let mut decoder = csv::ReaderBuilder::new(schema) + .with_header(has_header) + .with_delimiter(delimiter) + .with_batch_size(SKIP_BAD_RECORDS_BATCH_SIZE) + .with_projection(projection) + .build_decoder(); + + let path = path.to_string(); + let mut upstream = reader.fuse(); + let mut buffered = Bytes::new(); + let mut input_finished = false; + let stream = futures::stream::poll_fn(move |cx| { + loop { + while !input_finished { + if buffered.is_empty() { + match futures::ready!(upstream.poll_next_unpin(cx)) { + Some(Ok(bytes)) if bytes.is_empty() => continue, + Some(Ok(bytes)) => buffered = bytes, + Some(Err(error)) => return Poll::Ready(Some(Err(error.into()))), + None => input_finished = true, + } + } + + let decoded = decoder.decode(buffered.as_ref())?; + if decoded > 0 { + buffered.advance(decoded); + continue; + } + + if decoder.capacity() == 0 || input_finished { + break; + } + + if buffered.is_empty() { + continue; + } + + return Poll::Ready(Some(Err(ArrowError::ParseError( + "CSV decoder made no progress while input bytes remain".to_string(), + )))); + } + + match decoder.flush() { + Ok(Some(batch)) => return Poll::Ready(Some(Ok(batch))), + Ok(None) if input_finished => return Poll::Ready(None), + Ok(None) => continue, + Err(error) if is_skippable_arrow_error(&error) => { + warn!( + "Skipping bad CSV record while copying from {}: {}", + path, error + ); + } + Err(error) => return Poll::Ready(Some(Err(error))), + } + } + }) + .map(|result: std::result::Result| result.map_err(Into::into)); + + Ok(Box::pin(RecordBatchStreamAdapter::new( + projected_schema, + stream, + ))) +} + +pub fn is_skippable_arrow_error(error: &ArrowError) -> bool { + matches!( + error, + ArrowError::ParseError(_) + | ArrowError::CastError(_) + | ArrowError::ComputeError(_) + | ArrowError::InvalidArgumentError(_) + ) +} + #[cfg(test)] mod tests { use std::sync::Arc; + use arrow_schema::{DataType, Field}; use common_recordbatch::adapter::DfRecordBatchStreamAdapter; use common_recordbatch::{RecordBatch, RecordBatches}; use common_test_util::find_workspace_path; @@ -205,7 +348,7 @@ mod tests { use super::*; use crate::file_format::{ FORMAT_COMPRESSION_TYPE, FORMAT_DELIMITER, FORMAT_HAS_HEADER, - FORMAT_SCHEMA_INFER_MAX_RECORD, FileFormat, file_to_stream, + FORMAT_SCHEMA_INFER_MAX_RECORD, FORMAT_SKIP_BAD_RECORDS, FileFormat, file_to_stream, }; use crate::test_util::{format_schema, test_store}; @@ -331,11 +474,29 @@ mod tests { schema_infer_max_record: Some(2000), delimiter: b'\t', has_header: false, + skip_bad_records: false, timestamp_format: None, time_format: None, date_format: None } ); + + let map = HashMap::from([(FORMAT_SKIP_BAD_RECORDS.to_string(), "true".to_string())]); + let format = CsvFormat::try_from(&map).unwrap(); + + assert_eq!( + format, + CsvFormat { + skip_bad_records: true, + ..CsvFormat::default() + } + ); + } + + #[test] + fn test_try_from_rejects_invalid_bool_options() { + let map = HashMap::from([(FORMAT_SKIP_BAD_RECORDS.to_string(), "yes".to_string())]); + assert!(CsvFormat::try_from(&map).is_err()); } #[tokio::test] @@ -496,4 +657,63 @@ mod tests { assert_eq!(expected, pretty_print); } } + + #[tokio::test] + async fn test_tolerant_csv_stream_continues_after_parse_error() { + let temp_dir = common_test_util::temp_dir::create_temp_dir("test_tolerant_csv_stream"); + let csv_file_path = temp_dir.path().join("input.csv"); + std::fs::write( + &csv_file_path, + "id,name,value\n1,Alice,10.5\nbad,Bad,20.0\nworse,Bad,21.0\n2,Bob,30.5", + ) + .unwrap(); + + let store = test_store("/"); + let schema = Arc::new(arrow_schema::Schema::new(vec![ + Field::new("id", DataType::UInt32, false), + Field::new("name", DataType::Utf8, false), + Field::new("value", DataType::Float64, false), + ])); + let path = csv_file_path.to_str().unwrap(); + + let stream = + tolerant_csv_stream(&store, path, schema, vec![0, 1, 2], &CsvFormat::default()) + .await + .unwrap(); + let batches = stream.try_collect::>().await.unwrap(); + let pretty_print = arrow::util::pretty::pretty_format_batches(&batches) + .unwrap() + .to_string(); + let expected = r#"+----+-------+-------+ +| id | name | value | ++----+-------+-------+ +| 1 | Alice | 10.5 | +| 2 | Bob | 30.5 | ++----+-------+-------+"#; + assert_eq!(expected, pretty_print); + } + + #[tokio::test] + async fn test_tolerant_csv_stream_fails_on_structural_csv_error() { + let temp_dir = + common_test_util::temp_dir::create_temp_dir("test_tolerant_csv_stream_csv_error"); + let csv_file_path = temp_dir.path().join("input.csv"); + std::fs::write(&csv_file_path, "id,name,value\n1,Alice,10.5\n2,Bob\n").unwrap(); + + let store = test_store("/"); + let schema = Arc::new(arrow_schema::Schema::new(vec![ + Field::new("id", DataType::UInt32, false), + Field::new("name", DataType::Utf8, false), + Field::new("value", DataType::Float64, false), + ])); + let path = csv_file_path.to_str().unwrap(); + + let stream = + tolerant_csv_stream(&store, path, schema, vec![0, 1, 2], &CsvFormat::default()) + .await + .unwrap(); + let error = stream.try_collect::>().await.unwrap_err(); + + assert!(error.to_string().contains("incorrect number of fields")); + } } diff --git a/src/operator/src/statement/copy_table_from.rs b/src/operator/src/statement/copy_table_from.rs index 6f58603247..cae2835242 100644 --- a/src/operator/src/statement/copy_table_from.rs +++ b/src/operator/src/statement/copy_table_from.rs @@ -15,11 +15,15 @@ use std::collections::HashMap; use std::future::Future; use std::path::Path; +use std::pin::Pin; use std::sync::Arc; +use std::task::{Context, Poll}; use client::{Output, OutputData, OutputMeta}; use common_base::readable_size::ReadableSize; -use common_datasource::file_format::csv::CsvFormat; +use common_datasource::file_format::csv::{ + CsvFormat, is_skippable_arrow_error, tolerant_csv_stream, +}; use common_datasource::file_format::json::JsonFormat; use common_datasource::file_format::orc::{ReaderAdapter, infer_orc_schema, new_orc_stream_reader}; use common_datasource::file_format::{FileFormat, Format, file_to_stream}; @@ -33,10 +37,13 @@ use common_telemetry::{debug, tracing}; use datafusion::datasource::physical_plan::{CsvSource, FileSource, JsonSource}; use datafusion::parquet::arrow::ParquetRecordBatchStreamBuilder; use datafusion::parquet::arrow::arrow_reader::ArrowReaderMetadata; +use datafusion_common::DataFusionError; +use datafusion_common::arrow::error::ArrowError; use datafusion_common::config::CsvOptions; use datafusion_expr::Expr; use datatypes::arrow::compute::can_cast_types; use datatypes::arrow::datatypes::{DataType as ArrowDataType, Schema, SchemaRef}; +use datatypes::arrow::record_batch::RecordBatch; use datatypes::vectors::Helper; use futures_util::StreamExt; use object_store::{Entry, EntryMode, ObjectStore}; @@ -221,23 +228,42 @@ impl StatementExecutor { let csv_source = CsvSource::new(schema.clone()) .with_csv_options(options) .with_batch_size(DEFAULT_BATCH_SIZE); - let stream = file_to_stream( - object_store, - path, - csv_source, - Some(projection), - format.compression_type, - ) - .await - .context(error::BuildFileStreamSnafu)?; + let stream = if format.skip_bad_records { + let reader_schema = + csv_reader_schema_for_skip_bad_records(schema, &compat_schema); + tolerant_csv_stream( + object_store, + path, + Arc::new(reader_schema), + projection.clone(), + format, + ) + .await + .context(error::BuildFileStreamSnafu)? + } else { + file_to_stream( + object_store, + path, + csv_source, + Some(projection), + format.compression_type, + ) + .await + .context(error::BuildFileStreamSnafu)? + }; - Ok(Box::pin( + let stream = Box::pin( // The projection is already applied in the CSV reader when we created the stream, // so we pass None here to avoid double projection which would cause schema mismatch errors. RecordBatchStreamTypeAdapter::new(output_schema, stream, None) .with_filter(filters) .context(error::PhysicalExprSnafu)?, - )) + ); + if format.skip_bad_records { + Ok(Box::pin(SkipBadRecordsStream::new(stream, path))) + } else { + Ok(stream) + } } FileMetadata::Json { path, @@ -469,6 +495,58 @@ fn gen_insert_output(rows_inserted: usize, insert_cost: usize) -> Output { ) } +struct SkipBadRecordsStream { + inner: DfSendableRecordBatchStream, + path: String, +} + +impl SkipBadRecordsStream { + fn new(inner: DfSendableRecordBatchStream, path: impl Into) -> Self { + Self { + inner, + path: path.into(), + } + } +} + +impl datafusion::physical_plan::RecordBatchStream for SkipBadRecordsStream { + fn schema(&self) -> SchemaRef { + self.inner.schema() + } +} + +impl futures::Stream for SkipBadRecordsStream { + type Item = datafusion_common::Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + loop { + match this.inner.as_mut().poll_next(cx) { + Poll::Ready(Some(Err(error))) if is_skippable_record_error(&error) => { + common_telemetry::warn!( + "Skipping bad record while copying from {}: {}", + this.path, + error + ); + continue; + } + other => return other, + } + } + } +} + +fn is_skippable_record_error(error: &DataFusionError) -> bool { + match error { + DataFusionError::ArrowError(error, _) => is_skippable_arrow_error(error), + DataFusionError::External(error) => error + .downcast_ref::() + .is_some_and(is_skippable_arrow_error), + DataFusionError::Context(_, error) => is_skippable_record_error(error), + _ => false, + } +} + /// Executes all pending inserts all at once, drain pending requests and reset pending bytes. async fn batch_insert( pending: &mut Vec>>, @@ -498,6 +576,59 @@ fn can_cast_types_for_greptime(from: &ArrowDataType, to: &ArrowDataType) -> bool can_cast_types(from, to) } +fn csv_reader_schema_for_skip_bad_records(file: &SchemaRef, compat: &SchemaRef) -> Schema { + let fields = file + .fields() + .iter() + .enumerate() + .map(|(idx, file_field)| { + let compat_field = compat + .fields() + .find(file_field.name()) + .map(|(_, field)| field); + + match compat_field { + Some(compat_field) if can_csv_reader_parse_type(compat_field.data_type()) => { + compat_field.clone() + } + _ => file.fields()[idx].clone(), + } + }) + .collect::>(); + + Schema::new_with_metadata(fields, file.metadata().clone()) +} + +fn can_csv_reader_parse_type(data_type: &ArrowDataType) -> bool { + match data_type { + ArrowDataType::Boolean + | ArrowDataType::Decimal32(_, _) + | ArrowDataType::Decimal64(_, _) + | ArrowDataType::Decimal128(_, _) + | ArrowDataType::Decimal256(_, _) + | ArrowDataType::Int8 + | ArrowDataType::Int16 + | ArrowDataType::Int32 + | ArrowDataType::Int64 + | ArrowDataType::UInt8 + | ArrowDataType::UInt16 + | ArrowDataType::UInt32 + | ArrowDataType::UInt64 + | ArrowDataType::Float32 + | ArrowDataType::Float64 + | ArrowDataType::Date32 + | ArrowDataType::Date64 + | ArrowDataType::Time32(_) + | ArrowDataType::Time64(_) + | ArrowDataType::Timestamp(_, _) + | ArrowDataType::Null + | ArrowDataType::Utf8 + | ArrowDataType::Utf8View => true, + ArrowDataType::Dictionary(_, value_type) => value_type.as_ref() == &ArrowDataType::Utf8, + _ => false, + } +} + fn ensure_schema_compatible(from: &SchemaRef, to: &SchemaRef) -> Result<()> { let not_match = from .fields @@ -780,4 +911,31 @@ mod tests { assert_eq!(test.0.project(&fp).unwrap(), test.1.project(&tp).unwrap()); } } + + #[test] + fn test_csv_reader_schema_for_skip_bad_records() { + let file_schema = make_test_schema(&[ + Field::new("id", DataType::Utf8, true), + Field::new("jsons", DataType::Utf8, true), + Field::new("ts", DataType::Utf8, true), + ]); + let compat_schema = make_test_schema(&[ + Field::new("id", DataType::UInt32, true), + Field::new("jsons", DataType::Binary, true), + Field::new( + "ts", + DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None), + true, + ), + ]); + + let reader_schema = csv_reader_schema_for_skip_bad_records(&file_schema, &compat_schema); + + assert_eq!(reader_schema.field(0).data_type(), &DataType::UInt32); + assert_eq!(reader_schema.field(1).data_type(), &DataType::Utf8); + assert_eq!( + reader_schema.field(2).data_type(), + compat_schema.field(2).data_type() + ); + } } diff --git a/src/sql/src/parsers/copy_parser.rs b/src/sql/src/parsers/copy_parser.rs index 9a2eddcc78..491912c82e 100644 --- a/src/sql/src/parsers/copy_parser.rs +++ b/src/sql/src/parsers/copy_parser.rs @@ -401,6 +401,28 @@ mod tests { } } + #[test] + fn test_parse_copy_table_from_csv_options() { + let sql = + "COPY my_table FROM '/tmp/test.csv' WITH (FORMAT = 'CSV', SKIP_BAD_RECORDS = 'false')"; + let mut result = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) + .unwrap(); + assert_eq!(1, result.len()); + + let statement = result.remove(0); + assert_matches!(statement, Statement::Copy { .. }); + match statement { + Statement::Copy(crate::statements::copy::Copy::CopyTable(CopyTable::From( + copy_table, + ))) => { + assert_eq!(copy_table.with.get("format"), Some("CSV")); + assert_eq!(copy_table.with.get("skip_bad_records"), Some("false")); + } + _ => unreachable!(), + } + } + #[test] fn test_parse_copy_table_to() { struct Test<'a> { diff --git a/tests/cases/standalone/common/copy/copy_from_fs_csv.result b/tests/cases/standalone/common/copy/copy_from_fs_csv.result index 2e3373af15..ad81cf1f77 100644 --- a/tests/cases/standalone/common/copy/copy_from_fs_csv.result +++ b/tests/cases/standalone/common/copy/copy_from_fs_csv.result @@ -183,6 +183,24 @@ select * from csv_null_prefix_import; | final | 2023-11-14T22:13:23 | +-------+---------------------+ +CREATE TABLE csv_skip_bad_records(host_id int, host_name string, reading_value double, ts timestamp time index); + +Affected Rows: 0 + +-- SQLNESS ENV PWD +Copy csv_skip_bad_records FROM '$PWD/tests/data/csv/skip_bad_records.csv' WITH (format='csv', skip_bad_records='true'); + +Affected Rows: 2 + +select * from csv_skip_bad_records order by ts; + ++---------+-----------+---------------+---------------------+ +| host_id | host_name | reading_value | ts | ++---------+-----------+---------------+---------------------+ +| 1 | Alice | 10.5 | 2024-01-01T00:00:00 | +| 2 | Bob | 30.5 | 2024-01-01T00:00:02 | ++---------+-----------+---------------+---------------------+ + drop table demo; Affected Rows: 0 @@ -219,3 +237,7 @@ drop table csv_null_prefix_import; Affected Rows: 0 +drop table csv_skip_bad_records; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/copy/copy_from_fs_csv.sql b/tests/cases/standalone/common/copy/copy_from_fs_csv.sql index 0901048177..cd6cfcbd23 100644 --- a/tests/cases/standalone/common/copy/copy_from_fs_csv.sql +++ b/tests/cases/standalone/common/copy/copy_from_fs_csv.sql @@ -73,6 +73,13 @@ Copy csv_null_prefix_import FROM '${SQLNESS_HOME}/demo/export/csv_null_prefix.cs select * from csv_null_prefix_import; +CREATE TABLE csv_skip_bad_records(host_id int, host_name string, reading_value double, ts timestamp time index); + +-- SQLNESS ENV PWD +Copy csv_skip_bad_records FROM '$PWD/tests/data/csv/skip_bad_records.csv' WITH (format='csv', skip_bad_records='true'); + +select * from csv_skip_bad_records order by ts; + drop table demo; drop table with_filename; @@ -90,3 +97,5 @@ drop table demo_with_less_columns; drop table csv_null_prefix; drop table csv_null_prefix_import; + +drop table csv_skip_bad_records; diff --git a/tests/data/csv/skip_bad_records.csv b/tests/data/csv/skip_bad_records.csv new file mode 100644 index 0000000000..f4c40d5d6e --- /dev/null +++ b/tests/data/csv/skip_bad_records.csv @@ -0,0 +1,4 @@ +host_id,host_name,reading_value,ts +1,Alice,10.5,2024-01-01T00:00:00 +bad,Bad,20.0,2024-01-01T00:00:01 +2,Bob,30.5,2024-01-01T00:00:02 From d304df6e7540eafc780ae68b1b696294221d4494 Mon Sep 17 00:00:00 2001 From: discord9 Date: Wed, 3 Jun 2026 21:06:22 +0800 Subject: [PATCH 24/29] fix: run eval-interval flow without time window (#8231) * fix: run eval-interval flow without time window Signed-off-by: discord9 * test: cover eval-interval flow join query Signed-off-by: discord9 * fix: address eval interval flow review comments Signed-off-by: discord9 --------- Signed-off-by: discord9 --- src/flow/src/batching_mode/task.rs | 5 +- src/flow/src/batching_mode/task/test.rs | 32 +++ src/sql/src/util.rs | 200 +++++++++++++++--- .../flow/flow_batch_join_subquery.result | 130 ++++++++++++ .../common/flow/flow_batch_join_subquery.sql | 85 ++++++++ 5 files changed, 426 insertions(+), 26 deletions(-) create mode 100644 tests/cases/standalone/common/flow/flow_batch_join_subquery.result create mode 100644 tests/cases/standalone/common/flow/flow_batch_join_subquery.sql diff --git a/src/flow/src/batching_mode/task.rs b/src/flow/src/batching_mode/task.rs index 5bd328ad9a..3cd96b7525 100644 --- a/src/flow/src/batching_mode/task.rs +++ b/src/flow/src/batching_mode/task.rs @@ -959,7 +959,7 @@ impl BatchingTask { let (expire_lower_bound, expire_upper_bound) = match (expire_time_window_bound, &self.config.query_type) { (Some((Some(l), Some(u))), QueryType::Sql) => (l, u), - (None, QueryType::Sql) => { + (None, QueryType::Sql) if self.config.flow_eval_interval.is_none() => { // if it's sql query and no time window lower/upper bound is found, just return the original query(with auto columns) // use sink_table_meta to add to query the `update_at` and `__ts_placeholder` column's value too for compatibility reason debug!( @@ -980,7 +980,8 @@ impl BatchingTask { } _ => { // Clean dirty windows for full-query/non-scoped paths, - // such as TQL, that cannot use a time-window filter. + // such as TQL or evaluation-interval SQL without a recognized + // time-window expression, that cannot use a time-window filter. let (_, dirty_windows_to_restore) = self.drain_dirty_windows_signal(); let plan_info = self diff --git a/src/flow/src/batching_mode/task/test.rs b/src/flow/src/batching_mode/task/test.rs index d64b4ef1b9..c42d564ce2 100644 --- a/src/flow/src/batching_mode/task/test.rs +++ b/src/flow/src/batching_mode/task/test.rs @@ -974,6 +974,38 @@ async fn test_non_scoped_path_generates_plan_with_empty_dirty_signal() { assert!(task.state.read().unwrap().dirty_time_windows.is_empty()); } +#[tokio::test] +async fn test_no_time_window_sql_with_eval_interval_generates_plan_without_dirty_signal() { + let TestTaskParts { + mut task, + query_engine, + .. + } = new_test_task_engine_and_plan_with_query( + "SELECT number, ts FROM numbers_with_ts", + "missing_sink", + ) + .await; + Arc::get_mut(&mut task.config) + .expect("test task config should be uniquely owned") + .flow_eval_interval = Some(Duration::from_secs(60)); + task.state.write().unwrap().dirty_time_windows.clean(); + let sink_schema = Arc::new(Schema::new(vec![ + ColumnSchema::new("number", CDT::uint32_datatype(), false), + ColumnSchema::new("ts", CDT::timestamp_millisecond_datatype(), false).with_time_index(true), + ])); + + let plan = task + .gen_query_with_time_window(query_engine, &sink_schema, &[], false, None) + .await + .unwrap() + .expect( + "eval-interval SQL without a time-window expr should run by interval, not dirty signal", + ); + + assert!(plan.can_advance_checkpoints); + assert!(task.state.read().unwrap().dirty_time_windows.is_empty()); +} + #[tokio::test] async fn test_executed_query_failure_restores_scoped_dirty_windows_for_flush_path() { let (task, plan) = new_test_task_and_plan_with_missing_sink().await; diff --git a/src/sql/src/util.rs b/src/sql/src/util.rs index bbf5ce3277..f627c43e48 100644 --- a/src/sql/src/util.rs +++ b/src/sql/src/util.rs @@ -27,7 +27,7 @@ use serde::Serialize; use snafu::ensure; use sqlparser::ast::{ Array, Expr, Ident, ObjectName, ObjectNamePart, SetExpr, SqlOption, StructField, TableFactor, - Value, ValueWithSpan, + TableWithJoins, Value, ValueWithSpan, }; use sqlparser_derive::{Visit, VisitMut}; @@ -195,7 +195,7 @@ pub fn extract_tables_from_query(query: &SqlOrTql) -> impl Iterator { - extract_tables_from_set_expr(&query.inner.body, &mut names); + extract_tables_from_sql_query(&query.inner, &mut names); extract_tables_from_hybrid_cte_query(query, &mut names); } SqlOrTql::Tql(tql, _) => extract_tables_from_tql(tql, &mut names), @@ -205,26 +205,34 @@ pub fn extract_tables_from_query(query: &SqlOrTql) -> impl Iterator) { - let mut tql_names = HashSet::new(); - let mut cte_names: HashSet = HashSet::new(); if let Some(hybrid_cte) = &query.hybrid_cte { + let mut cte_names: HashSet = hybrid_cte + .cte_tables + .iter() + .map(|cte| ParserContext::canonicalize_identifier(cte.name.clone()).value) + .collect(); + remove_cte_names(sql_names, &cte_names); + + cte_names.clear(); for cte in &hybrid_cte.cte_tables { - cte_names.insert(ParserContext::canonicalize_identifier(cte.name.clone()).value); - if let CteContent::Tql(tql) = &cte.content { - extract_tables_from_tql(tql, &mut tql_names); + let cte_name = ParserContext::canonicalize_identifier(cte.name.clone()).value; + let mut cte_query_names = HashSet::new(); + match &cte.content { + CteContent::Sql(cte_query) => { + extract_tables_from_sql_query(cte_query, &mut cte_query_names) + } + CteContent::Tql(tql) => extract_tables_from_tql(tql, &mut cte_query_names), + } + if hybrid_cte.recursive { + cte_names.insert(cte_name.clone()); + } + remove_cte_names(&mut cte_query_names, &cte_names); + sql_names.extend(cte_query_names); + if !hybrid_cte.recursive { + cte_names.insert(cte_name); } } } - - if let Some(with) = &query.inner.with { - for cte in &with.cte_tables { - cte_names.insert(ParserContext::canonicalize_identifier(cte.alias.name.clone()).value); - } - } - - remove_cte_names(sql_names, &cte_names); - - sql_names.extend(tql_names); } fn remove_cte_names(names: &mut HashSet, cte_names: &HashSet) { @@ -339,6 +347,33 @@ pub fn location_to_index(sql: &str, location: &sqlparser::tokenizer::Location) - index - 1 } +/// Helper function for [extract_tables_from_query]. +/// +/// Handle [sqlparser::ast::Query]. +fn extract_tables_from_sql_query(query: &sqlparser::ast::Query, names: &mut HashSet) { + let mut cte_names = HashSet::new(); + if let Some(with) = &query.with { + for cte in &with.cte_tables { + let cte_name = ParserContext::canonicalize_identifier(cte.alias.name.clone()).value; + let mut cte_query_names = HashSet::new(); + extract_tables_from_sql_query(&cte.query, &mut cte_query_names); + if with.recursive { + cte_names.insert(cte_name.clone()); + } + remove_cte_names(&mut cte_query_names, &cte_names); + names.extend(cte_query_names); + if !with.recursive { + cte_names.insert(cte_name); + } + } + } + + let mut body_names = HashSet::new(); + extract_tables_from_set_expr(&query.body, &mut body_names); + remove_cte_names(&mut body_names, &cte_names); + names.extend(body_names); +} + /// Helper function for [extract_tables_from_query]. /// /// Handle [SetExpr]. @@ -346,14 +381,11 @@ fn extract_tables_from_set_expr(set_expr: &SetExpr, names: &mut HashSet { for from in &select.from { - table_factor_to_object_name(&from.relation, names); - for join in &from.joins { - table_factor_to_object_name(&join.relation, names); - } + extract_tables_from_table_with_joins(from, names); } } SetExpr::Query(query) => { - extract_tables_from_set_expr(&query.body, names); + extract_tables_from_sql_query(query, names); } SetExpr::SetOperation { left, right, .. } => { extract_tables_from_set_expr(left, names); @@ -363,12 +395,47 @@ fn extract_tables_from_set_expr(set_expr: &SetExpr, names: &mut HashSet, +) { + table_factor_to_object_name(&table_with_joins.relation, names); + for join in &table_with_joins.joins { + table_factor_to_object_name(&join.relation, names); + } +} + /// Helper function for [extract_tables_from_query]. /// /// Handle [TableFactor]. fn table_factor_to_object_name(table_factor: &TableFactor, names: &mut HashSet) { - if let TableFactor::Table { name, .. } = table_factor { - names.insert(name.to_owned()); + match table_factor { + TableFactor::Table { name, .. } => { + names.insert(name.to_owned()); + } + TableFactor::Derived { subquery, .. } => { + extract_tables_from_sql_query(subquery, names); + } + TableFactor::NestedJoin { + table_with_joins, .. + } => { + extract_tables_from_table_with_joins(table_with_joins, names); + } + TableFactor::Pivot { table, .. } + | TableFactor::Unpivot { table, .. } + | TableFactor::MatchRecognize { table, .. } => { + table_factor_to_object_name(table, names); + } + TableFactor::TableFunction { .. } + | TableFactor::Function { .. } + | TableFactor::UNNEST { .. } + | TableFactor::JsonTable { .. } + | TableFactor::OpenJsonTable { .. } + | TableFactor::XmlTable { .. } + | TableFactor::SemanticView { .. } => {} } } @@ -458,6 +525,91 @@ TQL EVAL (now() - '15s'::interval, now(), '5s') count_values("status_code", {__n } } + #[test] + fn test_extract_tables_from_sql_query_with_derived_join() { + let sql = r#" +CREATE FLOW flow_batch_join_subquery SINK TO flow_batch_join_sink +EVAL INTERVAL '1m' AS +SELECT a.symbol, b.mark_price +FROM ( + SELECT inst_id AS symbol, max(ts) AS mark_iv_ts + FROM flow_batch_join_opt_summary + GROUP BY inst_id +) a +LEFT JOIN ( + SELECT symbol, max(mark_price) AS mark_price + FROM flow_batch_join_market_v5 + WHERE "type" = 'OPTION_MARK' + GROUP BY symbol +) b ON a.symbol = b.symbol; +"#; + let mut stmts = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) + .unwrap(); + let Statement::CreateFlow(create_flow) = stmts.pop().unwrap() else { + unreachable!() + }; + + let mut tables = extract_tables_from_query(&create_flow.query) + .map(|table| format_raw_object_name(&table)) + .collect_vec(); + tables.sort(); + assert_eq!( + vec![ + "flow_batch_join_market_v5".to_string(), + "flow_batch_join_opt_summary".to_string(), + ], + tables + ); + } + + #[test] + fn test_extract_tables_from_sql_query_with_cte_scopes() { + let testcases = vec![ + ( + r#" +WITH source AS ( + SELECT * FROM source +) +SELECT * FROM source; +"#, + vec!["source".to_string()], + ), + ( + r#" +WITH first_cte AS ( + SELECT * FROM physical_source +), second_cte AS ( + SELECT * FROM first_cte +) +SELECT * FROM second_cte; +"#, + vec!["physical_source".to_string()], + ), + ]; + + for (sql, expected_tables) in testcases { + let mut stmts = ParserContext::create_with_dialect( + sql, + &GreptimeDbDialect {}, + ParseOptions::default(), + ) + .unwrap(); + let Statement::Query(query) = stmts.pop().unwrap() else { + unreachable!() + }; + + let mut tables = HashSet::new(); + extract_tables_from_sql_query(&query.inner, &mut tables); + let mut tables = tables + .into_iter() + .map(|table| format_raw_object_name(&table)) + .collect_vec(); + tables.sort(); + assert_eq!(expected_tables, tables); + } + } + #[test] fn test_extract_tables_from_tql_query_with_schema_matcher() { let sql = r#" diff --git a/tests/cases/standalone/common/flow/flow_batch_join_subquery.result b/tests/cases/standalone/common/flow/flow_batch_join_subquery.result new file mode 100644 index 0000000000..0d590cbdbc --- /dev/null +++ b/tests/cases/standalone/common/flow/flow_batch_join_subquery.result @@ -0,0 +1,130 @@ +CREATE DATABASE flow_join_fixture; + +Affected Rows: 1 + +CREATE TABLE flow_join_fixture."left_samples" ( + source_id STRING, + left_value DOUBLE, + event_ts TIMESTAMP, + observed_at TIMESTAMP TIME INDEX +); + +Affected Rows: 0 + +CREATE TABLE flow_join_fixture."right_samples" ( + source_id STRING, + right_value DOUBLE, + sample_kind STRING, + event_ts TIMESTAMP, + observed_at TIMESTAMP TIME INDEX +); + +Affected Rows: 0 + +-- Verify batching flow creation accepts aggregate subqueries joined by LEFT JOIN. +CREATE FLOW flow_batch_join_subquery SINK TO flow_batch_join_sink +EVAL INTERVAL '5m' AS +SELECT + l.source_id, + l.measure_name, + l.bucket_time, + l.left_event_ts, + l.left_value, + r.right_event_ts, + r.right_value +FROM ( + SELECT + source_id, + 'sample' AS measure_name, + date_trunc('minute', now()) AS bucket_time, + max(event_ts) AS left_event_ts, + last_value(left_value ORDER BY observed_at) AS left_value + FROM + flow_join_fixture."left_samples" + WHERE + observed_at BETWEEN date_trunc('minute', now()) - INTERVAL '5 minutes' + AND date_trunc('minute', now()) + GROUP BY + source_id +) l +LEFT JOIN ( + SELECT + source_id, + 'sample' AS measure_name, + date_trunc('minute', now()) AS bucket_time, + max(event_ts) AS right_event_ts, + last_value(right_value ORDER BY observed_at) AS right_value + FROM + flow_join_fixture."right_samples" + WHERE + observed_at BETWEEN date_trunc('minute', now()) - INTERVAL '5 minutes' + AND date_trunc('minute', now()) + AND sample_kind = 'primary' + GROUP BY + source_id +) r ON l.source_id = r.source_id AND l.bucket_time = r.bucket_time; + +Affected Rows: 0 + +SELECT + source_table_names LIKE '%left_samples%' AS has_left_source, + source_table_names LIKE '%right_samples%' AS has_right_source, + options LIKE '%"flow_type":"batching"%' AS is_batching_flow +FROM + INFORMATION_SCHEMA.FLOWS +WHERE + flow_name = 'flow_batch_join_subquery'; + ++-----------------+------------------+------------------+ +| has_left_source | has_right_source | is_batching_flow | ++-----------------+------------------+------------------+ +| true | true | true | ++-----------------+------------------+------------------+ + +INSERT INTO flow_join_fixture."left_samples" VALUES + ('source-a', 0.12, date_trunc('minute', now()), date_trunc('minute', now())); + +Affected Rows: 1 + +INSERT INTO flow_join_fixture."right_samples" VALUES + ('source-a', 100.5, 'primary', date_trunc('minute', now()), date_trunc('minute', now())); + +Affected Rows: 1 + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('flow_batch_join_subquery'); + ++----------------------------------------------+ +| ADMIN FLUSH_FLOW('flow_batch_join_subquery') | ++----------------------------------------------+ +| FLOW_FLUSHED | ++----------------------------------------------+ + +SELECT source_id, measure_name, left_value, right_value FROM flow_batch_join_sink ORDER BY source_id; + ++-----------+--------------+------------+-------------+ +| source_id | measure_name | left_value | right_value | ++-----------+--------------+------------+-------------+ +| source-a | sample | 0.12 | 100.5 | ++-----------+--------------+------------+-------------+ + +DROP FLOW flow_batch_join_subquery; + +Affected Rows: 0 + +DROP TABLE flow_batch_join_sink; + +Affected Rows: 0 + +DROP TABLE flow_join_fixture."left_samples"; + +Affected Rows: 0 + +DROP TABLE flow_join_fixture."right_samples"; + +Affected Rows: 0 + +DROP DATABASE flow_join_fixture; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/flow/flow_batch_join_subquery.sql b/tests/cases/standalone/common/flow/flow_batch_join_subquery.sql new file mode 100644 index 0000000000..f37aafdf4f --- /dev/null +++ b/tests/cases/standalone/common/flow/flow_batch_join_subquery.sql @@ -0,0 +1,85 @@ +CREATE DATABASE flow_join_fixture; + +CREATE TABLE flow_join_fixture."left_samples" ( + source_id STRING, + left_value DOUBLE, + event_ts TIMESTAMP, + observed_at TIMESTAMP TIME INDEX +); + +CREATE TABLE flow_join_fixture."right_samples" ( + source_id STRING, + right_value DOUBLE, + sample_kind STRING, + event_ts TIMESTAMP, + observed_at TIMESTAMP TIME INDEX +); + +-- Verify batching flow creation accepts aggregate subqueries joined by LEFT JOIN. +CREATE FLOW flow_batch_join_subquery SINK TO flow_batch_join_sink +EVAL INTERVAL '5m' AS +SELECT + l.source_id, + l.measure_name, + l.bucket_time, + l.left_event_ts, + l.left_value, + r.right_event_ts, + r.right_value +FROM ( + SELECT + source_id, + 'sample' AS measure_name, + date_trunc('minute', now()) AS bucket_time, + max(event_ts) AS left_event_ts, + last_value(left_value ORDER BY observed_at) AS left_value + FROM + flow_join_fixture."left_samples" + WHERE + observed_at BETWEEN date_trunc('minute', now()) - INTERVAL '5 minutes' + AND date_trunc('minute', now()) + GROUP BY + source_id +) l +LEFT JOIN ( + SELECT + source_id, + 'sample' AS measure_name, + date_trunc('minute', now()) AS bucket_time, + max(event_ts) AS right_event_ts, + last_value(right_value ORDER BY observed_at) AS right_value + FROM + flow_join_fixture."right_samples" + WHERE + observed_at BETWEEN date_trunc('minute', now()) - INTERVAL '5 minutes' + AND date_trunc('minute', now()) + AND sample_kind = 'primary' + GROUP BY + source_id +) r ON l.source_id = r.source_id AND l.bucket_time = r.bucket_time; + +SELECT + source_table_names LIKE '%left_samples%' AS has_left_source, + source_table_names LIKE '%right_samples%' AS has_right_source, + options LIKE '%"flow_type":"batching"%' AS is_batching_flow +FROM + INFORMATION_SCHEMA.FLOWS +WHERE + flow_name = 'flow_batch_join_subquery'; + +INSERT INTO flow_join_fixture."left_samples" VALUES + ('source-a', 0.12, date_trunc('minute', now()), date_trunc('minute', now())); + +INSERT INTO flow_join_fixture."right_samples" VALUES + ('source-a', 100.5, 'primary', date_trunc('minute', now()), date_trunc('minute', now())); + +-- SQLNESS REPLACE (ADMIN\sFLUSH_FLOW\('\w+'\)\s+\|\n\+-+\+\n\|\s+)[0-9]+\s+\| $1 FLOW_FLUSHED | +ADMIN FLUSH_FLOW('flow_batch_join_subquery'); + +SELECT source_id, measure_name, left_value, right_value FROM flow_batch_join_sink ORDER BY source_id; + +DROP FLOW flow_batch_join_subquery; +DROP TABLE flow_batch_join_sink; +DROP TABLE flow_join_fixture."left_samples"; +DROP TABLE flow_join_fixture."right_samples"; +DROP DATABASE flow_join_fixture; From da9d314fa4329854bb4dd17e09c98e91906c2603 Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com> Date: Thu, 4 Jun 2026 10:55:09 +0800 Subject: [PATCH 25/29] fix: align bulk insert schema handling (#8222) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: align bulk insert schema handling Carry `aligned_schema_version` through bulk insert requests so datanode can recheck stale schemas only when needed. Use the physical data region schema version for metric logical bulk inserts, while external bulk insert callers leave alignment unknown. Related files: - `Cargo.toml` - `Cargo.lock` - `src/store-api/src/region_request.rs` - `src/metric-engine/src/engine/bulk_insert.rs` - `src/mito2/src/request.rs` - `src/mito2/src/worker/handle_bulk_insert.rs` - `src/mito2/src/worker/handle_write.rs` - `src/operator/src/bulk_insert.rs` - `src/servers/src/pending_rows_batcher.rs` - `src/mito2/src/engine/edit_region_test.rs` Signed-off-by: Lei, HUANG * test: cover bulk insert missing columns Add Flight bulk insert coverage for batches that omit nullable columns and verify the database fills the missing values with nulls. Related files: - `tests-integration/src/grpc/flight.rs` Signed-off-by: Lei, HUANG * chore: update proto to main branch commit Signed-off-by: Lei, HUANG * fix: align physical metric bulk insert schema Pass the physical schema version through metric bulk insert requests so physical and logical region writes share the same schema alignment behavior. Files: - \`src/metric-engine/src/engine/bulk_insert.rs\` — updates \`bulk_insert_physical_region\` and reuses \`physical_schema_version\` for bulk insert schema alignment. Signed-off-by: Lei, HUANG * chore: add some doc Signed-off-by: Lei, HUANG --------- Signed-off-by: Lei, HUANG --- Cargo.lock | 2 +- Cargo.toml | 2 +- src/metric-engine/src/engine/bulk_insert.rs | 22 ++++++- src/mito2/src/engine/edit_region_test.rs | 1 + src/mito2/src/request.rs | 2 +- src/mito2/src/worker/handle_bulk_insert.rs | 11 +++- src/mito2/src/worker/handle_write.rs | 6 +- src/operator/src/bulk_insert.rs | 2 + src/servers/src/pending_rows_batcher.rs | 4 ++ src/store-api/src/region_request.rs | 3 + tests-integration/src/grpc/flight.rs | 64 +++++++++++++++++++++ 11 files changed, 111 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2485a5ceec..e3d0dfb69e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5918,7 +5918,7 @@ dependencies = [ [[package]] name = "greptime-proto" version = "0.1.0" -source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=7224c2ad6d11db612fbdb621c36135fc37ffce35#7224c2ad6d11db612fbdb621c36135fc37ffce35" +source = "git+https://github.com/GreptimeTeam/greptime-proto.git?rev=912f6c21017d5ab6c529568d36e7b8c30a94d84b#912f6c21017d5ab6c529568d36e7b8c30a94d84b" dependencies = [ "prost 0.14.1", "prost-types 0.14.1", diff --git a/Cargo.toml b/Cargo.toml index ee8d4dcf11..47b04be366 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -158,7 +158,7 @@ fs2 = "0.4" fst = "0.4.7" futures = "0.3" futures-util = "0.3" -greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "7224c2ad6d11db612fbdb621c36135fc37ffce35" } +greptime-proto = { git = "https://github.com/GreptimeTeam/greptime-proto.git", rev = "912f6c21017d5ab6c529568d36e7b8c30a94d84b" } hex = "0.4" http = "1" humantime = "2.1" diff --git a/src/metric-engine/src/engine/bulk_insert.rs b/src/metric-engine/src/engine/bulk_insert.rs index 6d33d39149..d700d858ac 100644 --- a/src/metric-engine/src/engine/bulk_insert.rs +++ b/src/metric-engine/src/engine/bulk_insert.rs @@ -18,9 +18,10 @@ use api::v1::{ArrowIpc, SemanticType}; use bytes::Bytes; use common_grpc::flight::{FlightEncoder, FlightMessage}; use datatypes::arrow::record_batch::RecordBatch; -use snafu::{OptionExt, ensure}; +use snafu::{OptionExt, ResultExt, ensure}; use store_api::codec::PrimaryKeyEncoding; use store_api::metadata::RegionMetadataRef; +use store_api::region_engine::RegionEngine; use store_api::region_request::{AffectedRows, RegionBulkInsertsRequest, RegionRequest}; use store_api::storage::RegionId; @@ -71,8 +72,11 @@ impl MetricEngineInner { async fn bulk_insert_physical_region( &self, region_id: RegionId, - request: RegionBulkInsertsRequest, + mut request: RegionBulkInsertsRequest, ) -> Result { + // Simply set the aligned schema to the data region schema version to avoid filling missing columns + // because that schema should be constant and callers have ensured request has the same schema. + request.aligned_schema_version = Some(self.physical_schema_version(region_id).await?); self.data_region .write_data(region_id, RegionRequest::BulkInserts(request)) .await @@ -117,6 +121,8 @@ impl MetricEngineInner { let (schema, data_header, payload) = record_batch_to_ipc(&modified_batch)?; let partition_expr_version = request.partition_expr_version; + let aligned_schema_version = Some(self.physical_schema_version(data_region_id).await?); + let request = RegionBulkInsertsRequest { region_id: data_region_id, payload: modified_batch, @@ -126,12 +132,22 @@ impl MetricEngineInner { payload, }, partition_expr_version, + aligned_schema_version, }; self.data_region .write_data(data_region_id, RegionRequest::BulkInserts(request)) .await } + async fn physical_schema_version(&self, region_id: RegionId) -> Result { + Ok(self + .mito + .get_metadata(region_id) + .await + .context(error::MitoReadOperationSnafu)? + .schema_version) + } + fn resolve_tag_columns_from_metadata( &self, logical_region_id: RegionId, @@ -282,6 +298,7 @@ mod tests { payload, }, partition_expr_version: None, + aligned_schema_version: None, }) } @@ -319,6 +336,7 @@ mod tests { payload: batch, raw_data: ArrowIpc::default(), partition_expr_version: None, + aligned_schema_version: None, }); let response = env .metric() diff --git a/src/mito2/src/engine/edit_region_test.rs b/src/mito2/src/engine/edit_region_test.rs index 1d9bf2101b..214060acf8 100644 --- a/src/mito2/src/engine/edit_region_test.rs +++ b/src/mito2/src/engine/edit_region_test.rs @@ -652,6 +652,7 @@ fn build_bulk_insert_request( payload: record_batch.data_body, }, partition_expr_version: None, + aligned_schema_version: None, } } diff --git a/src/mito2/src/request.rs b/src/mito2/src/request.rs index 8ae05177da..7e071965f2 100644 --- a/src/mito2/src/request.rs +++ b/src/mito2/src/request.rs @@ -552,7 +552,7 @@ pub(crate) struct SenderBulkRequest { pub(crate) sender: OptionOutputTx, pub(crate) region_id: RegionId, pub(crate) request: BulkPart, - pub(crate) region_metadata: RegionMetadataRef, + pub(crate) region_metadata: Option, pub(crate) partition_expr_version: Option, } diff --git a/src/mito2/src/worker/handle_bulk_insert.rs b/src/mito2/src/worker/handle_bulk_insert.rs index a413271db3..b4a7ae4a18 100644 --- a/src/mito2/src/worker/handle_bulk_insert.rs +++ b/src/mito2/src/worker/handle_bulk_insert.rs @@ -86,11 +86,20 @@ impl RegionWorkerLoop { timestamp_index: ts_index, raw_data: Some(request.raw_data), }; + + let aligned_region_metadata = if request + .aligned_schema_version + .is_some_and(|aligned| aligned == region_metadata.schema_version) + { + Some(region_metadata) + } else { + None + }; pending_bulk_request.push(SenderBulkRequest { sender, request: part, region_id: request.region_id, - region_metadata, + region_metadata: aligned_region_metadata, partition_expr_version: request.partition_expr_version, }); } diff --git a/src/mito2/src/worker/handle_write.rs b/src/mito2/src/worker/handle_write.rs index 721415411e..64fdd33edc 100644 --- a/src/mito2/src/worker/handle_write.rs +++ b/src/mito2/src/worker/handle_write.rs @@ -525,8 +525,10 @@ impl RegionWorkerLoop { } // Double-check the request schema - let need_fill_missing_columns = region_ctx.version().metadata.schema_version - != bulk_req.region_metadata.schema_version; + let need_fill_missing_columns = + !bulk_req.region_metadata.is_some_and(|aligned_schema| { + aligned_schema.schema_version == region_ctx.version().metadata.schema_version + }); // Fill missing columns if needed if need_fill_missing_columns diff --git a/src/operator/src/bulk_insert.rs b/src/operator/src/bulk_insert.rs index 38ced30135..2dbec530eb 100644 --- a/src/operator/src/bulk_insert.rs +++ b/src/operator/src/bulk_insert.rs @@ -106,6 +106,7 @@ impl Inserter { region_id: region_id.as_u64(), partition_expr_version: partition_expr_version .map(|value| PartitionExprVersion { value }), + aligned_schema_version: None, body: Some(bulk_insert_request::Body::ArrowIpc(ArrowIpc { schema: schema_bytes.clone(), data_header: raw_flight_data.data_header, @@ -220,6 +221,7 @@ impl Inserter { region_id: region_id.as_u64(), partition_expr_version: partition_expr_version .map(|value| PartitionExprVersion { value }), + aligned_schema_version: None, body: Some(bulk_insert_request::Body::ArrowIpc(ArrowIpc { schema: schema_bytes, data_header: header, diff --git a/src/servers/src/pending_rows_batcher.rs b/src/servers/src/pending_rows_batcher.rs index 307311a4e2..b800ffa05b 100644 --- a/src/servers/src/pending_rows_batcher.rs +++ b/src/servers/src/pending_rows_batcher.rs @@ -1380,6 +1380,7 @@ fn transform_logical_batches_to_physical( let modified = { let start = Instant::now(); + // The schema of modified batch is: __primary_key, timestamp, value, other partition columns... let batch = modify_batch_sparse( batch.clone(), table_id, @@ -1535,6 +1536,9 @@ fn encode_region_write_requests( body: Some(region_request::Body::BulkInsert(BulkInsertRequest { region_id: region_id.as_u64(), partition_expr_version: None, + // Set aligned_schema_version to None so that datanode will check the batch schema again to see if any + // column is missing. + aligned_schema_version: None, body: Some(bulk_insert_request::Body::ArrowIpc(ArrowIpc { schema: schema_bytes, data_header, diff --git a/src/store-api/src/region_request.rs b/src/store-api/src/region_request.rs index abf96736ec..8ab2e3abd2 100644 --- a/src/store-api/src/region_request.rs +++ b/src/store-api/src/region_request.rs @@ -409,6 +409,7 @@ fn make_region_truncate(truncate: TruncateRequest) -> Result Result> { let region_id = request.region_id.into(); let partition_expr_version = request.partition_expr_version.map(|v| v.value); + let aligned_schema_version = request.aligned_schema_version.map(|v| v.schema_version); let Some(Body::ArrowIpc(request)) = request.body else { return Ok(vec![]); }; @@ -429,6 +430,7 @@ fn make_region_bulk_inserts(request: BulkInsertRequest) -> Result, + pub aligned_schema_version: Option, } impl RegionBulkInsertsRequest { diff --git a/tests-integration/src/grpc/flight.rs b/tests-integration/src/grpc/flight.rs index d638926fff..d9bf15a64c 100644 --- a/tests-integration/src/grpc/flight.rs +++ b/tests-integration/src/grpc/flight.rs @@ -77,6 +77,35 @@ mod test { query_and_expect(db.frontend().as_ref(), sql, expected).await; } + #[tokio::test(flavor = "multi_thread")] + async fn test_standalone_flight_do_put_missing_nullable_columns() { + common_telemetry::init_default_ut_logging(); + + let (db, server) = setup_grpc_server( + StorageType::File, + "test_standalone_flight_do_put_missing_nullable_columns", + ) + .await; + let addr = server.bind_addr().unwrap().to_string(); + + let client = Client::with_urls(vec![addr]); + let client = Database::new_with_dbname("greptime-public", client); + + create_table(&client).await; + + let record_batches = create_record_batches_without_nullable_column(1); + test_put_record_batches(&client, record_batches).await; + + let sql = "select count(*) from foo where `B` is null"; + let expected = "\ ++----------+ +| count(*) | ++----------+ +| 9 | ++----------+"; + query_and_expect(db.frontend().as_ref(), sql, expected).await; + } + #[tokio::test(flavor = "multi_thread")] async fn test_distributed_flight_do_put() { common_telemetry::init_default_ut_logging(); @@ -281,6 +310,41 @@ mod test { assert_eq!(requests_count + 1, responses_count); } + fn create_record_batches_without_nullable_column(start: i64) -> Vec { + let schema = Arc::new(Schema::new(vec![ + ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ) + .with_time_index(true), + ColumnSchema::new("a", ConcreteDataType::int32_datatype(), false), + ])); + + let mut record_batches = Vec::with_capacity(3); + for chunk in &(start..start + 9).chunks(3) { + let vs = chunk.collect_vec(); + let x1 = vs[0]; + let x2 = vs[1]; + let x3 = vs[2]; + + record_batches.push( + RecordBatch::new( + schema.clone(), + vec![ + Arc::new(TimestampMillisecondVector::from_vec(vec![x1, x2, x3])) + as VectorRef, + Arc::new(Int32Vector::from_vec(vec![ + -x1 as i32, -x2 as i32, -x3 as i32, + ])), + ], + ) + .unwrap(), + ); + } + record_batches + } + fn create_record_batches(start: i64) -> Vec { let schema = Arc::new(Schema::new(vec![ ColumnSchema::new( From 3bd915865a4d5db7f40a9e56cefafd169e160875 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Thu, 4 Jun 2026 14:10:38 +0800 Subject: [PATCH 26/29] fix: improve remote WAL replay checkpoint handling (#8225) * fix: correct topic_latest_entry_id Signed-off-by: WenyXu * fix: use pruned wal id for replay checkpoint Signed-off-by: WenyXu * feat: persist remote wal checkpoints periodically Signed-off-by: WenyXu * test: cover remote wal topic latest entry id Signed-off-by: WenyXu * chore: apply suggestions Signed-off-by: WenyXu * fix: merge pruned wal id for migration checkpoint Signed-off-by: WenyXu --------- Signed-off-by: WenyXu --- src/common/meta/src/key/topic_name.rs | 34 +- src/common/meta/src/key/topic_region.rs | 80 ++++ src/datanode/src/utils.rs | 77 +++- .../src/procedure/region_migration.rs | 43 ++- .../upgrade_candidate_region.rs | 135 ++++++- src/meta-srv/src/region/flush_trigger.rs | 344 +++++++++++++++--- src/mito2/src/engine/flush_test.rs | 239 +++++++++++- src/mito2/src/region/opener.rs | 23 +- 8 files changed, 886 insertions(+), 89 deletions(-) diff --git a/src/common/meta/src/key/topic_name.rs b/src/common/meta/src/key/topic_name.rs index 99ae631a72..79454c7cd2 100644 --- a/src/common/meta/src/key/topic_name.rs +++ b/src/common/meta/src/key/topic_name.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; use std::fmt::{self, Display}; use serde::{Deserialize, Serialize}; @@ -27,7 +28,7 @@ use crate::key::{ use crate::kv_backend::KvBackendRef; use crate::kv_backend::txn::{Txn, TxnOp}; use crate::rpc::KeyValue; -use crate::rpc::store::{BatchPutRequest, RangeRequest}; +use crate::rpc::store::{BatchGetRequest, BatchPutRequest, RangeRequest}; #[derive(Debug, Clone, PartialEq)] pub struct TopicNameKey<'a> { @@ -205,6 +206,25 @@ impl TopicNameManager { .transpose() } + /// Batch get values for specific topics. + pub async fn batch_get( + &self, + topics: Vec>, + ) -> Result> { + let raw_keys = topics.iter().map(|key| key.to_bytes()).collect::>(); + let req = BatchGetRequest { keys: raw_keys }; + let resp = self.kv_backend.batch_get(req).await?; + + resp.kvs + .into_iter() + .map(|kv| { + let key = TopicNameKey::from_bytes(&kv.key)?; + let value = TopicNameValue::try_from_raw_value(&kv.value)?; + Ok((key.topic.to_string(), value)) + }) + .collect::>>() + } + /// Update the topic name key and value in the kv backend. pub async fn update( &self, @@ -295,5 +315,17 @@ mod tests { let err = manager.update(topic, 3, Some(value)).await.unwrap_err(); assert_matches!(err, error::Error::Unexpected { .. }); } + + let batch_topics = topics + .iter() + .take(2) + .map(|topic| TopicNameKey::new(topic)) + .chain(std::iter::once(TopicNameKey::new("missing-topic"))) + .collect::>(); + let values = manager.batch_get(batch_topics).await.unwrap(); + assert_eq!(values.len(), 2); + for topic in topics.iter().take(2) { + assert_eq!(values.get(topic).unwrap().pruned_entry_id, 1); + } } } diff --git a/src/common/meta/src/key/topic_region.rs b/src/common/meta/src/key/topic_region.rs index 9feac10804..e7db23cfc2 100644 --- a/src/common/meta/src/key/topic_region.rs +++ b/src/common/meta/src/key/topic_region.rs @@ -126,6 +126,34 @@ impl ReplayCheckpoint { metadata_entry_id, } } + + /// Merges the checkpoint with the topic pruned entry id. + pub fn merge_with_topic_pruned_entry_id( + checkpoint: Option, + pruned_entry_id: Option, + is_metric_engine: bool, + ) -> Option { + match (checkpoint, pruned_entry_id) { + (Some(checkpoint), Some(pruned_entry_id)) => Some(Self { + entry_id: checkpoint.entry_id.max(pruned_entry_id), + metadata_entry_id: if is_metric_engine { + Some( + checkpoint + .metadata_entry_id + .unwrap_or_default() + .max(pruned_entry_id), + ) + } else { + checkpoint.metadata_entry_id + }, + }), + (None, Some(pruned_entry_id)) => Some(Self { + entry_id: pruned_entry_id, + metadata_entry_id: is_metric_engine.then_some(pruned_entry_id), + }), + (checkpoint, None) => checkpoint, + } + } } impl TopicRegionValue { @@ -369,6 +397,58 @@ mod tests { use super::*; use crate::kv_backend::memory::MemoryKvBackend; + #[test] + fn test_merge_checkpoint_with_topic_pruned_entry_id_missing_pruned() { + let checkpoint = Some(ReplayCheckpoint::new(10, None)); + + assert_eq!( + ReplayCheckpoint::merge_with_topic_pruned_entry_id(checkpoint, None, true), + checkpoint + ); + } + + #[test] + fn test_merge_checkpoint_with_topic_pruned_entry_id_creates_checkpoint() { + assert_eq!( + ReplayCheckpoint::merge_with_topic_pruned_entry_id(None, Some(10), true), + Some(ReplayCheckpoint::new(10, Some(10))) + ); + } + + #[test] + fn test_merge_checkpoint_with_topic_pruned_entry_id_updates_both_ids() { + let checkpoint = ReplayCheckpoint::new(10, Some(5)); + + assert_eq!( + ReplayCheckpoint::merge_with_topic_pruned_entry_id(Some(checkpoint), Some(20), true), + Some(ReplayCheckpoint::new(20, Some(20))) + ); + } + + #[test] + fn test_merge_checkpoint_with_topic_pruned_entry_id_preserves_larger_ids() { + let checkpoint = ReplayCheckpoint::new(30, Some(40)); + + assert_eq!( + ReplayCheckpoint::merge_with_topic_pruned_entry_id(Some(checkpoint), Some(20), true), + Some(checkpoint) + ); + } + + #[test] + fn test_merge_checkpoint_with_topic_pruned_entry_id_for_mito() { + assert_eq!( + ReplayCheckpoint::merge_with_topic_pruned_entry_id(None, Some(10), false), + Some(ReplayCheckpoint::new(10, None)) + ); + + let checkpoint = ReplayCheckpoint::new(5, Some(8)); + assert_eq!( + ReplayCheckpoint::merge_with_topic_pruned_entry_id(Some(checkpoint), Some(10), false), + Some(ReplayCheckpoint::new(10, Some(8))) + ); + } + #[tokio::test] async fn test_topic_region_manager() { let kv_backend = Arc::new(MemoryKvBackend::default()); diff --git a/src/datanode/src/utils.rs b/src/datanode/src/utils.rs index 816ae021ba..88f4f1aa60 100644 --- a/src/datanode/src/utils.rs +++ b/src/datanode/src/utils.rs @@ -16,11 +16,16 @@ use std::collections::HashMap; use common_meta::DatanodeId; use common_meta::key::datanode_table::DatanodeTableManager; -use common_meta::key::topic_region::{TopicRegionKey, TopicRegionManager, TopicRegionValue}; +use common_meta::key::topic_name::{TopicNameKey, TopicNameManager, TopicNameValue}; +use common_meta::key::topic_region::{ + ReplayCheckpoint as MetadataReplayCheckpoint, TopicRegionKey, TopicRegionManager, + TopicRegionValue, +}; use common_meta::kv_backend::KvBackendRef; use common_meta::wal_provider::{extract_topic_from_wal_options, prepare_wal_options}; use futures::TryStreamExt; use snafu::ResultExt; +use store_api::metric_engine_consts::METRIC_ENGINE_NAME; use store_api::path_utils::table_dir; use store_api::region_request::{PathType, RegionOpenRequest, ReplayCheckpoint}; use store_api::storage::{RegionId, RegionNumber}; @@ -63,14 +68,45 @@ fn group_region_by_topic( } } +fn region_pruned_entry_ids( + topic_regions: &HashMap>, + topic_name_values: &HashMap, +) -> HashMap { + topic_regions + .iter() + .flat_map(|(topic, region_ids)| { + topic_name_values + .get(topic) + .into_iter() + .flat_map(move |value| { + region_ids + .iter() + .map(move |region_id| (*region_id, value.pruned_entry_id)) + }) + }) + .collect() +} + fn get_replay_checkpoint( region_id: RegionId, topic_region_values: &Option>, + pruned_entry_id: Option, + is_metric_engine: bool, ) -> Option { - let topic_region_values = topic_region_values.as_ref()?; - let topic_region_value = topic_region_values.get(®ion_id); - let replay_checkpoint = topic_region_value.and_then(|value| value.checkpoint); - replay_checkpoint.map(|checkpoint| ReplayCheckpoint { + let checkpoint = topic_region_values + .as_ref() + .and_then(|values| values.get(®ion_id)) + .and_then(|value| value.checkpoint) + .map(|checkpoint| { + MetadataReplayCheckpoint::new(checkpoint.entry_id, checkpoint.metadata_entry_id) + }); + + MetadataReplayCheckpoint::merge_with_topic_pruned_entry_id( + checkpoint, + pruned_entry_id, + is_metric_engine, + ) + .map(|checkpoint| ReplayCheckpoint { entry_id: checkpoint.entry_id, metadata_entry_id: checkpoint.metadata_entry_id, }) @@ -88,7 +124,8 @@ pub async fn build_region_open_requests( .await .context(GetMetadataSnafu)?; - let topic_region_manager = TopicRegionManager::new(kv_backend); + let topic_region_manager = TopicRegionManager::new(kv_backend.clone()); + let topic_name_manager = TopicNameManager::new(kv_backend); let mut topic_regions = HashMap::>::new(); let mut regions = vec![]; #[cfg(feature = "enterprise")] @@ -161,10 +198,36 @@ pub async fn build_region_open_requests( None }; + let topic_name_values = if !topic_regions.is_empty() { + let topics = topic_regions + .keys() + .map(|topic| TopicNameKey::new(topic)) + .collect::>(); + Some( + topic_name_manager + .batch_get(topics) + .await + .context(GetMetadataSnafu)?, + ) + } else { + None + }; + let region_pruned_entry_ids = topic_name_values + .as_ref() + .map(|values| region_pruned_entry_ids(&topic_regions, values)); + let mut leader_region_requests = Vec::with_capacity(regions.len()); for (region_id, engine, store_path, options) in regions { let table_dir = table_dir(&store_path, region_id.table_id()); - let checkpoint = get_replay_checkpoint(region_id, &topic_region_values); + let pruned_entry_id = region_pruned_entry_ids + .as_ref() + .and_then(|values| values.get(®ion_id).copied()); + let checkpoint = get_replay_checkpoint( + region_id, + &topic_region_values, + pruned_entry_id, + engine == METRIC_ENGINE_NAME, + ); info!("region_id: {}, checkpoint: {:?}", region_id, checkpoint); leader_region_requests.push(( region_id, diff --git a/src/meta-srv/src/procedure/region_migration.rs b/src/meta-srv/src/procedure/region_migration.rs index 6563da63bc..c0ab352fc6 100644 --- a/src/meta-srv/src/procedure/region_migration.rs +++ b/src/meta-srv/src/procedure/region_migration.rs @@ -39,6 +39,7 @@ use common_meta::ddl::RegionFailureDetectorControllerRef; use common_meta::instruction::CacheIdent; use common_meta::key::datanode_table::{DatanodeTableKey, DatanodeTableValue}; use common_meta::key::table_route::TableRouteValue; +use common_meta::key::topic_name::TopicNameKey; use common_meta::key::topic_region::{ReplayCheckpoint, TopicRegionKey}; use common_meta::key::{DeserializedValueWithBytes, TableMetadataManagerRef}; use common_meta::kv_backend::{KvBackendRef, ResettableKvBackendRef}; @@ -661,11 +662,15 @@ impl Context { .await; } - /// Fetches the replay checkpoints for the given topic region keys. - pub async fn get_replay_checkpoints( + /// Fetches replay checkpoints and merges them with topic pruned entry ids. + pub async fn get_replay_checkpoints_with_topic_pruned_entry_ids( &self, - topic_region_keys: Vec>, + region_topics: &[(RegionId, String, bool)], ) -> Result> { + let topic_region_keys = region_topics + .iter() + .map(|(region_id, topic, _)| TopicRegionKey::new(*region_id, topic)) + .collect::>(); let topic_region_values = self .table_metadata_manager .topic_region_manager() @@ -673,9 +678,37 @@ impl Context { .await .context(error::TableMetadataManagerSnafu)?; - let replay_checkpoints = topic_region_values + let topic_name_keys = region_topics + .iter() + .map(|(_, topic, _)| topic.as_str()) + .collect::>() .into_iter() - .flat_map(|(key, value)| value.checkpoint.map(|value| (key, value))) + .map(TopicNameKey::new) + .collect::>(); + let topic_name_values = self + .table_metadata_manager + .topic_name_manager() + .batch_get(topic_name_keys) + .await + .context(error::TableMetadataManagerSnafu)?; + + let replay_checkpoints = region_topics + .iter() + .filter_map(|(region_id, topic, is_metric_engine)| { + let checkpoint = topic_region_values + .get(region_id) + .and_then(|value| value.checkpoint); + let pruned_entry_id = topic_name_values + .get(topic) + .map(|value| value.pruned_entry_id); + + ReplayCheckpoint::merge_with_topic_pruned_entry_id( + checkpoint, + pruned_entry_id, + *is_metric_engine, + ) + .map(|checkpoint| (*region_id, checkpoint)) + }) .collect::>(); Ok(replay_checkpoints) diff --git a/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs b/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs index 4c60215cf7..2ffd64bb47 100644 --- a/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs +++ b/src/meta-srv/src/procedure/region_migration/upgrade_candidate_region.rs @@ -21,7 +21,6 @@ use common_meta::ddl::utils::parse_region_wal_options; use common_meta::instruction::{ Instruction, InstructionReply, UpgradeRegion, UpgradeRegionReply, UpgradeRegionsReply, }; -use common_meta::key::topic_region::TopicRegionKey; use common_meta::lock_key::RemoteWalLock; use common_meta::wal_provider::extract_topic_from_wal_options; use common_procedure::{Context as ProcedureContext, Status}; @@ -30,6 +29,7 @@ use common_telemetry::{error, info}; use common_wal::options::WalOptions; use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, ensure}; +use store_api::metric_engine_consts::METRIC_ENGINE_NAME; use tokio::time::{Instant, sleep}; use crate::error::{self, Result}; @@ -133,17 +133,14 @@ impl UpgradeCandidateRegion { &datanode_table_value.region_info.region_wal_options, ) { - region_topic.push((*region_id, topic)); + let is_metric_engine = + datanode_table_value.region_info.engine == METRIC_ENGINE_NAME; + region_topic.push((*region_id, topic, is_metric_engine)); } } let replay_checkpoints = ctx - .get_replay_checkpoints( - region_topic - .iter() - .map(|(region_id, topic)| TopicRegionKey::new(*region_id, topic)) - .collect(), - ) + .get_replay_checkpoints_with_topic_pruned_entry_ids(®ion_topic) .await?; // Build upgrade regions instruction. let mut upgrade_regions = Vec::with_capacity(region_ids.len()); @@ -358,8 +355,11 @@ mod tests { use common_meta::key::table_route::TableRouteValue; use common_meta::key::test_utils::new_test_table_info; + use common_meta::key::topic_name::TopicNameKey; + use common_meta::key::topic_region::{ReplayCheckpoint, TopicRegionKey, TopicRegionValue}; use common_meta::peer::Peer; use common_meta::rpc::router::{Region, RegionRoute}; + use common_wal::options::KafkaWalOptions; use store_api::storage::RegionId; use super::*; @@ -382,9 +382,28 @@ mod tests { ) } + fn kafka_wal_options(topic: &str) -> HashMap { + HashMap::from([( + 1, + serde_json::to_string(&WalOptions::Kafka(KafkaWalOptions { + topic: topic.to_string(), + })) + .unwrap(), + )]) + } + async fn prepare_table_metadata(ctx: &Context, wal_options: HashMap) { + prepare_table_metadata_with_engine(ctx, wal_options, "engine").await; + } + + async fn prepare_table_metadata_with_engine( + ctx: &Context, + wal_options: HashMap, + engine: &str, + ) { let region_id = ctx.persistent_ctx.region_ids[0]; - let table_info = new_test_table_info(region_id.table_id()); + let mut table_info = new_test_table_info(region_id.table_id()); + table_info.meta.engine = engine.to_string(); let region_routes = vec![RegionRoute { region: Region::new_test(region_id), leader_peer: Some(ctx.persistent_ctx.from_peer.clone()), @@ -401,6 +420,104 @@ mod tests { .unwrap(); } + #[tokio::test] + async fn test_build_upgrade_region_instruction_merges_topic_pruned_entry_id() { + let state = UpgradeCandidateRegion::default(); + let persistent_context = new_persistent_context(); + let env = TestingEnv::new(); + let mut ctx = env.context_factory().new_context(persistent_context); + let region_id = ctx.persistent_ctx.region_ids[0]; + let topic = "test_topic"; + prepare_table_metadata(&ctx, kafka_wal_options(topic)).await; + ctx.table_metadata_manager + .topic_region_manager() + .batch_put(&[( + TopicRegionKey::new(region_id, topic), + Some(TopicRegionValue::new(Some(ReplayCheckpoint::new(10, None)))), + )]) + .await + .unwrap(); + ctx.table_metadata_manager + .topic_name_manager() + .batch_put(vec![TopicNameKey::new(topic)]) + .await + .unwrap(); + let prev = ctx + .table_metadata_manager + .topic_name_manager() + .get(topic) + .await + .unwrap(); + ctx.table_metadata_manager + .topic_name_manager() + .update(topic, 20, prev) + .await + .unwrap(); + + let instruction = state + .build_upgrade_region_instruction(&mut ctx, Duration::from_secs(1)) + .await + .unwrap(); + let Instruction::UpgradeRegions(upgrade_regions) = instruction else { + unreachable!() + }; + + assert_eq!(upgrade_regions.len(), 1); + assert_eq!(upgrade_regions[0].replay_entry_id, Some(20)); + assert_eq!(upgrade_regions[0].metadata_replay_entry_id, None); + } + + #[tokio::test] + async fn test_build_upgrade_region_instruction_merges_metric_metadata_pruned_entry_id() { + let state = UpgradeCandidateRegion::default(); + let persistent_context = new_persistent_context(); + let env = TestingEnv::new(); + let mut ctx = env.context_factory().new_context(persistent_context); + let region_id = ctx.persistent_ctx.region_ids[0]; + let topic = "test_topic"; + prepare_table_metadata_with_engine(&ctx, kafka_wal_options(topic), METRIC_ENGINE_NAME) + .await; + ctx.table_metadata_manager + .topic_region_manager() + .batch_put(&[( + TopicRegionKey::new(region_id, topic), + Some(TopicRegionValue::new(Some(ReplayCheckpoint::new( + 10, + Some(5), + )))), + )]) + .await + .unwrap(); + ctx.table_metadata_manager + .topic_name_manager() + .batch_put(vec![TopicNameKey::new(topic)]) + .await + .unwrap(); + let prev = ctx + .table_metadata_manager + .topic_name_manager() + .get(topic) + .await + .unwrap(); + ctx.table_metadata_manager + .topic_name_manager() + .update(topic, 20, prev) + .await + .unwrap(); + + let instruction = state + .build_upgrade_region_instruction(&mut ctx, Duration::from_secs(1)) + .await + .unwrap(); + let Instruction::UpgradeRegions(upgrade_regions) = instruction else { + unreachable!() + }; + + assert_eq!(upgrade_regions.len(), 1); + assert_eq!(upgrade_regions[0].replay_entry_id, Some(20)); + assert_eq!(upgrade_regions[0].metadata_replay_entry_id, Some(20)); + } + #[tokio::test] async fn test_datanode_is_unreachable() { let state = UpgradeCandidateRegion::default(); diff --git a/src/meta-srv/src/region/flush_trigger.rs b/src/meta-srv/src/region/flush_trigger.rs index cdaa4339f4..d13fb9f244 100644 --- a/src/meta-srv/src/region/flush_trigger.rs +++ b/src/meta-srv/src/region/flush_trigger.rs @@ -45,6 +45,9 @@ const TICKER_INTERVAL: Duration = Duration::from_secs(60); /// The duration of the recent period. const RECENT_DURATION: Duration = Duration::from_secs(300); +/// The interval to periodically persist region checkpoints regardless of replay size. +const PERIODIC_CHECKPOINT_PERSIST_INTERVAL: Duration = Duration::from_secs(60 * 60); + /// [`Event`] represents various types of events that can be processed by the region flush ticker. /// /// Variants: @@ -84,6 +87,8 @@ pub struct RegionFlushTrigger { flush_trigger_size: ReadableSize, /// The checkpoint trigger size. checkpoint_trigger_size: ReadableSize, + /// The last timestamp in milliseconds when a region checkpoint was persisted. + last_checkpoint_persist_millis_by_region: HashMap, /// The receiver of events. receiver: Receiver, } @@ -123,6 +128,7 @@ impl RegionFlushTrigger { server_addr, flush_trigger_size, checkpoint_trigger_size, + last_checkpoint_persist_millis_by_region: HashMap::new(), receiver: rx, }; (region_flush_trigger, region_flush_ticker) @@ -147,14 +153,15 @@ impl RegionFlushTrigger { } } - async fn handle_tick(&self) { + async fn handle_tick(&mut self) { if let Err(e) = self.trigger_flush().await { error!(e; "Failed to trigger flush"); } } - async fn trigger_flush(&self) -> Result<()> { + async fn trigger_flush(&mut self) -> Result<()> { let now = Instant::now(); + let now_millis = current_time_millis(); let topics = self .table_metadata_manager .topic_name_manager() @@ -162,17 +169,19 @@ impl RegionFlushTrigger { .await .context(error::TableMetadataManagerSnafu)?; + let mut active_region_ids = HashSet::new(); for topic in &topics { - let Some((latest_entry_id, avg_record_size)) = self.retrieve_topic_stat(topic) else { - continue; - }; if let Err(e) = self - .flush_regions_in_topic(topic, latest_entry_id, avg_record_size) + .handle_topic(topic, now_millis, &mut active_region_ids) .await { - error!(e; "Failed to flush regions in topic: {}", topic); + error!(e; "Failed to handle regions in topic: {}", topic); } } + retain_checkpoint_persist_records( + &mut self.last_checkpoint_persist_millis_by_region, + &active_region_ids, + ); debug!( "Triggered flush for {} topics in {:?}", @@ -182,6 +191,79 @@ impl RegionFlushTrigger { Ok(()) } + async fn handle_topic( + &mut self, + topic: &str, + now_millis: i64, + active_region_ids: &mut HashSet, + ) -> Result<()> { + let topic_regions = self + .table_metadata_manager + .topic_region_manager() + .regions(topic) + .await + .context(error::TableMetadataManagerSnafu)?; + + if topic_regions.is_empty() { + debug!("No regions found for topic: {}", topic); + return Ok(()); + } + active_region_ids.extend(topic_regions.keys().copied()); + + let topic_stat = self.retrieve_topic_stat(topic); + let size_based_regions = topic_stat + .map(|(latest_entry_id, avg_record_size)| { + filter_regions_by_replay_size( + topic, + topic_regions.iter().map(|(region_id, value)| { + (*region_id, value.min_entry_id().unwrap_or_default()) + }), + avg_record_size as u64, + latest_entry_id, + self.checkpoint_trigger_size, + ) + }) + .unwrap_or_default(); + // Periodic checkpoint persistence is intentionally independent of topic stats freshness: + // Kafka retention can advance even when recent write stats are unavailable. + let periodic_regions = filter_regions_for_periodic_checkpoint( + topic_regions.keys().copied(), + &self.last_checkpoint_persist_millis_by_region, + now_millis, + PERIODIC_CHECKPOINT_PERSIST_INTERVAL, + ); + let regions_to_persist = merge_region_ids(size_based_regions, periodic_regions); + let region_manifests = self + .leader_region_registry + .batch_get(topic_regions.keys().cloned()); + + match self + .persist_region_checkpoints( + topic, + ®ions_to_persist, + &topic_regions, + ®ion_manifests, + ) + .await + { + // Only mark regions that were actually written to KV. If the checkpoint is stale, + // already persisted, or the write fails, the next tick should retry. + Ok(region_ids) => mark_checkpoint_persisted( + &mut self.last_checkpoint_persist_millis_by_region, + ®ion_ids, + now_millis, + ), + Err(err) => error!(err; "Failed to persist region checkpoints for topic: {}", topic), + } + + if let Some((latest_entry_id, avg_record_size)) = topic_stat { + self.flush_regions_in_topic(topic, latest_entry_id, avg_record_size, region_manifests) + .await?; + } + + Ok(()) + } + /// Retrieves the latest entry id and average record size of a topic. /// /// Returns `None` if the topic is not found or the latest entry id is not recent. @@ -226,7 +308,7 @@ impl RegionFlushTrigger { region_ids: &[RegionId], topic_regions: &HashMap, leader_regions: &HashMap, - ) -> Result<()> { + ) -> Result> { let regions = region_ids .iter() .flat_map(|region_id| match leader_regions.get(region_id) { @@ -237,27 +319,26 @@ impl RegionFlushTrigger { .cloned() .and_then(|value| value.checkpoint), ) - .map(|checkpoint| { - ( - TopicRegionKey::new(*region_id, topic), - Some(TopicRegionValue::new(Some(checkpoint))), - ) - }), + .map(|checkpoint| (*region_id, TopicRegionValue::new(Some(checkpoint)))), None => None, }) .collect::>(); - // The`chunks` will panic if chunks_size is zero, so we return early if there are no regions to persist. + // `chunks` will panic if chunk size is zero, so return early if there are no regions to persist. if regions.is_empty() { - return Ok(()); + return Ok(Vec::new()); } let max_txn_ops = self.table_metadata_manager.kv_backend().max_txn_ops(); let batch_size = max_txn_ops.min(regions.len()); for batch in regions.chunks(batch_size) { + let batch = batch + .iter() + .map(|(region_id, value)| (TopicRegionKey::new(*region_id, topic), Some(*value))) + .collect::>(); self.table_metadata_manager .topic_region_manager() - .batch_put(batch) + .batch_put(&batch) .await .context(error::TableMetadataManagerSnafu)?; } @@ -266,7 +347,10 @@ impl RegionFlushTrigger { .with_label_values(&[topic]) .inc_by(regions.len() as u64); - Ok(()) + Ok(regions + .into_iter() + .map(|(region_id, _)| region_id) + .collect()) } async fn flush_regions_in_topic( @@ -274,45 +358,8 @@ impl RegionFlushTrigger { topic: &str, latest_entry_id: u64, avg_record_size: usize, + region_manifests: HashMap, ) -> Result<()> { - let topic_regions = self - .table_metadata_manager - .topic_region_manager() - .regions(topic) - .await - .context(error::TableMetadataManagerSnafu)?; - - if topic_regions.is_empty() { - debug!("No regions found for topic: {}", topic); - return Ok(()); - } - - // Filters regions need to persist checkpoints. - let regions_to_persist = filter_regions_by_replay_size( - topic, - topic_regions - .iter() - .map(|(region_id, value)| (*region_id, value.min_entry_id().unwrap_or_default())), - avg_record_size as u64, - latest_entry_id, - self.checkpoint_trigger_size, - ); - let region_manifests = self - .leader_region_registry - .batch_get(topic_regions.keys().cloned()); - - if let Err(err) = self - .persist_region_checkpoints( - topic, - ®ions_to_persist, - &topic_regions, - ®ion_manifests, - ) - .await - { - error!(err; "Failed to persist region checkpoints for topic: {}", topic); - } - let regions = region_manifests .into_iter() .map(|(region_id, region)| (region_id, region.manifest.prunable_entry_id())) @@ -447,6 +494,56 @@ fn filter_regions_by_replay_size>( regions_to_flush } +/// Filters regions that need periodic checkpoint persistence. +fn filter_regions_for_periodic_checkpoint( + regions: I, + last_persisted: &HashMap, + now_millis: i64, + interval: Duration, +) -> Vec +where + I: Iterator, +{ + let interval_millis = interval.as_millis() as i64; + regions + .filter(|region_id| { + last_persisted + .get(region_id) + .is_none_or(|last_persist_millis| { + now_millis.saturating_sub(*last_persist_millis) >= interval_millis + }) + }) + .collect() +} + +/// Merges two region id lists and removes duplicates. +fn merge_region_ids(left: Vec, right: Vec) -> Vec { + left.into_iter() + .chain(right) + .collect::>() + .into_iter() + .collect() +} + +/// Marks checkpoint persistence timestamps for regions. +fn mark_checkpoint_persisted( + last_persisted: &mut HashMap, + region_ids: &[RegionId], + now_millis: i64, +) { + for region_id in region_ids { + last_persisted.insert(*region_id, now_millis); + } +} + +/// Retains checkpoint persistence records for active regions. +fn retain_checkpoint_persist_records( + last_persisted: &mut HashMap, + active_region_ids: &HashSet, +) { + last_persisted.retain(|region_id, _| active_region_ids.contains(region_id)); +} + /// Group regions by leader. /// /// The regions are grouped by the leader of the region. @@ -527,6 +624,71 @@ mod tests { assert!(!is_recent(now - 1001, now, Duration::from_secs(1))); } + #[test] + fn test_filter_regions_for_periodic_checkpoint() { + let now_millis = 10_000; + let interval = Duration::from_secs(5); + let regions = vec![region_id(1, 1), region_id(1, 2), region_id(1, 3)]; + let last_persisted = HashMap::from([ + (region_id(1, 1), now_millis - 4_000), + (region_id(1, 2), now_millis - 5_000), + ]); + + let result = filter_regions_for_periodic_checkpoint( + regions.into_iter(), + &last_persisted, + now_millis, + interval, + ); + + assert_eq!(result, vec![region_id(1, 2), region_id(1, 3)]); + } + + #[test] + fn test_merge_region_ids() { + let merged = merge_region_ids( + vec![region_id(1, 1), region_id(1, 2)], + vec![region_id(1, 2), region_id(1, 3)], + ); + let merged = merged.into_iter().collect::>(); + + assert_eq!( + merged, + HashSet::from([region_id(1, 1), region_id(1, 2), region_id(1, 3)]) + ); + } + + #[test] + fn test_mark_checkpoint_persisted() { + let now_millis = 10_000; + let mut last_persisted = HashMap::from([(region_id(1, 1), 1_000)]); + + mark_checkpoint_persisted( + &mut last_persisted, + &[region_id(1, 1), region_id(1, 2)], + now_millis, + ); + + assert_eq!(last_persisted.get(®ion_id(1, 1)), Some(&now_millis)); + assert_eq!(last_persisted.get(®ion_id(1, 2)), Some(&now_millis)); + } + + #[test] + fn test_retain_checkpoint_persist_records() { + let mut last_persisted = HashMap::from([ + (region_id(1, 1), 1_000), + (region_id(1, 2), 2_000), + (region_id(1, 3), 3_000), + ]); + let active_regions = HashSet::from([region_id(1, 1), region_id(1, 3)]); + + retain_checkpoint_persist_records(&mut last_persisted, &active_regions); + + assert_eq!(last_persisted.len(), 2); + assert!(last_persisted.contains_key(®ion_id(1, 1))); + assert!(last_persisted.contains_key(®ion_id(1, 3))); + } + fn region_id(table: u32, region: u32) -> RegionId { RegionId::new(table, region) } @@ -735,6 +897,76 @@ mod tests { assert!(result.is_none()); } + #[tokio::test] + async fn test_persist_region_checkpoints_returns_written_region_ids() { + let kv_backend = Arc::new(MemoryKvBackend::new()); + let table_metadata_manager = Arc::new(TableMetadataManager::new(kv_backend.clone())); + let leader_region_registry = Arc::new(LeaderRegionRegistry::new()); + let topic_stats_registry = Arc::new(TopicStatsRegistry::default()); + let mailbox_sequence = SequenceBuilder::new( + "test_persist_region_checkpoints_returns_written_region_ids", + kv_backend, + ) + .build(); + let mailbox_ctx = MailboxContext::new(mailbox_sequence); + + let (trigger, _ticker) = RegionFlushTrigger::new( + table_metadata_manager.clone(), + leader_region_registry, + topic_stats_registry, + mailbox_ctx.mailbox().clone(), + "127.0.0.1:3002".to_string(), + ReadableSize(1), + ReadableSize(1), + ); + + let topic = "test_topic"; + let region_to_write = region_id(1, 1); + let region_already_persisted = region_id(1, 2); + let region_without_leader = region_id(1, 3); + let topic_regions = HashMap::from([ + (region_to_write, TopicRegionValue::new(None)), + ( + region_already_persisted, + TopicRegionValue::new(Some(ReplayCheckpoint::new(100, None))), + ), + (region_without_leader, TopicRegionValue::new(None)), + ]); + let leader_regions = HashMap::from([ + (region_to_write, mito_leader_region(100)), + (region_already_persisted, mito_leader_region(100)), + ]); + + let written_region_ids = trigger + .persist_region_checkpoints( + topic, + &[ + region_to_write, + region_already_persisted, + region_without_leader, + ], + &topic_regions, + &leader_regions, + ) + .await + .unwrap(); + + assert_eq!(written_region_ids, vec![region_to_write]); + let persisted = table_metadata_manager + .topic_region_manager() + .get(TopicRegionKey::new(region_to_write, topic)) + .await + .unwrap() + .unwrap(); + assert_eq!(persisted.checkpoint, Some(ReplayCheckpoint::new(100, None))); + let skipped = table_metadata_manager + .topic_region_manager() + .get(TopicRegionKey::new(region_already_persisted, topic)) + .await + .unwrap(); + assert!(skipped.is_none()); + } + #[tokio::test] async fn test_send_flush_instructions_payload_includes_remote_wal_prune_reason() { let kv_backend = Arc::new(MemoryKvBackend::new()); diff --git a/src/mito2/src/engine/flush_test.rs b/src/mito2/src/engine/flush_test.rs index b86e75c72a..9279d7def9 100644 --- a/src/mito2/src/engine/flush_test.rs +++ b/src/mito2/src/engine/flush_test.rs @@ -14,6 +14,7 @@ //! Flush tests for mito engine. +use std::collections::HashMap; use std::sync::Arc; use std::sync::atomic::{AtomicI64, Ordering}; use std::time::Duration; @@ -21,11 +22,14 @@ use std::time::Duration; use api::v1::Rows; use common_recordbatch::RecordBatches; use common_time::util::current_time_millis; -use common_wal::options::WAL_OPTIONS_KEY; +use common_wal::options::{KafkaWalOptions, WAL_OPTIONS_KEY, WalOptions}; use rstest::rstest; use rstest_reuse::{self, apply}; use store_api::region_engine::RegionEngine; -use store_api::region_request::{RegionFlushRequest, RegionRequest}; +use store_api::region_request::{ + PathType, RegionCloseRequest, RegionFlushRequest, RegionOpenRequest, RegionRequest, + ReplayCheckpoint, +}; use store_api::storage::{RegionId, ScanRequest}; use crate::config::MitoConfig; @@ -319,8 +323,6 @@ async fn test_flush_empty_with_format(flat_format: bool) { async fn test_flush_reopen_region(factory: Option) { use std::collections::HashMap; - use common_wal::options::{KafkaWalOptions, WalOptions}; - common_telemetry::init_default_ut_logging(); let Some(factory) = factory else { return; @@ -394,6 +396,235 @@ async fn test_flush_reopen_region(factory: Option) { assert_eq!(5, version_data.committed_sequence); } +#[apply(single_kafka_log_store_factory)] +async fn test_skip_remote_wal_replay_sets_topic_latest_entry_id(factory: Option) { + common_telemetry::init_default_ut_logging(); + let Some(factory) = factory else { + return; + }; + + let mut env = TestEnv::new().await.with_log_store_factory(factory.clone()); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + env.get_schema_metadata_manager() + .register_region_table_info( + region_id.table_id(), + "test_table", + "test_catalog", + "test_schema", + None, + env.get_kv_backend(), + ) + .await; + + let topic = prepare_test_for_kafka_log_store(&factory).await; + let request = CreateRequestBuilder::new() + .kafka_topic(topic.clone()) + .build(); + let table_dir = request.table_dir.clone(); + let column_schemas = rows_schema(&request); + let options = kafka_wal_options(&topic); + + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + + let rows = Rows { + schema: column_schemas, + rows: build_rows_for_key("a", 0, 2, 0), + }; + put_rows(&engine, region_id, rows).await; + flush_region(&engine, region_id, None).await; + // The empty flush updates `topic_latest_entry_id` from KafkaLogStore's topic stats. + flush_region(&engine, region_id, None).await; + + let region = engine.get_region(region_id).unwrap(); + assert_eq!(1, region.topic_latest_entry_id.load(Ordering::Relaxed)); + + engine + .handle_request(region_id, RegionRequest::Close(RegionCloseRequest {})) + .await + .unwrap(); + + engine + .handle_request( + region_id, + RegionRequest::Open(RegionOpenRequest { + engine: String::new(), + table_dir, + path_type: PathType::Bare, + options, + skip_wal_replay: true, + checkpoint: Some(ReplayCheckpoint { + entry_id: 1, + metadata_entry_id: None, + }), + requirements: Default::default(), + }), + ) + .await + .unwrap(); + + let region = engine.get_region(region_id).unwrap(); + assert_eq!(1, region.topic_latest_entry_id.load(Ordering::Relaxed)); +} + +#[apply(single_kafka_log_store_factory)] +async fn test_remote_wal_open_with_replayed_memtable_sets_topic_latest_entry_id( + factory: Option, +) { + common_telemetry::init_default_ut_logging(); + let Some(factory) = factory else { + return; + }; + + let mut env = TestEnv::new().await.with_log_store_factory(factory.clone()); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + env.get_schema_metadata_manager() + .register_region_table_info( + region_id.table_id(), + "test_table", + "test_catalog", + "test_schema", + None, + env.get_kv_backend(), + ) + .await; + + let topic = prepare_test_for_kafka_log_store(&factory).await; + let request = CreateRequestBuilder::new() + .kafka_topic(topic.clone()) + .build(); + let table_dir = request.table_dir.clone(); + let column_schemas = rows_schema(&request); + let options = kafka_wal_options(&topic); + + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + let rows = Rows { + schema: column_schemas.clone(), + rows: build_rows_for_key("a", 0, 2, 0), + }; + put_rows(&engine, region_id, rows).await; + flush_region(&engine, region_id, None).await; + + let rows = Rows { + schema: column_schemas, + rows: build_rows_for_key("b", 0, 2, 0), + }; + put_rows(&engine, region_id, rows).await; + + let engine = env.reopen_engine(engine, MitoConfig::default()).await; + engine + .handle_request( + region_id, + RegionRequest::Open(RegionOpenRequest { + engine: String::new(), + table_dir, + path_type: PathType::Bare, + options, + skip_wal_replay: false, + checkpoint: None, + requirements: Default::default(), + }), + ) + .await + .unwrap(); + + let region = engine.get_region(region_id).unwrap(); + assert_eq!(1, region.version().flushed_entry_id); + assert!(!region.version().memtables.is_empty()); + assert_eq!(1, region.topic_latest_entry_id.load(Ordering::Relaxed)); +} + +#[apply(single_kafka_log_store_factory)] +async fn test_remote_wal_open_without_replayed_memtable_sets_topic_latest_entry_id( + factory: Option, +) { + common_telemetry::init_default_ut_logging(); + let Some(factory) = factory else { + return; + }; + + let mut env = TestEnv::new().await.with_log_store_factory(factory.clone()); + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + env.get_schema_metadata_manager() + .register_region_table_info( + region_id.table_id(), + "test_table", + "test_catalog", + "test_schema", + None, + env.get_kv_backend(), + ) + .await; + + let topic = prepare_test_for_kafka_log_store(&factory).await; + let request = CreateRequestBuilder::new() + .kafka_topic(topic.clone()) + .build(); + let table_dir = request.table_dir.clone(); + let column_schemas = rows_schema(&request); + let options = kafka_wal_options(&topic); + + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + let rows = Rows { + schema: column_schemas, + rows: build_rows_for_key("a", 0, 2, 0), + }; + put_rows(&engine, region_id, rows).await; + flush_region(&engine, region_id, None).await; + // The empty flush updates `topic_latest_entry_id` from KafkaLogStore's topic stats. + flush_region(&engine, region_id, None).await; + engine + .handle_request(region_id, RegionRequest::Close(RegionCloseRequest {})) + .await + .unwrap(); + + engine + .handle_request( + region_id, + RegionRequest::Open(RegionOpenRequest { + engine: String::new(), + table_dir, + path_type: PathType::Bare, + options, + skip_wal_replay: false, + checkpoint: None, + requirements: Default::default(), + }), + ) + .await + .unwrap(); + + let region = engine.get_region(region_id).unwrap(); + assert!(region.version().memtables.is_empty()); + assert_eq!(1, region.topic_latest_entry_id.load(Ordering::Relaxed)); +} + +fn kafka_wal_options(topic: &Option) -> HashMap { + topic + .as_ref() + .map(|topic| { + HashMap::from([( + WAL_OPTIONS_KEY.to_string(), + serde_json::to_string(&WalOptions::Kafka(KafkaWalOptions { + topic: topic.clone(), + })) + .unwrap(), + )]) + }) + .unwrap_or_default() +} + #[derive(Debug)] pub(crate) struct MockTimeProvider { now: AtomicI64, diff --git a/src/mito2/src/region/opener.rs b/src/mito2/src/region/opener.rs index 412172aead..e09a2e2aa0 100644 --- a/src/mito2/src/region/opener.rs +++ b/src/mito2/src/region/opener.rs @@ -339,6 +339,7 @@ impl RegionOpener { let version = VersionBuilder::new(metadata, mutable) .options(options) + .flushed_entry_id(flushed_entry_id) .build(); let version_control = Arc::new(VersionControl::new(version)); let access_layer = Arc::new(AccessLayer::new( @@ -371,7 +372,7 @@ impl RegionOpener { last_flush_millis: AtomicI64::new(now), last_schedule_compaction_millis: AtomicI64::new(now), time_provider: self.time_provider.clone(), - topic_latest_entry_id: AtomicU64::new(0), + topic_latest_entry_id: AtomicU64::new(flushed_entry_id), written_bytes: Arc::new(AtomicU64::new(0)), stats: self.stats, })) @@ -534,11 +535,11 @@ impl RegionOpener { let flushed_entry_id = version.flushed_entry_id; let version_control = Arc::new(VersionControl::new(version)); + let replay_from_entry_id = self + .replay_checkpoint + .unwrap_or_default() + .max(flushed_entry_id); let topic_latest_entry_id = if !self.skip_wal_replay { - let replay_from_entry_id = self - .replay_checkpoint - .unwrap_or_default() - .max(flushed_entry_id); info!( "Start replaying memtable at replay_from_entry_id: {} for region {}, manifest version: {}, flushed entry id: {}, elapsed: {:?}", replay_from_entry_id, @@ -561,7 +562,11 @@ impl RegionOpener { // Only set after the WAL replay is completed. if provider.is_remote_wal() && version_control.current().version.memtables.is_empty() { - wal.store().latest_entry_id(&provider).unwrap_or(0) + wal.store() + .latest_entry_id(&provider) + .unwrap_or(replay_from_entry_id) + } else if provider.is_remote_wal() { + replay_from_entry_id } else { 0 } @@ -574,7 +579,11 @@ impl RegionOpener { now.elapsed() ); - 0 + if provider.is_remote_wal() { + replay_from_entry_id + } else { + 0 + } }; if let Some(committed_in_manifest) = manifest.committed_sequence { From a6cb4d20832b71e37e833dc16ac4c1ed78b6fe66 Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Thu, 4 Jun 2026 15:41:49 +0800 Subject: [PATCH 27/29] feat: look up cache with range calculation (#8123) * feat: loop up cache with range calculation Signed-off-by: Ruihang Xia * copy fragements Signed-off-by: Ruihang Xia * release dashmap reference Signed-off-by: Ruihang Xia * fix bugs Signed-off-by: Ruihang Xia * remove dashmap Signed-off-by: Ruihang Xia * remove keys Signed-off-by: Ruihang Xia * clean up Signed-off-by: Ruihang Xia --------- Signed-off-by: Ruihang Xia --- src/mito2/src/cache.rs | 573 ++++++++++++++++++---- src/mito2/src/sst/parquet.rs | 19 +- src/mito2/src/sst/parquet/push_decoder.rs | 256 ++++++++-- 3 files changed, 713 insertions(+), 135 deletions(-) diff --git a/src/mito2/src/cache.rs b/src/mito2/src/cache.rs index eee1cfae0a..89e84fadcc 100644 --- a/src/mito2/src/cache.rs +++ b/src/mito2/src/cache.rs @@ -23,9 +23,10 @@ pub(crate) mod manifest_cache; pub(crate) mod test_util; pub(crate) mod write_cache; +use std::collections::{BTreeMap, HashMap}; use std::mem; use std::ops::Range; -use std::sync::Arc; +use std::sync::{Arc, RwLock}; use bytes::Bytes; use common_base::readable_size::ReadableSize; @@ -527,20 +528,33 @@ impl CacheStrategy { } } - /// Calls [CacheManager::get_pages()]. + /// Calls [CacheManager::get_page_ranges()]. /// It returns None if the strategy is [CacheStrategy::Compaction] or [CacheStrategy::Disabled]. - pub fn get_pages(&self, page_key: &PageKey) -> Option> { + pub fn get_page_ranges( + &self, + file_id: FileId, + row_group_idx: usize, + ranges: &[Range], + ) -> Option { match self { - CacheStrategy::EnableAll(cache_manager) => cache_manager.get_pages(page_key), + CacheStrategy::EnableAll(cache_manager) => { + cache_manager.get_page_ranges(file_id, row_group_idx, ranges) + } CacheStrategy::Compaction(_) | CacheStrategy::Disabled => None, } } - /// Calls [CacheManager::put_pages()]. + /// Calls [CacheManager::put_page_ranges()]. /// It does nothing if the strategy isn't [CacheStrategy::EnableAll]. - pub fn put_pages(&self, page_key: PageKey, pages: Arc) { + pub fn put_page_ranges( + &self, + file_id: FileId, + row_group_idx: usize, + ranges: &[Range], + pages: &[Bytes], + ) { if let CacheStrategy::EnableAll(cache_manager) = self { - cache_manager.put_pages(page_key, pages); + cache_manager.put_page_ranges(file_id, row_group_idx, ranges, pages); } } @@ -720,8 +734,8 @@ pub struct CacheManager { sst_meta_cache: Option, /// Cache for vectors. vector_cache: Option, - /// Cache for SST pages. - page_cache: Option, + /// Cache for SST byte ranges. + page_cache: Option>, /// A Cache for writing files to object stores. write_cache: Option, /// Cache for inverted index. @@ -904,21 +918,35 @@ impl CacheManager { } } - /// Gets pages for the row group. - pub fn get_pages(&self, page_key: &PageKey) -> Option> { - self.page_cache.as_ref().and_then(|page_cache| { - let value = page_cache.get(page_key); - update_hit_miss(value, PAGE_TYPE) + /// Gets cached byte fragments for the requested ranges. + pub fn get_page_ranges( + &self, + file_id: FileId, + row_group_idx: usize, + ranges: &[Range], + ) -> Option { + self.page_cache.as_ref().map(|page_cache| { + let lookup = page_cache.lookup(file_id, row_group_idx, ranges); + if lookup.cached_bytes > 0 { + CACHE_HIT.with_label_values(&[PAGE_TYPE]).inc(); + } + if !lookup.missing_ranges.is_empty() { + CACHE_MISS.with_label_values(&[PAGE_TYPE]).inc(); + } + lookup }) } - /// Puts pages of the row group into the cache. - pub fn put_pages(&self, page_key: PageKey, pages: Arc) { + /// Puts byte fragments into the page cache. + pub fn put_page_ranges( + &self, + file_id: FileId, + row_group_idx: usize, + ranges: &[Range], + pages: &[Bytes], + ) { if let Some(cache) = &self.page_cache { - CACHE_BYTES - .with_label_values(&[PAGE_TYPE]) - .add(page_cache_weight(&page_key, &pages).into()); - cache.insert(page_key, pages); + cache.insert_ranges(file_id, row_group_idx, ranges, pages); } } @@ -1192,19 +1220,8 @@ impl CacheManagerBuilder { }) .build() }); - let page_cache = (self.page_cache_size != 0).then(|| { - Cache::builder() - .max_capacity(self.page_cache_size) - .weigher(page_cache_weight) - .eviction_listener(|k, v, cause| { - let size = page_cache_weight(&k, &v); - CACHE_BYTES.with_label_values(&[PAGE_TYPE]).sub(size.into()); - CACHE_EVICTION - .with_label_values(&[PAGE_TYPE, removal_cause_str(cause)]) - .inc(); - }) - .build() - }); + let page_cache = + (self.page_cache_size != 0).then(|| PageRangeCache::new(self.page_cache_size)); let inverted_index_cache = InvertedIndexCache::new( self.index_metadata_size, self.index_content_size, @@ -1288,8 +1305,8 @@ fn vector_cache_weight(_k: &(ConcreteDataType, Value), v: &VectorRef) -> u32 { (mem::size_of::() + mem::size_of::() + v.memory_size()) as u32 } -fn page_cache_weight(k: &PageKey, v: &Arc) -> u32 { - (k.estimated_size() + v.estimated_size()) as u32 +fn page_cache_weight(k: &PageFragmentKey, v: &Bytes) -> u32 { + (k.estimated_size() + mem::size_of::() + v.len()) as u32 } fn selector_result_cache_weight(k: &SelectorResultKey, v: &Arc) -> u32 { @@ -1321,73 +1338,281 @@ impl SstMetaKey { } } -/// Path to column pages in the SST file. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct ColumnPagePath { - /// Region id of the SST file to cache. - region_id: RegionId, - /// Id of the SST file to cache. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +struct PageFragmentGroupKey { + file_id: FileId, + row_group_idx: usize, +} + +/// Cache key for one byte fragment in an SST row group. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct PageFragmentKey { + /// Id of the SST file. file_id: FileId, /// Index of the row group. row_group_idx: usize, - /// Index of the column in the row group. - column_idx: usize, + /// Start offset of the cached byte fragment. + start: u64, + /// End offset of the cached byte fragment. + end: u64, } -/// Cache key to pages in a row group (after projection). -/// -/// Different projections will have different cache keys. -/// We cache all ranges together because they may refer to the same `Bytes`. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct PageKey { - /// Id of the SST file to cache. - file_id: FileId, - /// Index of the row group. - row_group_idx: usize, - /// Byte ranges of the pages to cache. - ranges: Vec>, -} - -impl PageKey { - /// Creates a key for a list of pages. - pub fn new(file_id: FileId, row_group_idx: usize, ranges: Vec>) -> PageKey { - PageKey { +impl PageFragmentKey { + fn new(file_id: FileId, row_group_idx: usize, range: &Range) -> PageFragmentKey { + PageFragmentKey { file_id, row_group_idx, - ranges, + start: range.start, + end: range.end, + } + } + + fn group_key(&self) -> PageFragmentGroupKey { + PageFragmentGroupKey { + file_id: self.file_id, + row_group_idx: self.row_group_idx, } } /// Returns memory used by the key (estimated). fn estimated_size(&self) -> usize { - mem::size_of::() + mem::size_of_val(self.ranges.as_slice()) + mem::size_of::() } } -/// Cached row group pages for a column. -// We don't use enum here to make it easier to mock and use the struct. -#[derive(Default)] -pub struct PageValue { - /// Compressed page in the row group. - pub compressed: Vec, - /// Total size of the pages (may be larger than sum of compressed bytes due to gaps). - pub page_size: u64, +/// One cached byte fragment that overlaps a requested range. +#[derive(Clone)] +pub struct PageRangePart { + /// Range covered by `bytes`. + pub range: Range, + /// Bytes for `range`. + pub bytes: Bytes, } -impl PageValue { - /// Creates a new value from a range of compressed pages. - pub fn new(bytes: Vec, page_size: u64) -> PageValue { - PageValue { - compressed: bytes, - page_size, +/// Result of looking up request ranges in the page range cache. +pub struct PageRangeLookup { + /// Cached fragments grouped by the original requested range index. + pub cached_parts: Vec>, + /// Ranges that are not covered by cached fragments and need fetching. + pub missing_ranges: Vec>, + /// Number of cached fragments used. + pub cached_range_count: usize, + /// Number of requested bytes served from cached fragments. + pub cached_bytes: u64, +} + +impl PageRangeLookup { + pub fn is_fully_cached(&self) -> bool { + self.missing_ranges.is_empty() + } +} + +type PageFragmentRangeIndex = BTreeMap<(u64, u64), PageFragmentKey>; +type PageFragmentIndex = HashMap; + +/// Byte-fragment cache for Parquet row-group reads. +pub struct PageRangeCache { + cache: Cache, + index: RwLock, +} + +impl PageRangeCache { + fn new(capacity: u64) -> Arc { + Arc::new_cyclic(|weak_cache: &std::sync::Weak| { + let cache = Cache::builder() + .max_capacity(capacity) + .weigher(page_cache_weight) + .eviction_listener({ + let weak_cache = weak_cache.clone(); + move |k, v, cause| { + let size = page_cache_weight(&k, &v); + CACHE_BYTES.with_label_values(&[PAGE_TYPE]).sub(size.into()); + CACHE_EVICTION + .with_label_values(&[PAGE_TYPE, removal_cause_str(cause)]) + .inc(); + + if let Some(cache) = weak_cache.upgrade() + && !matches!(cause, RemovalCause::Replaced) + { + cache.remove_index_entry(*k); + } + } + }) + .build(); + + PageRangeCache { + cache, + index: RwLock::new(HashMap::new()), + } + }) + } + + fn lookup( + &self, + file_id: FileId, + row_group_idx: usize, + ranges: &[Range], + ) -> PageRangeLookup { + let mut cached_parts = Vec::with_capacity(ranges.len()); + let mut missing_ranges = Vec::new(); + let mut cached_range_count = 0; + let mut cached_bytes = 0; + + for range in ranges { + if range.start >= range.end { + cached_parts.push(Vec::new()); + continue; + } + + let mut parts = Vec::new(); + let candidates = self.find_index_candidates(file_id, row_group_idx, range); + let mut stale_keys = Vec::new(); + + for fragment_key in candidates { + if let Some(bytes) = self.cache.get(&fragment_key) { + let part_start = range.start.max(fragment_key.start); + let part_end = range.end.min(fragment_key.end); + let slice_start = (part_start - fragment_key.start) as usize; + let slice_end = (part_end - fragment_key.start) as usize; + parts.push(PageRangePart { + range: part_start..part_end, + bytes: bytes.slice(slice_start..slice_end), + }); + } else { + stale_keys.push(fragment_key); + } + } + for key in stale_keys { + self.remove_uncached_index_entry(key); + } + + let mut cursor = range.start; + let mut compacted_parts: Vec = Vec::with_capacity(parts.len()); + for part in parts { + if part.range.end <= cursor { + continue; + } + + let part = if part.range.start < cursor { + let offset = (cursor - part.range.start) as usize; + PageRangePart { + range: cursor..part.range.end, + bytes: part.bytes.slice(offset..), + } + } else { + part + }; + + if cursor < part.range.start { + missing_ranges.push(cursor..part.range.start); + } + cached_bytes += part.range.end - part.range.start; + cached_range_count += 1; + cursor = part.range.end; + compacted_parts.push(part); + + if cursor >= range.end { + break; + } + } + + if cursor < range.end { + missing_ranges.push(cursor..range.end); + } + cached_parts.push(compacted_parts); + } + + PageRangeLookup { + cached_parts, + missing_ranges, + cached_range_count, + cached_bytes, } } - /// Returns memory used by the value (estimated). - fn estimated_size(&self) -> usize { - mem::size_of::() - + self.page_size as usize - + self.compressed.iter().map(mem::size_of_val).sum::() + fn insert_ranges( + &self, + file_id: FileId, + row_group_idx: usize, + ranges: &[Range], + pages: &[Bytes], + ) { + for (range, bytes) in ranges.iter().zip(pages) { + if range.start >= range.end || bytes.len() as u64 != range.end - range.start { + continue; + } + + let key = PageFragmentKey::new(file_id, row_group_idx, range); + let bytes = Bytes::copy_from_slice(bytes.as_ref()); + let size = page_cache_weight(&key, &bytes); + CACHE_BYTES.with_label_values(&[PAGE_TYPE]).add(size.into()); + self.cache.insert(key, bytes); + let mut index = self.index.write().unwrap(); + index + .entry(key.group_key()) + .or_default() + .insert((key.start, key.end), key); + } + } + + fn find_index_candidates( + &self, + file_id: FileId, + row_group_idx: usize, + range: &Range, + ) -> Vec { + let group_key = PageFragmentGroupKey { + file_id, + row_group_idx, + }; + let index = self.index.read().unwrap(); + index + .get(&group_key) + .map(|ranges| { + ranges + .range(..(range.end, 0)) + .filter_map(|(_, fragment_key)| { + (fragment_key.end > range.start).then_some(*fragment_key) + }) + .collect() + }) + .unwrap_or_default() + } + + fn remove_uncached_index_entry(&self, key: PageFragmentKey) { + let group_key = key.group_key(); + let mut index = self.index.write().unwrap(); + if self.cache.contains_key(&key) { + return; + } + + Self::remove_index_entry_locked(&mut index, group_key, key); + } + + fn remove_index_entry(&self, key: PageFragmentKey) { + let group_key = key.group_key(); + let mut index = self.index.write().unwrap(); + Self::remove_index_entry_locked(&mut index, group_key, key); + } + + fn remove_index_entry_locked( + index: &mut PageFragmentIndex, + group_key: PageFragmentGroupKey, + key: PageFragmentKey, + ) { + let Some(ranges) = index.get_mut(&group_key) else { + return; + }; + + let removed = ranges + .get(&(key.start, key.end)) + .is_some_and(|current| current == &key); + if removed { + ranges.remove(&(key.start, key.end)); + } + if ranges.is_empty() { + index.remove(&group_key); + } } } @@ -1455,8 +1680,6 @@ type SstMetaCache = Cache>; /// /// e.g. `"hello" => ["hello", "hello", "hello"]` type VectorCache = Cache<(ConcreteDataType, Value), VectorRef>; -/// Maps (region, file, row group, column) to [PageValue]. -type PageCache = Cache>; /// Maps (file id, row group id, time series row selector) to [SelectorResultValue]. type SelectorResultCache = Cache>; /// Maps partition-range scan key to cached flat batches. @@ -1514,10 +1737,17 @@ mod tests { .is_none() ); - let key = PageKey::new(file_id.file_id(), 1, vec![Range { start: 0, end: 5 }]); - let pages = Arc::new(PageValue::default()); - cache.put_pages(key.clone(), pages); - assert!(cache.get_pages(&key).is_none()); + cache.put_page_ranges( + file_id.file_id(), + 1, + &[Range { start: 0, end: 5 }], + &[Bytes::from_static(b"abcde")], + ); + assert!( + cache + .get_page_ranges(file_id.file_id(), 1, &[Range { start: 0, end: 5 }]) + .is_none() + ); assert!(cache.write_cache().is_none()); } @@ -1675,11 +1905,168 @@ mod tests { fn test_page_cache() { let cache = CacheManager::builder().page_cache_size(1000).build(); let file_id = FileId::random(); - let key = PageKey::new(file_id, 0, vec![(0..10), (10..20)]); - assert!(cache.get_pages(&key).is_none()); - let pages = Arc::new(PageValue::default()); - cache.put_pages(key.clone(), pages); - assert!(cache.get_pages(&key).is_some()); + let uncached = 0..10; + assert_eq!( + vec![0..10], + cache + .get_page_ranges(file_id, 0, std::slice::from_ref(&uncached)) + .unwrap() + .missing_ranges + ); + + let cached = 100..500; + cache.put_page_ranges( + file_id, + 0, + std::slice::from_ref(&cached), + &[Bytes::from(vec![7; 400])], + ); + + let subrange = 200..300; + let lookup = cache + .get_page_ranges(file_id, 0, std::slice::from_ref(&subrange)) + .unwrap(); + assert!(lookup.is_fully_cached()); + assert_eq!(100, lookup.cached_bytes); + assert_eq!(1, lookup.cached_parts.len()); + assert_eq!(200..300, lookup.cached_parts[0][0].range); + assert_eq!(100, lookup.cached_parts[0][0].bytes.len()); + + let overlapping = 400..600; + let lookup = cache + .get_page_ranges(file_id, 0, std::slice::from_ref(&overlapping)) + .unwrap(); + assert!(!lookup.is_fully_cached()); + assert_eq!(100, lookup.cached_bytes); + assert_eq!(vec![500..600], lookup.missing_ranges); + assert_eq!(400..500, lookup.cached_parts[0][0].range); + } + + #[test] + fn test_page_cache_detaches_fragment_bytes() { + let cache = PageRangeCache::new(1000); + let file_id = FileId::random(); + let backing = Bytes::from(vec![1; 1024]); + let page = backing.slice(512..522); + let page_ptr = page.as_ptr(); + let range = 0..10; + + cache.insert_ranges( + file_id, + 0, + std::slice::from_ref(&range), + std::slice::from_ref(&page), + ); + + let lookup = cache.lookup(file_id, 0, std::slice::from_ref(&range)); + assert!(lookup.is_fully_cached()); + assert_eq!(1, lookup.cached_parts[0].len()); + assert_eq!(&page[..], &lookup.cached_parts[0][0].bytes[..]); + assert_ne!(page_ptr, lookup.cached_parts[0][0].bytes.as_ptr()); + } + + #[test] + fn test_page_cache_replaces_fragment() { + let cache = PageRangeCache::new(1000); + let file_id = FileId::random(); + let range = 0..10; + + cache.insert_ranges( + file_id, + 0, + std::slice::from_ref(&range), + &[Bytes::from(vec![1; 10])], + ); + cache.insert_ranges( + file_id, + 0, + std::slice::from_ref(&range), + &[Bytes::from(vec![2; 10])], + ); + cache.cache.run_pending_tasks(); + assert_eq!( + vec![PageFragmentKey::new(file_id, 0, &range)], + cache.find_index_candidates(file_id, 0, &range) + ); + + let lookup = cache.lookup(file_id, 0, std::slice::from_ref(&range)); + assert!(lookup.is_fully_cached()); + assert_eq!(&vec![2; 10][..], &lookup.cached_parts[0][0].bytes[..]); + } + + #[test] + fn test_page_cache_retains_disjoint_inserts_for_same_row_group() { + let cache = PageRangeCache::new(1000); + let file_id = FileId::random(); + let range1 = 0..10; + let range2 = 20..30; + + cache.insert_ranges( + file_id, + 0, + std::slice::from_ref(&range1), + &[Bytes::from(vec![1; 10])], + ); + cache.insert_ranges( + file_id, + 0, + std::slice::from_ref(&range2), + &[Bytes::from(vec![2; 10])], + ); + + let lookup = cache.lookup(file_id, 0, &[range1, range2]); + assert!(lookup.is_fully_cached()); + assert_eq!(2, lookup.cached_range_count); + assert_eq!(&vec![1; 10][..], &lookup.cached_parts[0][0].bytes[..]); + assert_eq!(&vec![2; 10][..], &lookup.cached_parts[1][0].bytes[..]); + } + + #[test] + fn test_page_cache_fragment_eviction() { + let file_id = FileId::random(); + let range = 0..10; + let key = PageFragmentKey::new(file_id, 0, &range); + let page = Bytes::from(vec![1; 10]); + let cache = PageRangeCache::new(page_cache_weight(&key, &page) as u64); + + cache.insert_ranges( + file_id, + 0, + std::slice::from_ref(&range), + &[Bytes::from(vec![1; 10])], + ); + assert!( + cache + .lookup(file_id, 0, std::slice::from_ref(&range)) + .is_fully_cached() + ); + + cache.cache.invalidate(&key); + cache.cache.run_pending_tasks(); + assert!(cache.find_index_candidates(file_id, 0, &range).is_empty()); + + let lookup = cache.lookup(file_id, 0, std::slice::from_ref(&range)); + assert!(!lookup.is_fully_cached()); + assert_eq!(vec![0..10], lookup.missing_ranges); + } + + #[test] + fn test_page_cache_rejects_oversized_fragment() { + let cache = PageRangeCache::new(1); + let file_id = FileId::random(); + let range = 0..10; + + cache.insert_ranges( + file_id, + 0, + std::slice::from_ref(&range), + &[Bytes::from(vec![1; 10])], + ); + cache.cache.run_pending_tasks(); + + let lookup = cache.lookup(file_id, 0, std::slice::from_ref(&range)); + assert!(!lookup.is_fully_cached()); + assert_eq!(vec![0..10], lookup.missing_ranges); } #[test] diff --git a/src/mito2/src/sst/parquet.rs b/src/mito2/src/sst/parquet.rs index 48474d9fe4..bd0c1bced4 100644 --- a/src/mito2/src/sst/parquet.rs +++ b/src/mito2/src/sst/parquet.rs @@ -140,7 +140,7 @@ mod tests { use crate::access_layer::{FilePathProvider, Metrics, RegionFilePathFactory, WriteType}; use crate::cache::index::result_cache::PredicateKey; use crate::cache::test_util::assert_parquet_metadata_equal; - use crate::cache::{CacheManager, CacheStrategy, PageKey}; + use crate::cache::{CacheManager, CacheStrategy}; use crate::config::IndexConfig; use crate::read::FlatSource; use crate::region::options::{IndexOptions, InvertedIndexOptions}; @@ -340,11 +340,20 @@ mod tests { // Cache 4 row groups. for i in 0..4 { - let page_key = PageKey::new(handle.file_id().file_id(), i, get_ranges(i)); - assert!(cache.get_pages(&page_key).is_some()); + let lookup = cache + .get_page_ranges(handle.file_id().file_id(), i, &get_ranges(i)) + .unwrap(); + assert!(lookup.is_fully_cached()); } - let page_key = PageKey::new(handle.file_id().file_id(), 5, vec![]); - assert!(cache.get_pages(&page_key).is_none()); + let missing_range = 0..10; + let lookup = cache + .get_page_ranges( + handle.file_id().file_id(), + 5, + std::slice::from_ref(&missing_range), + ) + .unwrap(); + assert_eq!(vec![0..10], lookup.missing_ranges); } #[tokio::test] diff --git a/src/mito2/src/sst/parquet/push_decoder.rs b/src/mito2/src/sst/parquet/push_decoder.rs index c44813a5ab..43b64a436d 100644 --- a/src/mito2/src/sst/parquet/push_decoder.rs +++ b/src/mito2/src/sst/parquet/push_decoder.rs @@ -15,9 +15,8 @@ //! Push decoder stream implementation for SST parquet files. use std::ops::Range; -use std::sync::Arc; -use bytes::Bytes; +use bytes::{Bytes, BytesMut}; use datatypes::arrow::record_batch::RecordBatch; use futures::StreamExt; use futures::stream::BoxStream; @@ -26,11 +25,11 @@ use parquet::DecodeResult; use parquet::arrow::ProjectionMask; use parquet::arrow::arrow_reader::{ArrowReaderMetadata, RowSelection}; use parquet::arrow::push_decoder::ParquetPushDecoderBuilder; -use snafu::ResultExt; +use snafu::{ResultExt, ensure}; use crate::cache::file_cache::{FileType, IndexKey}; -use crate::cache::{CacheStrategy, PageKey, PageValue}; -use crate::error::{OpenDalSnafu, ReadParquetSnafu, Result}; +use crate::cache::{CacheStrategy, PageRangePart}; +use crate::error::{OpenDalSnafu, ReadParquetSnafu, Result, UnexpectedSnafu}; use crate::metrics::{READ_STAGE_ELAPSED, READ_STAGE_FETCH_PAGES}; use crate::sst::file::RegionFileId; use crate::sst::parquet::DEFAULT_READ_BATCH_SIZE; @@ -85,30 +84,44 @@ impl SstParquetRangeFetcher { .map(|_| std::time::Instant::now()); let _timer = READ_STAGE_FETCH_PAGES.start_timer(); - let page_key = PageKey::new( + let mut page_lookup = self.cache_strategy.get_page_ranges( self.region_file_id.file_id(), self.row_group_idx, - ranges.clone(), + &ranges, ); - - // Check page cache first. - if let Some(pages) = self.cache_strategy.get_pages(&page_key) { - if let Some(metrics) = &self.fetch_metrics { - let total_size: u64 = ranges.iter().map(|r| r.end - r.start).sum(); - let mut metrics_data = metrics.data.lock().unwrap(); - metrics_data.page_cache_hit += 1; - metrics_data.pages_to_fetch_mem += ranges.len(); - metrics_data.page_size_to_fetch_mem += total_size; - metrics_data.page_size_needed += total_size; - if let Some(start) = fetch_start { - metrics_data.total_fetch_elapsed += start.elapsed(); - } - } - return Ok(pages.compressed.clone()); + if let Some(lookup) = &page_lookup + && lookup.cached_bytes > 0 + && let Some(metrics) = &self.fetch_metrics + { + let mut metrics_data = metrics.data.lock().unwrap(); + metrics_data.page_cache_hit += 1; + metrics_data.pages_to_fetch_mem += lookup.cached_range_count; + metrics_data.page_size_to_fetch_mem += lookup.cached_bytes; + metrics_data.page_size_needed += lookup.cached_bytes; } + // Fast path: all requested ranges can be assembled from cached fragments. + if page_lookup + .as_ref() + .map(|lookup| lookup.is_fully_cached()) + .unwrap_or(false) + { + let lookup = page_lookup.take().unwrap(); + if let Some(metrics) = &self.fetch_metrics + && let Some(start) = fetch_start + { + metrics.data.lock().unwrap().total_fetch_elapsed += start.elapsed(); + } + return assemble_ranges(&ranges, lookup.cached_parts, &[]); + } + + let missing_ranges = page_lookup + .as_ref() + .map(|lookup| lookup.missing_ranges.clone()) + .unwrap_or_else(|| ranges.clone()); + // Calculate total range size for metrics. - let (total_range_size, unaligned_size) = compute_total_range_size(&ranges); + let (_, unaligned_size) = compute_total_range_size(&missing_ranges); // Check write cache. let key = IndexKey::new( @@ -121,21 +134,22 @@ impl SstParquetRangeFetcher { .as_ref() .map(|_| std::time::Instant::now()); let write_cache_result = match self.cache_strategy.write_cache() { - Some(cache) => cache.file_cache().read_ranges(key, &ranges).await, + Some(cache) => cache.file_cache().read_ranges(key, &missing_ranges).await, None => None, }; - let pages = match write_cache_result { + let fetched_pages = match write_cache_result { Some(data) => { if let Some(metrics) = &self.fetch_metrics { let elapsed = fetch_write_cache_start .map(|start| start.elapsed()) .unwrap_or_default(); - let range_size_needed: u64 = ranges.iter().map(|r| r.end - r.start).sum(); + let range_size_needed: u64 = + missing_ranges.iter().map(|r| r.end - r.start).sum(); let mut metrics_data = metrics.data.lock().unwrap(); metrics_data.write_cache_fetch_elapsed += elapsed; metrics_data.write_cache_hit += 1; - metrics_data.pages_to_fetch_write_cache += ranges.len(); + metrics_data.pages_to_fetch_write_cache += missing_ranges.len(); metrics_data.page_size_to_fetch_write_cache += unaligned_size; metrics_data.page_size_needed += range_size_needed; } @@ -151,37 +165,153 @@ impl SstParquetRangeFetcher { .fetch_metrics .as_ref() .map(|_| std::time::Instant::now()); - let data = fetch_byte_ranges(&self.file_path, self.object_store.clone(), &ranges) - .await - .context(OpenDalSnafu)?; + let data = + fetch_byte_ranges(&self.file_path, self.object_store.clone(), &missing_ranges) + .await + .context(OpenDalSnafu)?; if let Some(metrics) = &self.fetch_metrics { let elapsed = start.map(|start| start.elapsed()).unwrap_or_default(); - let range_size_needed: u64 = ranges.iter().map(|r| r.end - r.start).sum(); + let range_size_needed: u64 = + missing_ranges.iter().map(|r| r.end - r.start).sum(); let mut metrics_data = metrics.data.lock().unwrap(); metrics_data.store_fetch_elapsed += elapsed; metrics_data.cache_miss += 1; - metrics_data.pages_to_fetch_store += ranges.len(); + metrics_data.pages_to_fetch_store += missing_ranges.len(); metrics_data.page_size_to_fetch_store += unaligned_size; metrics_data.page_size_needed += range_size_needed; } data } }; + ensure!( + fetched_pages.len() == missing_ranges.len(), + UnexpectedSnafu { + reason: format!( + "Invalid parquet range fetch: {} missing ranges but {} fetched byte ranges", + missing_ranges.len(), + fetched_pages.len() + ), + } + ); - // Put pages back to the cache. - let page_value = PageValue::new(pages.clone(), total_range_size); - self.cache_strategy - .put_pages(page_key, Arc::new(page_value)); + self.cache_strategy.put_page_ranges( + self.region_file_id.file_id(), + self.row_group_idx, + &missing_ranges, + &fetched_pages, + ); if let (Some(metrics), Some(start)) = (&self.fetch_metrics, fetch_start) { metrics.data.lock().unwrap().total_fetch_elapsed += start.elapsed(); } - Ok(pages) + if let Some(lookup) = page_lookup { + let fetched_parts = missing_ranges + .into_iter() + .zip(fetched_pages) + .map(|(range, bytes)| PageRangePart { range, bytes }) + .collect::>(); + return assemble_ranges(&ranges, lookup.cached_parts, &fetched_parts); + } + + Ok(fetched_pages) } } +fn assemble_ranges( + ranges: &[Range], + cached_parts: Vec>, + fetched_parts: &[PageRangePart], +) -> Result> { + ensure!( + ranges.len() == cached_parts.len(), + UnexpectedSnafu { + reason: format!( + "Invalid parquet range assembly: {} requested ranges but {} cached part groups", + ranges.len(), + cached_parts.len() + ), + } + ); + + ranges + .iter() + .zip(cached_parts) + .map(|(range, mut parts)| { + parts.extend( + fetched_parts + .iter() + .filter_map(|part| overlapping_part(range, part)), + ); + assemble_range(range, parts) + }) + .collect() +} + +fn overlapping_part(range: &Range, part: &PageRangePart) -> Option { + let start = range.start.max(part.range.start); + let end = range.end.min(part.range.end); + if start >= end { + return None; + } + + let slice_start = (start - part.range.start) as usize; + let slice_end = (end - part.range.start) as usize; + Some(PageRangePart { + range: start..end, + bytes: part.bytes.slice(slice_start..slice_end), + }) +} + +fn assemble_range(range: &Range, mut parts: Vec) -> Result { + if range.start >= range.end { + return Ok(Bytes::new()); + } + + parts.sort_unstable_by_key(|part| part.range.start); + if parts.len() == 1 && parts[0].range == *range { + return Ok(parts.pop().unwrap().bytes); + } + + let mut cursor = range.start; + let mut output = BytesMut::with_capacity((range.end - range.start) as usize); + for part in parts { + ensure!( + part.range.start <= cursor, + UnexpectedSnafu { + reason: format!( + "Missing cached parquet bytes for range {}..{}, next part starts at {}", + range.start, range.end, part.range.start + ), + } + ); + if part.range.end <= cursor { + continue; + } + + let slice_start = (cursor - part.range.start) as usize; + let slice_end = (part.range.end.min(range.end) - part.range.start) as usize; + output.extend_from_slice(&part.bytes.slice(slice_start..slice_end)); + cursor = part.range.end.min(range.end); + if cursor >= range.end { + break; + } + } + + ensure!( + cursor == range.end, + UnexpectedSnafu { + reason: format!( + "Missing cached parquet bytes for range {}..{}, assembled through {}", + range.start, range.end, cursor + ), + } + ); + + Ok(output.freeze()) +} + /// Builds a parquet record batch stream driven directly by [ParquetPushDecoderBuilder]. pub(crate) fn build_sst_parquet_record_batch_stream( arrow_metadata: ArrowReaderMetadata, @@ -220,3 +350,55 @@ pub(crate) fn build_sst_parquet_record_batch_stream( } .boxed()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_assemble_range_from_cached_subrange_and_fetched_tail() { + let cached_parts = vec![vec![PageRangePart { + range: 400..500, + bytes: Bytes::from(vec![1; 100]), + }]]; + let fetched_parts = vec![PageRangePart { + range: 500..600, + bytes: Bytes::from(vec![2; 100]), + }]; + + let requested = 400..600; + let output = assemble_ranges( + std::slice::from_ref(&requested), + cached_parts, + &fetched_parts, + ) + .unwrap(); + assert_eq!(1, output.len()); + assert_eq!(vec![1; 100].as_slice(), &output[0][..100]); + assert_eq!(vec![2; 100].as_slice(), &output[0][100..]); + } + + #[test] + fn test_assemble_range_returns_single_covering_part_without_copy() { + let bytes = Bytes::from_static(b"abcdef"); + let cached_parts = vec![vec![PageRangePart { + range: 10..16, + bytes: bytes.clone(), + }]]; + + let requested = 10..16; + let output = assemble_ranges(std::slice::from_ref(&requested), cached_parts, &[]).unwrap(); + assert_eq!(bytes, output[0]); + } + + #[test] + fn test_assemble_range_clamps_overlapping_part_to_requested_end() { + let parts = vec![PageRangePart { + range: 0..10, + bytes: Bytes::from_static(b"0123456789"), + }]; + + let output = assemble_range(&(2..5), parts).unwrap(); + assert_eq!(Bytes::from_static(b"234"), output); + } +} From bb1de2a744f983521d2b1888a2c4bc470cfc09ca Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Thu, 4 Jun 2026 15:58:58 +0800 Subject: [PATCH 28/29] test: move CSV skip bad records coverage to integration (#8237) Signed-off-by: WenyXu --- tests-integration/src/tests/instance_test.rs | 40 +++++++++++++++++++ .../common/copy/copy_from_fs_csv.result | 22 ---------- .../common/copy/copy_from_fs_csv.sql | 9 ----- 3 files changed, 40 insertions(+), 31 deletions(-) diff --git a/tests-integration/src/tests/instance_test.rs b/tests-integration/src/tests/instance_test.rs index 3be45d8970..67f9975ba9 100644 --- a/tests-integration/src/tests/instance_test.rs +++ b/tests-integration/src/tests/instance_test.rs @@ -2348,6 +2348,46 @@ async fn test_cast_type_issue_1594(instance: Arc) { check_output_stream(output, expected).await; } +#[apply(both_instances_cases)] +async fn test_copy_from_csv_skip_bad_records(instance: Arc) { + let instance = instance.frontend(); + + assert!(matches!(execute_sql( + &instance, + "create table csv_skip_bad_records(host_id INT, host_name STRING, reading_value DOUBLE, ts TIMESTAMP TIME INDEX, PRIMARY KEY(host_id));", + ) + .await.data, OutputData::AffectedRows(0))); + + let filepath = find_testing_resource("/tests/data/csv/skip_bad_records.csv"); + + let output = execute_sql( + &instance, + &format!( + "copy csv_skip_bad_records from '{}' WITH(FORMAT='csv', skip_bad_records='true');", + &filepath + ), + ) + .await + .data; + + assert!(matches!(output, OutputData::AffectedRows(2))); + + let output = execute_sql( + &instance, + "select * from csv_skip_bad_records order by host_id;", + ) + .await + .data; + let expected = "\ ++---------+-----------+---------------+---------------------+ +| host_id | host_name | reading_value | ts | ++---------+-----------+---------------+---------------------+ +| 1 | Alice | 10.5 | 2024-01-01T00:00:00 | +| 2 | Bob | 30.5 | 2024-01-01T00:00:02 | ++---------+-----------+---------------+---------------------+"; + check_output_stream(output, expected).await; +} + #[apply(both_instances_cases)] async fn test_information_schema_dot_tables(instance: Arc) { let instance = instance.frontend(); diff --git a/tests/cases/standalone/common/copy/copy_from_fs_csv.result b/tests/cases/standalone/common/copy/copy_from_fs_csv.result index ad81cf1f77..2e3373af15 100644 --- a/tests/cases/standalone/common/copy/copy_from_fs_csv.result +++ b/tests/cases/standalone/common/copy/copy_from_fs_csv.result @@ -183,24 +183,6 @@ select * from csv_null_prefix_import; | final | 2023-11-14T22:13:23 | +-------+---------------------+ -CREATE TABLE csv_skip_bad_records(host_id int, host_name string, reading_value double, ts timestamp time index); - -Affected Rows: 0 - --- SQLNESS ENV PWD -Copy csv_skip_bad_records FROM '$PWD/tests/data/csv/skip_bad_records.csv' WITH (format='csv', skip_bad_records='true'); - -Affected Rows: 2 - -select * from csv_skip_bad_records order by ts; - -+---------+-----------+---------------+---------------------+ -| host_id | host_name | reading_value | ts | -+---------+-----------+---------------+---------------------+ -| 1 | Alice | 10.5 | 2024-01-01T00:00:00 | -| 2 | Bob | 30.5 | 2024-01-01T00:00:02 | -+---------+-----------+---------------+---------------------+ - drop table demo; Affected Rows: 0 @@ -237,7 +219,3 @@ drop table csv_null_prefix_import; Affected Rows: 0 -drop table csv_skip_bad_records; - -Affected Rows: 0 - diff --git a/tests/cases/standalone/common/copy/copy_from_fs_csv.sql b/tests/cases/standalone/common/copy/copy_from_fs_csv.sql index cd6cfcbd23..0901048177 100644 --- a/tests/cases/standalone/common/copy/copy_from_fs_csv.sql +++ b/tests/cases/standalone/common/copy/copy_from_fs_csv.sql @@ -73,13 +73,6 @@ Copy csv_null_prefix_import FROM '${SQLNESS_HOME}/demo/export/csv_null_prefix.cs select * from csv_null_prefix_import; -CREATE TABLE csv_skip_bad_records(host_id int, host_name string, reading_value double, ts timestamp time index); - --- SQLNESS ENV PWD -Copy csv_skip_bad_records FROM '$PWD/tests/data/csv/skip_bad_records.csv' WITH (format='csv', skip_bad_records='true'); - -select * from csv_skip_bad_records order by ts; - drop table demo; drop table with_filename; @@ -97,5 +90,3 @@ drop table demo_with_less_columns; drop table csv_null_prefix; drop table csv_null_prefix_import; - -drop table csv_skip_bad_records; From edd563d2e1a47d102cb62cfdad6916e449f0e096 Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Thu, 4 Jun 2026 16:13:24 +0800 Subject: [PATCH 29/29] feat: join simplifier for promql binary op (#8211) * basic impl Signed-off-by: Ruihang Xia * refactor, remove shallow methods Signed-off-by: Ruihang Xia * more sqlness cases Signed-off-by: Ruihang Xia * clear ctx Signed-off-by: Ruihang Xia * prefix leaf aliase Signed-off-by: Ruihang Xia --------- Signed-off-by: Ruihang Xia --- src/query/src/promql/planner.rs | 795 +++++++++++++++++- .../common/promql/set_operation.result | 32 +- .../promql/tsid_binary_join_regression.result | 398 +++++++++ .../promql/tsid_binary_join_regression.sql | 118 +++ 4 files changed, 1317 insertions(+), 26 deletions(-) diff --git a/src/query/src/promql/planner.rs b/src/query/src/promql/planner.rs index 0dacc136e8..46740a8b36 100644 --- a/src/query/src/promql/planner.rs +++ b/src/query/src/promql/planner.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::{BTreeSet, HashSet, VecDeque}; +use std::collections::{BTreeSet, HashMap, HashSet, VecDeque}; use std::sync::Arc; use std::time::UNIX_EPOCH; @@ -127,6 +127,9 @@ const FIELD_COLUMN_MATCHER: &str = "__field__"; const SCHEMA_COLUMN_MATCHER: &str = "__schema__"; const DB_COLUMN_MATCHER: &str = "__database__"; +/// Prefix for generated binary island leaf aliases. +const BINARY_ISLAND_LEAF_ALIAS_PREFIX: &str = "__prom_v"; + /// Threshold for scatter scan mode const MAX_SCATTER_POINTS: i64 = 400; @@ -161,6 +164,176 @@ struct PromPlannerContext { range: Option, } +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct VectorLeafKey { + metric_name: String, + matchers: Vec<(String, String, String)>, + or_matchers: Vec>, + offset_ms: i128, + at: String, +} + +#[derive(Debug, Clone)] +struct IslandLeaf { + selector: VectorSelector, + display_table: String, +} + +#[derive(Debug, Clone)] +enum IslandExpr { + VectorLeaf(usize), + Scalar(DfExpr), + Unary { + input: Box, + }, + Binary { + op: TokenType, + lhs: Box, + rhs: Box, + }, +} + +impl IslandExpr { + fn try_new(expr: &PromExpr, env: &mut IslandCollectEnv) -> Option { + if let Some(expr) = PromPlanner::try_build_literal_expr(expr) { + return Some(Self::Scalar(expr)); + } + + match expr { + PromExpr::Paren(ParenExpr { expr }) => Self::try_new(expr, env), + PromExpr::VectorSelector(selector) => { + let leaf = env.intern_leaf(selector)?; + Some(Self::VectorLeaf(leaf)) + } + PromExpr::Unary(UnaryExpr { expr }) => { + let input = Self::try_new(expr, env)?; + Some(Self::Unary { + input: Box::new(input), + }) + } + PromExpr::Binary(PromBinaryExpr { + lhs, + rhs, + op, + modifier, + }) if matches!( + op.id(), + token::T_ADD + | token::T_SUB + | token::T_MUL + | token::T_DIV + | token::T_MOD + | token::T_POW + | token::T_ATAN2 + ) && modifier.as_ref().is_none_or(|modifier| { + !modifier.return_bool + && modifier.matching.is_none() + && matches!(modifier.card, VectorMatchCardinality::OneToOne) + }) => + { + let lhs = Self::try_new(lhs, env)?; + let rhs = Self::try_new(rhs, env)?; + Some(Self::Binary { + op: *op, + lhs: Box::new(lhs), + rhs: Box::new(rhs), + }) + } + _ => None, + } + } +} + +#[derive(Debug, Default)] +struct IslandCollectEnv { + leaf_by_key: HashMap, + leaves: Vec, + vector_occurrences: usize, +} + +#[derive(Debug)] +struct PlannedIslandLeaf { + plan: LogicalPlan, + ctx: PromPlannerContext, + alias: TableReference, + display_table: String, +} + +#[derive(Debug)] +struct IslandFieldExprs { + exprs: Vec, + names: Vec, + scalar: bool, +} + +impl VectorLeafKey { + fn from_selector(selector: &VectorSelector) -> Option { + let mut metric_name = selector.name.clone(); + let mut matchers = Vec::with_capacity(selector.matchers.matchers.len()); + let matcher_key = |matcher: &Matcher| { + ( + matcher.name.clone(), + matcher.op.to_string(), + matcher.value.clone(), + ) + }; + + for matcher in &selector.matchers.matchers { + if matcher.name == METRIC_NAME { + if matcher.op != MatchOp::Equal || metric_name.is_some() { + return None; + } + metric_name = Some(matcher.value.clone()); + } else { + matchers.push(matcher_key(matcher)); + } + } + matchers.sort(); + + let mut or_matchers = selector + .matchers + .or_matchers + .iter() + .map(|group| { + let mut group = group.iter().map(matcher_key).collect::>(); + group.sort(); + group + }) + .collect::>(); + or_matchers.sort(); + + Some(Self { + metric_name: metric_name?, + matchers, + or_matchers, + offset_ms: match &selector.offset { + Some(Offset::Pos(duration)) => duration.as_millis() as i128, + Some(Offset::Neg(duration)) => -(duration.as_millis() as i128), + None => 0, + }, + at: format!("{:?}", selector.at), + }) + } +} + +impl IslandCollectEnv { + fn intern_leaf(&mut self, selector: &VectorSelector) -> Option { + self.vector_occurrences += 1; + let key = VectorLeafKey::from_selector(selector)?; + if let Some(id) = self.leaf_by_key.get(&key) { + return Some(*id); + } + + let id = self.leaves.len(); + self.leaves.push(IslandLeaf { + selector: selector.clone(), + display_table: key.metric_name.clone(), + }); + self.leaf_by_key.insert(key, id); + Some(id) + } +} + impl PromPlannerContext { fn from_eval_stmt(stmt: &EvalStmt) -> Self { Self { @@ -607,11 +780,323 @@ impl PromPlanner { }) } + async fn try_plan_binary_island( + &mut self, + binary_expr: &PromBinaryExpr, + ) -> Result> { + let original_ctx = self.ctx.clone(); + let mut collect_env = IslandCollectEnv::default(); + let Some(island_expr) = + IslandExpr::try_new(&PromExpr::Binary(binary_expr.clone()), &mut collect_env) + else { + return Ok(None); + }; + + if collect_env.leaves.is_empty() + || collect_env.vector_occurrences <= collect_env.leaves.len() + { + return Ok(None); + } + + let mut planned_leaves = Vec::with_capacity(collect_env.leaves.len()); + for (idx, leaf) in collect_env.leaves.iter().enumerate() { + let plan = self + .prom_vector_selector_to_plan(&leaf.selector, false) + .await?; + let ctx = self.ctx.clone(); + let alias = TableReference::bare(format!("{BINARY_ISLAND_LEAF_ALIAS_PREFIX}{idx}")); + let plan = LogicalPlanBuilder::from(plan) + .alias(alias.clone()) + .context(DataFusionPlanningSnafu)? + .build() + .context(DataFusionPlanningSnafu)?; + planned_leaves.push(PlannedIslandLeaf { + plan, + ctx, + alias, + display_table: leaf.display_table.clone(), + }); + } + + if !Self::binary_island_join_contexts_supported(&planned_leaves) { + self.ctx = original_ctx; + return Ok(None); + } + + let mut input = planned_leaves[0].plan.clone(); + for right_idx in 1..planned_leaves.len() { + input = self.join_binary_island_leaf( + input, + &planned_leaves[0], + &planned_leaves[right_idx], + )?; + } + + let field_exprs = + Self::build_binary_island_field_exprs(&island_expr, &planned_leaves, input.schema())?; + if field_exprs.scalar || field_exprs.exprs.is_empty() { + self.ctx = original_ctx; + return Ok(None); + } + + let plan = self.project_binary_island( + input, + &planned_leaves[0].alias, + &planned_leaves[0].ctx, + field_exprs, + )?; + Ok(Some(plan)) + } + + fn binary_island_join_contexts_supported(leaves: &[PlannedIslandLeaf]) -> bool { + if leaves + .iter() + .any(|leaf| leaf.ctx.time_index_column.is_none()) + { + return false; + } + + if leaves.len() <= 1 { + return true; + } + + let first_tags = leaves[0].ctx.tag_columns.iter().collect::>(); + + leaves.iter().skip(1).all(|leaf| { + (Self::plan_has_tsid_column(&leaves[0].plan) && Self::plan_has_tsid_column(&leaf.plan)) + || leaf.ctx.tag_columns.iter().collect::>() == first_tags + }) + } + + fn join_binary_island_leaf( + &self, + left: LogicalPlan, + first_leaf: &PlannedIslandLeaf, + right_leaf: &PlannedIslandLeaf, + ) -> Result { + let only_join_time_index = + first_leaf.ctx.tag_columns.is_empty() || right_leaf.ctx.tag_columns.is_empty(); + let (mut left_keys, mut right_keys) = Self::binary_join_key_columns( + &left, + &right_leaf.plan, + &first_leaf.ctx, + &right_leaf.ctx, + only_join_time_index, + &None, + ); + + if let (Some(left_time_index_column), Some(right_time_index_column)) = ( + first_leaf.ctx.time_index_column.clone(), + right_leaf.ctx.time_index_column.clone(), + ) { + left_keys.insert(left_time_index_column); + right_keys.insert(right_time_index_column); + } + + LogicalPlanBuilder::from(left) + .join_detailed( + right_leaf.plan.clone(), + JoinType::Inner, + ( + left_keys + .into_iter() + .map(|name| Column::new(Some(first_leaf.alias.clone()), name)) + .collect::>(), + right_keys + .into_iter() + .map(|name| Column::new(Some(right_leaf.alias.clone()), name)) + .collect::>(), + ), + None, + NullEquality::NullEqualsNull, + ) + .context(DataFusionPlanningSnafu)? + .build() + .context(DataFusionPlanningSnafu) + } + + fn build_binary_island_field_exprs( + expr: &IslandExpr, + leaves: &[PlannedIslandLeaf], + schema: &DFSchemaRef, + ) -> Result { + match expr { + IslandExpr::VectorLeaf(id) => { + let leaf = &leaves[*id]; + let exprs = leaf + .ctx + .field_columns + .iter() + .map(|field| { + schema + .qualified_field_with_name(Some(&leaf.alias), field) + .context(DataFusionPlanningSnafu) + .map(|field| DfExpr::Column(field.into())) + }) + .collect::>>()?; + let names = leaf + .ctx + .field_columns + .iter() + .map(|field| format!("{}.{}", leaf.display_table, field)) + .collect(); + Ok(IslandFieldExprs { + exprs, + names, + scalar: false, + }) + } + IslandExpr::Scalar(expr) => Ok(IslandFieldExprs { + exprs: vec![expr.clone()], + names: vec![expr.schema_name().to_string()], + scalar: true, + }), + IslandExpr::Unary { input } => { + let input = Self::build_binary_island_field_exprs(input, leaves, schema)?; + let mut exprs = Vec::with_capacity(input.exprs.len()); + let mut names = Vec::with_capacity(input.names.len()); + for (expr, name) in input.exprs.into_iter().zip(input.names) { + exprs.push(DfExpr::Negative(Box::new(expr))); + names.push(format!("-{name}")); + } + Ok(IslandFieldExprs { + exprs, + names, + scalar: input.scalar, + }) + } + IslandExpr::Binary { op, lhs, rhs } => { + let same_leaf = match (&**lhs, &**rhs) { + (IslandExpr::VectorLeaf(left), IslandExpr::VectorLeaf(right)) + if left == right => + { + Some(*left) + } + _ => None, + }; + let lhs = Self::build_binary_island_field_exprs(lhs, leaves, schema)?; + let rhs = Self::build_binary_island_field_exprs(rhs, leaves, schema)?; + let expr_builder = Self::prom_token_to_binary_expr_builder(*op)?; + let scalar = lhs.scalar && rhs.scalar; + let op = op.to_string(); + + let (exprs, names) = match (lhs.scalar, rhs.scalar) { + (true, true) => { + let expr = expr_builder(lhs.exprs[0].clone(), rhs.exprs[0].clone())?; + let name = format!("{} {op} {}", lhs.names[0], rhs.names[0]); + (vec![expr], vec![name]) + } + (true, false) => { + let mut exprs = Vec::with_capacity(rhs.exprs.len()); + let mut names = Vec::with_capacity(rhs.names.len()); + for (rhs_expr, rhs_name) in rhs.exprs.into_iter().zip(rhs.names) { + exprs.push(expr_builder(lhs.exprs[0].clone(), rhs_expr)?); + names.push(format!("{} {op} {rhs_name}", lhs.names[0])); + } + (exprs, names) + } + (false, true) => { + let mut exprs = Vec::with_capacity(lhs.exprs.len()); + let mut names = Vec::with_capacity(lhs.names.len()); + for (lhs_expr, lhs_name) in lhs.exprs.into_iter().zip(lhs.names) { + exprs.push(expr_builder(lhs_expr, rhs.exprs[0].clone())?); + names.push(format!("{lhs_name} {op} {}", rhs.names[0])); + } + (exprs, names) + } + (false, false) => { + let mut exprs = Vec::new(); + let mut names = Vec::new(); + for (idx, ((lhs_expr, rhs_expr), (mut lhs_name, mut rhs_name))) in lhs + .exprs + .into_iter() + .zip(rhs.exprs) + .zip(lhs.names.into_iter().zip(rhs.names)) + .enumerate() + { + if let Some(leaf) = same_leaf { + let field = leaves[leaf] + .ctx + .field_columns + .get(idx) + .cloned() + .unwrap_or_else(|| lhs_name.clone()); + lhs_name = format!("lhs.{field}"); + rhs_name = format!("rhs.{field}"); + } + exprs.push(expr_builder(lhs_expr, rhs_expr)?); + names.push(format!("{lhs_name} {op} {rhs_name}")); + } + (exprs, names) + } + }; + + Ok(IslandFieldExprs { + exprs, + names, + scalar, + }) + } + } + } + + fn project_binary_island( + &mut self, + input: LogicalPlan, + base_alias: &TableReference, + base_ctx: &PromPlannerContext, + field_exprs: IslandFieldExprs, + ) -> Result { + self.ctx = base_ctx.clone(); + + let schema = input.schema(); + let non_field_exprs = base_ctx + .tag_columns + .iter() + .chain(base_ctx.time_index_column.iter()) + .map(|column| { + schema + .qualified_field_with_name(Some(base_alias), column) + .context(DataFusionPlanningSnafu) + .map(|field| DfExpr::Column(field.into())) + }); + let tsid_expr = Self::optional_tsid_projection(schema, Some(base_alias), base_ctx.use_tsid) + .into_iter() + .map(Ok); + + self.ctx.field_columns = field_exprs.names; + let field_exprs = field_exprs + .exprs + .into_iter() + .zip(self.ctx.field_columns.iter()) + .map(|(expr, name)| Ok(DfExpr::Alias(Alias::new(expr, None::, name)))); + + let project_exprs = non_field_exprs + .chain(tsid_expr) + .chain(field_exprs) + .collect::>>()?; + + let plan = LogicalPlanBuilder::from(input) + .project(project_exprs) + .context(DataFusionPlanningSnafu)? + .build() + .context(DataFusionPlanningSnafu)?; + + self.ctx.table_name = None; + self.ctx.schema_name = None; + + Ok(plan) + } + async fn prom_binary_expr_to_plan( &mut self, query_engine_state: &QueryEngineState, binary_expr: &PromBinaryExpr, ) -> Result { + if let Some(plan) = self.try_plan_binary_island(binary_expr).await? { + return Ok(plan); + } + let PromBinaryExpr { lhs, rhs, @@ -3493,9 +3978,10 @@ impl PromPlanner { } fn binary_join_key_columns( - &self, left: &LogicalPlan, right: &LogicalPlan, + left_ctx: &PromPlannerContext, + right_ctx: &PromPlannerContext, only_join_time_index: bool, modifier: &Option, ) -> (BTreeSet, BTreeSet) { @@ -3516,13 +4002,21 @@ impl PromPlanner { let left_tag_columns = if only_join_time_index { BTreeSet::new() } else { - self.ctx + left_ctx + .tag_columns + .iter() + .cloned() + .collect::>() + }; + let right_tag_columns = if only_join_time_index { + BTreeSet::new() + } else { + right_ctx .tag_columns .iter() .cloned() .collect::>() }; - let right_tag_columns = left_tag_columns.clone(); (left_tag_columns, right_tag_columns) }; @@ -3562,8 +4056,14 @@ impl PromPlanner { only_join_time_index: bool, modifier: &Option, ) -> Result { - let (mut left_tag_columns, mut right_tag_columns) = - self.binary_join_key_columns(&left, &right, only_join_time_index, modifier); + let (mut left_tag_columns, mut right_tag_columns) = Self::binary_join_key_columns( + &left, + &right, + &self.ctx, + &self.ctx, + only_join_time_index, + modifier, + ); // push time index column if it exists if let (Some(left_time_index_column), Some(right_time_index_column)) = @@ -4974,7 +5474,7 @@ mod test { } #[tokio::test] - async fn repeated_tsid_binary_operand_keeps_tsid_join_keys() { + async fn repeated_tsid_binary_operand_reuses_leaf_plan() { let eval_stmt = build_eval_stmt("((some_metric - some_alt_metric) / some_metric) * 100"); let table_provider = build_test_table_provider_with_tsid( @@ -4995,12 +5495,24 @@ mod test { .unwrap(); let plan_str = plan.display_indent_schema().to_string(); - assert_eq!(plan_str.matches("__tsid =").count(), 2, "{plan_str}"); + assert_eq!(plan_str.matches("__tsid =").count(), 1, "{plan_str}"); + assert_eq!( + plan_str + .matches("Filter: phy.__table_id = UInt32(1024)") + .count(), + 1, + "{plan_str}" + ); + assert_eq!( + plan_str.matches("PromInstantManipulate").count(), + 2, + "{plan_str}" + ); assert!(!plan_str.contains("tag_0 ="), "{plan_str}"); } #[tokio::test] - async fn repeated_tsid_binary_operand_keeps_shorter_field_side() { + async fn repeated_tsid_binary_operand_reuses_shorter_field_side() { let eval_stmt = build_eval_stmt("((two_field_metric - one_field_metric) / one_field_metric) * 100"); @@ -5043,10 +5555,273 @@ mod test { .count(); assert_eq!(value_columns, 1, "{field_names:?}"); let plan_str = plan.display_indent_schema().to_string(); - assert_eq!(plan_str.matches("__tsid =").count(), 2, "{plan_str}"); + assert_eq!(plan_str.matches("__tsid =").count(), 1, "{plan_str}"); + assert_eq!( + plan_str + .matches("Filter: phy.__table_id = UInt32(1025)") + .count(), + 1, + "{plan_str}" + ); assert!(!plan_str.contains("tag_0 ="), "{plan_str}"); } + #[tokio::test] + async fn binary_island_reuses_self_operand_without_join() { + let eval_stmt = build_eval_stmt("some_metric / some_metric"); + + let table_provider = build_test_table_provider_with_tsid( + &[(DEFAULT_SCHEMA_NAME.to_string(), "some_metric".to_string())], + 1, + 1, + ) + .await; + let plan = + PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_query_engine_state()) + .await + .unwrap(); + + let plan_str = plan.display_indent_schema().to_string(); + assert_eq!(plan_str.matches("__tsid =").count(), 0, "{plan_str}"); + assert_eq!( + plan_str + .matches("Filter: phy.__table_id = UInt32(1024)") + .count(), + 1, + "{plan_str}" + ); + assert_eq!( + plan_str.matches("PromInstantManipulate").count(), + 1, + "{plan_str}" + ); + } + + #[tokio::test] + async fn binary_island_reuses_leaf_across_two_branches() { + let eval_stmt = + build_eval_stmt("(some_metric + some_alt_metric) / (some_metric + third_metric)"); + + let table_provider = build_test_table_provider_with_tsid( + &[ + (DEFAULT_SCHEMA_NAME.to_string(), "some_metric".to_string()), + ( + DEFAULT_SCHEMA_NAME.to_string(), + "some_alt_metric".to_string(), + ), + (DEFAULT_SCHEMA_NAME.to_string(), "third_metric".to_string()), + ], + 1, + 1, + ) + .await; + let plan = + PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_query_engine_state()) + .await + .unwrap(); + + let plan_str = plan.display_indent_schema().to_string(); + assert_eq!(plan_str.matches("__tsid =").count(), 2, "{plan_str}"); + assert_eq!( + plan_str + .matches("Filter: phy.__table_id = UInt32(1024)") + .count(), + 1, + "{plan_str}" + ); + assert_eq!( + plan_str.matches("PromInstantManipulate").count(), + 3, + "{plan_str}" + ); + } + + #[tokio::test] + async fn binary_island_generated_alias_avoids_user_column_names() { + let eval_stmt = build_eval_stmt("(some_metric + some_alt_metric) / some_metric"); + + let table_provider = build_test_table_provider_with_fields( + &[ + (DEFAULT_SCHEMA_NAME.to_string(), "some_metric".to_string()), + ( + DEFAULT_SCHEMA_NAME.to_string(), + "some_alt_metric".to_string(), + ), + ], + &["prom_v0", "__prom_v0"], + ) + .await; + let plan = + PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_query_engine_state()) + .await + .unwrap(); + + let field_names = plan.schema().field_names(); + assert!(field_names.iter().any(|name| name.ends_with(".prom_v0"))); + assert!(field_names.iter().any(|name| name.ends_with(".__prom_v0"))); + + let plan_str = plan.display_indent_schema().to_string(); + assert!(plan_str.contains("SubqueryAlias: __prom_v0"), "{plan_str}"); + assert_eq!( + plan_str.matches("PromInstantManipulate").count(), + 2, + "{plan_str}" + ); + } + + #[tokio::test] + async fn binary_island_clears_qualifier_for_nested_unary_projection() { + let eval_stmt = build_eval_stmt("-((some_metric + some_alt_metric) / some_metric)"); + + let table_provider = build_test_table_provider_with_tsid( + &[ + (DEFAULT_SCHEMA_NAME.to_string(), "some_metric".to_string()), + ( + DEFAULT_SCHEMA_NAME.to_string(), + "some_alt_metric".to_string(), + ), + ], + 1, + 1, + ) + .await; + let plan = + PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_query_engine_state()) + .await + .unwrap(); + + let plan_str = plan.display_indent_schema().to_string(); + assert_eq!(plan_str.matches("__tsid =").count(), 1, "{plan_str}"); + assert_eq!( + plan_str.matches("PromInstantManipulate").count(), + 2, + "{plan_str}" + ); + } + + #[tokio::test] + async fn binary_island_keeps_distinct_matcher_leaves() { + let eval_stmt = build_eval_stmt( + "(some_metric{tag_0=\"foo\"} + some_alt_metric) / some_metric{tag_0=\"bar\"}", + ); + + let table_provider = build_test_table_provider_with_tsid( + &[ + (DEFAULT_SCHEMA_NAME.to_string(), "some_metric".to_string()), + ( + DEFAULT_SCHEMA_NAME.to_string(), + "some_alt_metric".to_string(), + ), + ], + 1, + 1, + ) + .await; + let plan = + PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_query_engine_state()) + .await + .unwrap(); + + let plan_str = plan.display_indent_schema().to_string(); + assert_eq!(plan_str.matches("__tsid =").count(), 2, "{plan_str}"); + assert_eq!( + plan_str.matches("PromInstantManipulate").count(), + 3, + "{plan_str}" + ); + } + + #[tokio::test] + async fn binary_island_keeps_offset_leaves_distinct() { + let eval_stmt = build_eval_stmt("(some_metric offset 5m + some_alt_metric) / some_metric"); + + let table_provider = build_test_table_provider_with_tsid( + &[ + (DEFAULT_SCHEMA_NAME.to_string(), "some_metric".to_string()), + ( + DEFAULT_SCHEMA_NAME.to_string(), + "some_alt_metric".to_string(), + ), + ], + 1, + 1, + ) + .await; + let plan = + PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_query_engine_state()) + .await + .unwrap(); + + let plan_str = plan.display_indent_schema().to_string(); + assert_eq!(plan_str.matches("__tsid =").count(), 2, "{plan_str}"); + assert_eq!( + plan_str.matches("PromInstantManipulate").count(), + 3, + "{plan_str}" + ); + } + + #[tokio::test] + async fn binary_island_falls_back_for_group_modifier() { + let eval_stmt = build_eval_stmt( + "(some_metric + ignoring(tag_0) group_left some_alt_metric) / some_metric", + ); + + let table_provider = build_test_table_provider_with_tsid( + &[ + (DEFAULT_SCHEMA_NAME.to_string(), "some_metric".to_string()), + ( + DEFAULT_SCHEMA_NAME.to_string(), + "some_alt_metric".to_string(), + ), + ], + 1, + 1, + ) + .await; + let plan = + PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_query_engine_state()) + .await + .unwrap(); + + let plan_str = plan.display_indent_schema().to_string(); + assert_eq!( + plan_str.matches("PromInstantManipulate").count(), + 3, + "{plan_str}" + ); + } + + #[tokio::test] + async fn binary_island_falls_back_for_comparison_filter() { + let eval_stmt = build_eval_stmt("(some_metric > some_alt_metric) / some_metric"); + + let table_provider = build_test_table_provider_with_tsid( + &[ + (DEFAULT_SCHEMA_NAME.to_string(), "some_metric".to_string()), + ( + DEFAULT_SCHEMA_NAME.to_string(), + "some_alt_metric".to_string(), + ), + ], + 1, + 1, + ) + .await; + let plan = + PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &build_query_engine_state()) + .await + .unwrap(); + + let plan_str = plan.display_indent_schema().to_string(); + assert_eq!(plan_str.matches("__tsid =").count(), 2, "{plan_str}"); + assert_eq!( + plan_str.matches("PromInstantManipulate").count(), + 3, + "{plan_str}" + ); + } + #[tokio::test] async fn tsid_binary_join_uses_shorter_field_side() { let eval_stmt = build_eval_stmt("one_field_metric / two_field_metric"); diff --git a/tests/cases/standalone/common/promql/set_operation.result b/tests/cases/standalone/common/promql/set_operation.result index 5bb1d04d8b..83af6fc875 100644 --- a/tests/cases/standalone/common/promql/set_operation.result +++ b/tests/cases/standalone/common/promql/set_operation.result @@ -616,14 +616,14 @@ Affected Rows: 4 -- SQLNESS SORT_RESULT 3 1 tql eval (3, 4, '1s') cache_hit / (cache_miss + cache_hit); -+-------+---------------------+-------------------------------------------------------------------------------+ -| job | ts | lhs.greptime_value / rhs.cache_miss.greptime_value + cache_hit.greptime_value | -+-------+---------------------+-------------------------------------------------------------------------------+ -| read | 1970-01-01T00:00:03 | 0.5 | -| read | 1970-01-01T00:00:04 | 0.75 | -| write | 1970-01-01T00:00:03 | 0.5 | -| write | 1970-01-01T00:00:04 | 0.6666666666666666 | -+-------+---------------------+-------------------------------------------------------------------------------+ ++-------+---------------------+---------------------------------------------------------------------------------+ +| job | ts | cache_hit.greptime_value / cache_miss.greptime_value + cache_hit.greptime_value | ++-------+---------------------+---------------------------------------------------------------------------------+ +| read | 1970-01-01T00:00:03 | 0.5 | +| read | 1970-01-01T00:00:04 | 0.75 | +| write | 1970-01-01T00:00:03 | 0.5 | +| write | 1970-01-01T00:00:04 | 0.6666666666666666 | ++-------+---------------------+---------------------------------------------------------------------------------+ drop table cache_hit; @@ -672,14 +672,14 @@ Affected Rows: 4 -- SQLNESS SORT_RESULT 3 1 tql eval (3, 4, '1s') cache_hit_with_null_label / (cache_miss_with_null_label + cache_hit_with_null_label); -+-------+------------+---------------------+---------------------------------------------------------------------------------------------------------------+ -| job | null_label | ts | lhs.greptime_value / rhs.cache_miss_with_null_label.greptime_value + cache_hit_with_null_label.greptime_value | -+-------+------------+---------------------+---------------------------------------------------------------------------------------------------------------+ -| read | | 1970-01-01T00:00:03 | 0.5 | -| read | | 1970-01-01T00:00:04 | 0.75 | -| write | | 1970-01-01T00:00:03 | 0.5 | -| write | | 1970-01-01T00:00:04 | 0.6666666666666666 | -+-------+------------+---------------------+---------------------------------------------------------------------------------------------------------------+ ++-------+------------+---------------------+---------------------------------------------------------------------------------------------------------------------------------+ +| job | null_label | ts | cache_hit_with_null_label.greptime_value / cache_miss_with_null_label.greptime_value + cache_hit_with_null_label.greptime_value | ++-------+------------+---------------------+---------------------------------------------------------------------------------------------------------------------------------+ +| read | | 1970-01-01T00:00:03 | 0.5 | +| read | | 1970-01-01T00:00:04 | 0.75 | +| write | | 1970-01-01T00:00:03 | 0.5 | +| write | | 1970-01-01T00:00:04 | 0.6666666666666666 | ++-------+------------+---------------------+---------------------------------------------------------------------------------------------------------------------------------+ -- SQLNESS SORT_RESULT 3 1 tql eval (3, 4, '1s') cache_hit_with_null_label / ignoring(null_label) (cache_miss_with_null_label + ignoring(null_label) cache_hit_with_null_label); diff --git a/tests/cases/standalone/common/promql/tsid_binary_join_regression.result b/tests/cases/standalone/common/promql/tsid_binary_join_regression.result index d414eb6bba..722360e6b1 100644 --- a/tests/cases/standalone/common/promql/tsid_binary_join_regression.result +++ b/tests/cases/standalone/common/promql/tsid_binary_join_regression.result @@ -38,6 +38,21 @@ WITH( Affected Rows: 0 +CREATE TABLE tsid_binary_join_third ( + host STRING NULL, + job STRING NULL, + ts TIMESTAMP(3) NOT NULL, + greptime_value DOUBLE NULL, + TIME INDEX (ts), + PRIMARY KEY(host, job), +) +ENGINE = metric +WITH( + on_physical_table = 'tsid_binary_join_physical' +); + +Affected Rows: 0 + INSERT INTO tsid_binary_join_left (host, job, ts, greptime_value) VALUES ('host1', 'job1', 0, 12), ('host2', 'job2', 0, 18), @@ -54,6 +69,14 @@ INSERT INTO tsid_binary_join_right (host, job, ts, greptime_value) VALUES Affected Rows: 4 +INSERT INTO tsid_binary_join_third (host, job, ts, greptime_value) VALUES + ('host1', 'job1', 0, 2), + ('host2', 'job2', 0, 3), + ('host1', 'job1', 5000, 4), + ('host2', 'job2', 5000, 6); + +Affected Rows: 4 + -- Default vector-vector arithmetic should join on `__tsid` and time index. -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED @@ -93,6 +116,95 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / tsid_binary_join_right; |_|_| Total rows: 4_| +-+-+-+ +-- Repeated operands in a safe arithmetic island should be planned once and reused +-- in the final projection. +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE Hash\(\[__tsid@1,\sts@2\],.* Hash([__tsid@1, ts@2],REDACTED +-- SQLNESS REPLACE Hash\(\[__tsid@3,\sts@4\],.* Hash([__tsid@3, ts@4],REDACTED +-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED +-- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left + tsid_binary_join_right) / tsid_binary_join_left; + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_ProjectionExec: expr=[host@1 as host, job@2 as job, ts@4 as ts, __tsid@3 as __tsid, (greptime_value@0 + greptime_value@5) / greptime_value@0 as tsid_binary_join_left.greptime_value + tsid_binary_join_right.greptime_value / tsid_binary_join_left.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@3, __tsid@1), (ts@4, ts@2)], projection=[greptime_value@0, host@1, job@2, __tsid@3, ts@4, greptime_value@5], NullsEqual: true REDACTED +|_|_|_RepartitionExec: partitioning=Hash([__tsid@3, ts@4],REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_RepartitionExec: partitioning=Hash([__tsid@1, ts@2],REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +|_|_| Total rows: 4_| ++-+-+-+ + +-- A larger arithmetic island should still plan each distinct vector selector only once +-- while reusing the repeated left operand in multiple branches. +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED +-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED +-- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left + tsid_binary_join_right) * (tsid_binary_join_left - tsid_binary_join_third)) / (tsid_binary_join_left + 2); + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_ProjectionExec: expr=[host@1 as host, job@2 as job, ts@4 as ts, __tsid@3 as __tsid, (greptime_value@0 + greptime_value@5) * (greptime_value@0 - greptime_value@6) / (greptime_value@0 + 2) as tsid_binary_join_left.greptime_value + tsid_binary_join_right.greptime_value * tsid_binary_join_left.greptime_value - tsid_binary_join_third.greptime_value / tsid_binary_join_left.greptime_value + Float64(2)] REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@3, __tsid@1), (ts@4, ts@2)], projection=[greptime_value@0, host@1, job@2, __tsid@3, ts@4, greptime_value@5, greptime_value@6], NullsEqual: true REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@3, __tsid@1), (ts@4, ts@2)], projection=[greptime_value@0, host@1, job@2, __tsid@3, ts@4, greptime_value@5], NullsEqual: true REDACTED +|_|_|_RepartitionExec: partitioning=Hash([REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_RepartitionExec: partitioning=Hash([REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_RepartitionExec: partitioning=Hash([REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +|_|_| Total rows: 4_| ++-+-+-+ + -- Label modifiers must disable the TSID shortcut and keep matching on the remaining labels. -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED @@ -211,6 +323,218 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > bool tsid_binary_join_right; |_|_| Total rows: 4_| +-+-+-+ +-- Comparison filters are a barrier for binary island coalescing because they filter the +-- vector domain instead of producing only a value expression. +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED +-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED +-- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > tsid_binary_join_right) / tsid_binary_join_left) * 100; + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_ProjectionExec: expr=[host@2 as host, job@3 as job, ts@5 as ts, __tsid@4 as __tsid, greptime_value@0 / greptime_value@1 * 100 as .greptime_value / tsid_binary_join_left.greptime_value * Float64(100)] REDACTED +|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@2, __tsid@3), (ts@0, ts@4)], projection=[greptime_value@1, greptime_value@3, host@4, job@5, __tsid@6, ts@7], NullsEqual: true REDACTED +|_|_|_CoalescePartitionsExec REDACTED +|_|_|_ProjectionExec: expr=[ts@2 as ts, greptime_value@0 as greptime_value, __tsid@1 as __tsid] REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@2, ts@2)], filter=greptime_value@1 < greptime_value@0, projection=[greptime_value@0, __tsid@1, ts@2], NullsEqual: true REDACTED +|_|_|_RepartitionExec: partitioning=Hash([REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_RepartitionExec: partitioning=Hash([REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +|_|_| Total rows: 4_| ++-+-+-+ + +-- Bool comparisons are intentionally outside the first island optimization version. +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED +-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED +-- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > bool tsid_binary_join_right) + tsid_binary_join_left) / tsid_binary_join_left; + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_ProjectionExec: expr=[host@2 as host, job@3 as job, ts@5 as ts, __tsid@4 as __tsid, tsid_binary_join_right.tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value + tsid_binary_join_left.greptime_value@0 / greptime_value@1 as lhs.tsid_binary_join_right.tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value + tsid_binary_join_left.greptime_value / rhs.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@0, ts@4)], projection=[tsid_binary_join_right.tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value + tsid_binary_join_left.greptime_value@2, greptime_value@3, host@4, job@5, __tsid@6, ts@7], NullsEqual: true REDACTED +|_|_|_ProjectionExec: expr=[ts@3 as ts, __tsid@2 as __tsid, tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value@0 + greptime_value@1 as tsid_binary_join_right.tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value + tsid_binary_join_left.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@0, ts@2)], projection=[tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value@2, greptime_value@3, __tsid@4, ts@5], NullsEqual: true REDACTED +|_|_|_ProjectionExec: expr=[ts@3 as ts, __tsid@2 as __tsid, CAST(greptime_value@1 < greptime_value@0 AS Float64) as tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@2, ts@2)], projection=[greptime_value@0, greptime_value@3, __tsid@4, ts@5], NullsEqual: true REDACTED +|_|_|_RepartitionExec: partitioning=Hash([REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_RepartitionExec: partitioning=Hash([REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_RepartitionExec: partitioning=Hash([REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_RepartitionExec: partitioning=Hash([REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +|_|_| Total rows: 4_| ++-+-+-+ + +-- Set operators are a barrier because they have distinct matching and output-domain +-- semantics. +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED +-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED +-- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left or tsid_binary_join_right) / tsid_binary_join_left; + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_ProjectionExec: expr=[host@2 as host, job@3 as job, ts@5 as ts, __tsid@4 as __tsid, greptime_value@0 / greptime_value@1 as tsid_binary_join_right.greptime_value / tsid_binary_join_left.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@0, ts@4)], projection=[greptime_value@2, greptime_value@3, host@4, job@5, __tsid@6, ts@7], NullsEqual: true REDACTED +|_|_|_ProjectionExec: expr=[ts@0 as ts, __tsid@1 as __tsid, greptime_value@2 as greptime_value] REDACTED +|_|_|_UnionDistinctOnExec: on col=[["host", "job"]], ts_col=[ts] REDACTED +|_|_|_CoalescePartitionsExec REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_CoalescePartitionsExec REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_ProjectionExec: expr=[ts@4 as ts, __tsid@3 as __tsid, greptime_value@0 as greptime_value, host@1 as host, job@2 as job] REDACTED +|_|_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +| 1_| 0_|_ProjectionExec: expr=[ts@4 as ts, __tsid@3 as __tsid, greptime_value@0 as greptime_value, host@1 as host, job@2 as job] REDACTED +|_|_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +|_|_| Total rows: 4_| ++-+-+-+ + +-- Group modifiers are many-to-one/one-to-many matching barriers and must stay on the +-- legacy path. +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED +-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED +-- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left / ignoring(host) group_left tsid_binary_join_right) / tsid_binary_join_left; + ++-+-+-+ +| stage | node | plan_| ++-+-+-+ +| 0_| 0_|_ProjectionExec: expr=[host@2 as host, job@3 as job, ts@5 as ts, __tsid@4 as __tsid, tsid_binary_join_left.greptime_value / tsid_binary_join_right.greptime_value@0 / greptime_value@1 as tsid_binary_join_right.tsid_binary_join_left.greptime_value / tsid_binary_join_right.greptime_value / tsid_binary_join_left.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@0, ts@4)], projection=[tsid_binary_join_left.greptime_value / tsid_binary_join_right.greptime_value@2, greptime_value@3, host@4, job@5, __tsid@6, ts@7], NullsEqual: true REDACTED +|_|_|_RepartitionExec: partitioning=Hash([REDACTED +|_|_|_ProjectionExec: expr=[ts@3 as ts, __tsid@2 as __tsid, greptime_value@0 / greptime_value@1 as tsid_binary_join_left.greptime_value / tsid_binary_join_right.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(job@1, job@1), (ts@2, ts@3)], projection=[greptime_value@0, greptime_value@3, __tsid@5, ts@6], NullsEqual: true REDACTED +|_|_|_RepartitionExec: partitioning=Hash([REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, job@2 as job, ts@4 as ts] REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_RepartitionExec: partitioning=Hash([REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, job@2 as job, __tsid@3 as __tsid, ts@4 as ts] REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_RepartitionExec: partitioning=Hash([REDACTED +|_|_|_MergeScanExec: REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +| 1_| 0_|_PromInstantManipulateExec: range=[0..5000], lookback=[300000], interval=[5000], time index=[ts] REDACTED +|_|_|_PromSeriesDivideExec: tags=["__tsid"] REDACTED +|_|_|_ProjectionExec: expr=[greptime_value@1 as greptime_value, host@3 as host, job@4 as job, __tsid@2 as __tsid, ts@0 as ts] REDACTED +|_|_|_CooperativeExec REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":REDACTED, "distribution":"PerSeries" REDACTED +|_|_|_| +|_|_| Total rows: 4_| ++-+-+-+ + -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 5, '5s') tsid_binary_join_left / tsid_binary_join_right; @@ -223,6 +547,80 @@ TQL EVAL (0, 5, '5s') tsid_binary_join_left / tsid_binary_join_right; | host2 | job2 | 1970-01-01T00:00:05 | 3.0 | +-------+------+---------------------+------------------------------------------------------------------------------+ +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 5, '5s') (tsid_binary_join_left + tsid_binary_join_right) / tsid_binary_join_left; + ++-------+------+---------------------+---------------------------------------------------------------------------------------------------------------------+ +| host | job | ts | tsid_binary_join_left.greptime_value + tsid_binary_join_right.greptime_value / tsid_binary_join_left.greptime_value | ++-------+------+---------------------+---------------------------------------------------------------------------------------------------------------------+ +| host1 | job1 | 1970-01-01T00:00:00 | 1.25 | +| host1 | job1 | 1970-01-01T00:00:05 | 1.3333333333333333 | +| host2 | job2 | 1970-01-01T00:00:00 | 1.3333333333333333 | +| host2 | job2 | 1970-01-01T00:00:05 | 1.3333333333333333 | ++-------+------+---------------------+---------------------------------------------------------------------------------------------------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 5, '5s') ((tsid_binary_join_left + tsid_binary_join_right) * (tsid_binary_join_left - tsid_binary_join_third)) / (tsid_binary_join_left + 2); + ++-------+------+---------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| host | job | ts | tsid_binary_join_left.greptime_value + tsid_binary_join_right.greptime_value * tsid_binary_join_left.greptime_value - tsid_binary_join_third.greptime_value / tsid_binary_join_left.greptime_value + Float64(2) | ++-------+------+---------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| host1 | job1 | 1970-01-01T00:00:00 | 10.714285714285714 | +| host1 | job1 | 1970-01-01T00:00:05 | 12.941176470588236 | +| host2 | job2 | 1970-01-01T00:00:00 | 18.0 | +| host2 | job2 | 1970-01-01T00:00:05 | 18.26086956521739 | ++-------+------+---------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 5, '5s') ((tsid_binary_join_left > tsid_binary_join_right) / tsid_binary_join_left) * 100; + ++-------+------+---------------------+-----------------------------------------------------------------------+ +| host | job | ts | .greptime_value / tsid_binary_join_left.greptime_value * Float64(100) | ++-------+------+---------------------+-----------------------------------------------------------------------+ +| host1 | job1 | 1970-01-01T00:00:00 | 100.0 | +| host1 | job1 | 1970-01-01T00:00:05 | 100.0 | +| host2 | job2 | 1970-01-01T00:00:00 | 100.0 | +| host2 | job2 | 1970-01-01T00:00:05 | 100.0 | ++-------+------+---------------------+-----------------------------------------------------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 5, '5s') ((tsid_binary_join_left > bool tsid_binary_join_right) + tsid_binary_join_left) / tsid_binary_join_left; + ++-------+------+---------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| host | job | ts | lhs.tsid_binary_join_right.tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value + tsid_binary_join_left.greptime_value / rhs.greptime_value | ++-------+------+---------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| host1 | job1 | 1970-01-01T00:00:00 | 1.0833333333333333 | +| host1 | job1 | 1970-01-01T00:00:05 | 1.0666666666666667 | +| host2 | job2 | 1970-01-01T00:00:00 | 1.0555555555555556 | +| host2 | job2 | 1970-01-01T00:00:05 | 1.0476190476190477 | ++-------+------+---------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 5, '5s') (tsid_binary_join_left or tsid_binary_join_right) / tsid_binary_join_left; + ++-------+------+---------------------+------------------------------------------------------------------------------+ +| host | job | ts | tsid_binary_join_right.greptime_value / tsid_binary_join_left.greptime_value | ++-------+------+---------------------+------------------------------------------------------------------------------+ +| host1 | job1 | 1970-01-01T00:00:00 | 1.0 | +| host1 | job1 | 1970-01-01T00:00:05 | 1.0 | +| host2 | job2 | 1970-01-01T00:00:00 | 1.0 | +| host2 | job2 | 1970-01-01T00:00:05 | 1.0 | ++-------+------+---------------------+------------------------------------------------------------------------------+ + +-- Range functions are outside the first island version; the range selector subtree must +-- remain a barrier. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 5, '5s') rate(tsid_binary_join_left[5s]) / tsid_binary_join_left; + ++------+-----+----+----------------------------------------------------------------------------+ +| host | job | ts | lhs.prom_rate(ts_range,greptime_value,ts,Int64(5000)) / rhs.greptime_value | ++------+-----+----+----------------------------------------------------------------------------+ ++------+-----+----+----------------------------------------------------------------------------+ + +DROP TABLE tsid_binary_join_third; + +Affected Rows: 0 + DROP TABLE tsid_binary_join_right; Affected Rows: 0 diff --git a/tests/cases/standalone/common/promql/tsid_binary_join_regression.sql b/tests/cases/standalone/common/promql/tsid_binary_join_regression.sql index 61dfd23765..74ef2a2529 100644 --- a/tests/cases/standalone/common/promql/tsid_binary_join_regression.sql +++ b/tests/cases/standalone/common/promql/tsid_binary_join_regression.sql @@ -33,6 +33,19 @@ WITH( on_physical_table = 'tsid_binary_join_physical' ); +CREATE TABLE tsid_binary_join_third ( + host STRING NULL, + job STRING NULL, + ts TIMESTAMP(3) NOT NULL, + greptime_value DOUBLE NULL, + TIME INDEX (ts), + PRIMARY KEY(host, job), +) +ENGINE = metric +WITH( + on_physical_table = 'tsid_binary_join_physical' +); + INSERT INTO tsid_binary_join_left (host, job, ts, greptime_value) VALUES ('host1', 'job1', 0, 12), ('host2', 'job2', 0, 18), @@ -45,6 +58,12 @@ INSERT INTO tsid_binary_join_right (host, job, ts, greptime_value) VALUES ('host1', 'job1', 5000, 5), ('host2', 'job2', 5000, 7); +INSERT INTO tsid_binary_join_third (host, job, ts, greptime_value) VALUES + ('host1', 'job1', 0, 2), + ('host2', 'job2', 0, 3), + ('host1', 'job1', 5000, 4), + ('host2', 'job2', 5000, 6); + -- Default vector-vector arithmetic should join on `__tsid` and time index. -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED @@ -58,6 +77,33 @@ INSERT INTO tsid_binary_join_right (host, job, ts, greptime_value) VALUES -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED TQL ANALYZE (0, 5, '5s') tsid_binary_join_left / tsid_binary_join_right; +-- Repeated operands in a safe arithmetic island should be planned once and reused +-- in the final projection. +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE Hash\(\[__tsid@1,\sts@2\],.* Hash([__tsid@1, ts@2],REDACTED +-- SQLNESS REPLACE Hash\(\[__tsid@3,\sts@4\],.* Hash([__tsid@3, ts@4],REDACTED +-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED +-- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left + tsid_binary_join_right) / tsid_binary_join_left; + +-- A larger arithmetic island should still plan each distinct vector selector only once +-- while reusing the repeated left operand in multiple branches. +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED +-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED +-- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left + tsid_binary_join_right) * (tsid_binary_join_left - tsid_binary_join_third)) / (tsid_binary_join_left + 2); + -- Label modifiers must disable the TSID shortcut and keep matching on the remaining labels. -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED @@ -98,9 +144,81 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > tsid_binary_join_right; -- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > bool tsid_binary_join_right; +-- Comparison filters are a barrier for binary island coalescing because they filter the +-- vector domain instead of producing only a value expression. +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED +-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED +-- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > tsid_binary_join_right) / tsid_binary_join_left) * 100; + +-- Bool comparisons are intentionally outside the first island optimization version. +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED +-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED +-- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > bool tsid_binary_join_right) + tsid_binary_join_left) / tsid_binary_join_left; + +-- Set operators are a barrier because they have distinct matching and output-domain +-- semantics. +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED +-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED +-- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left or tsid_binary_join_right) / tsid_binary_join_left; + +-- Group modifiers are many-to-one/one-to-many matching barriers and must stay on the +-- legacy path. +-- SQLNESS REPLACE (metrics.*) REDACTED +-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED +-- SQLNESS REPLACE (-+) - +-- SQLNESS REPLACE (\s\s+) _ +-- SQLNESS REPLACE (peers.*) REDACTED +-- SQLNESS REPLACE Hash\(\[[^\]]+\],.* Hash([REDACTED +-- SQLNESS REPLACE input_partitions=\d+ input_partitions=REDACTED +-- SQLNESS REPLACE "partition_count":\{(.*?)\} "partition_count":REDACTED +-- SQLNESS REPLACE region=\d+\(\d+,\s+\d+\) region=REDACTED +TQL ANALYZE (0, 5, '5s') (tsid_binary_join_left / ignoring(host) group_left tsid_binary_join_right) / tsid_binary_join_left; + -- SQLNESS SORT_RESULT 3 1 TQL EVAL (0, 5, '5s') tsid_binary_join_left / tsid_binary_join_right; +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 5, '5s') (tsid_binary_join_left + tsid_binary_join_right) / tsid_binary_join_left; + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 5, '5s') ((tsid_binary_join_left + tsid_binary_join_right) * (tsid_binary_join_left - tsid_binary_join_third)) / (tsid_binary_join_left + 2); + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 5, '5s') ((tsid_binary_join_left > tsid_binary_join_right) / tsid_binary_join_left) * 100; + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 5, '5s') ((tsid_binary_join_left > bool tsid_binary_join_right) + tsid_binary_join_left) / tsid_binary_join_left; + +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 5, '5s') (tsid_binary_join_left or tsid_binary_join_right) / tsid_binary_join_left; + +-- Range functions are outside the first island version; the range selector subtree must +-- remain a barrier. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 5, '5s') rate(tsid_binary_join_left[5s]) / tsid_binary_join_left; + +DROP TABLE tsid_binary_join_third; DROP TABLE tsid_binary_join_right; DROP TABLE tsid_binary_join_left; DROP TABLE tsid_binary_join_physical;