From 6420082685b56221a40ea1d5763be18db85fcebe Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Thu, 12 Nov 2026 12:03:33 +0000 Subject: [PATCH] fix(meta): skip metric engine identity columns when merging partition columns (#10368) CreateLogicalTables copies every physical partition column into the logical schema as a nullable String tag. After a physical metric table is repartitioned by the internal `__table_id` column, that column reached the region create request and the metric engine rejected it with "Internal column __table_id is reserved", so no logical table could be created. Filter engine-owned identity columns while merging, reusing `is_metric_engine_internal_column`. Filtering at the merge point also protects procedures that persisted an unfiltered `physical_partition_columns` list. User partition columns keep the previous behavior. Write routing for a physical table partitioned by `__table_id` is still tracked by #10352; this change only unblocks logical table creation. Signed-off-by: WenyXu --- .../meta/src/ddl/create_logical_tables.rs | 6 +- .../src/ddl/create_logical_tables/metadata.rs | 9 ++ .../src/ddl/tests/create_logical_tables.rs | 141 +++++++++++++++++- 3 files changed, 154 insertions(+), 2 deletions(-) diff --git a/src/common/meta/src/ddl/create_logical_tables.rs b/src/common/meta/src/ddl/create_logical_tables.rs index 9b231f3c538..1e65a7ef724 100644 --- a/src/common/meta/src/ddl/create_logical_tables.rs +++ b/src/common/meta/src/ddl/create_logical_tables.rs @@ -97,7 +97,7 @@ impl CreateLogicalTablesProcedure { self.check_input_tasks()?; // Sets physical region numbers self.fill_physical_table_info().await?; - // Add partition columns from physical table to logical table schemas + // Add the physical table's user partition columns to logical table schemas. self.merge_partition_columns_into_logical_tables()?; // Checks if the tables exist self.check_tables_already_exist().await?; @@ -325,6 +325,10 @@ pub struct CreateTablesData { physical_table_id: TableId, physical_region_numbers: Vec, physical_columns: Vec, + /// Partition column names of the physical table. + /// + /// Procedures persisted before engine identity columns were filtered may store them + /// here, so the merge re-checks the names instead of trusting this list. physical_partition_columns: Vec, } diff --git a/src/common/meta/src/ddl/create_logical_tables/metadata.rs b/src/common/meta/src/ddl/create_logical_tables/metadata.rs index 22424668946..da9cbc1708f 100644 --- a/src/common/meta/src/ddl/create_logical_tables/metadata.rs +++ b/src/common/meta/src/ddl/create_logical_tables/metadata.rs @@ -18,6 +18,7 @@ use std::sync::Arc; use datatypes::prelude::ConcreteDataType; use datatypes::schema::{ColumnSchema, Schema}; use snafu::OptionExt; +use store_api::metric_engine_consts::is_metric_engine_internal_column; use crate::ddl::create_logical_tables::CreateLogicalTablesProcedure; use crate::error::Result; @@ -82,6 +83,14 @@ impl CreateLogicalTablesProcedure { // Add missing partition columns for partition_column in partition_columns { + // The engine owns its identity columns: they are derived from the logical + // table at write time, so a logical schema must never declare them. Older + // procedures may have persisted an unfiltered partition column list, hence + // filtering here rather than only while collecting it. + if is_metric_engine_internal_column(partition_column) { + continue; + } + if !existing_column_names.contains(partition_column) { let new_column_index = column_schemas.len() + new_columns.len(); diff --git a/src/common/meta/src/ddl/tests/create_logical_tables.rs b/src/common/meta/src/ddl/tests/create_logical_tables.rs index 89ec486ee03..a74ef63403e 100644 --- a/src/common/meta/src/ddl/tests/create_logical_tables.rs +++ b/src/common/meta/src/ddl/tests/create_logical_tables.rs @@ -23,11 +23,17 @@ use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; use common_procedure::{Context as ProcedureContext, Procedure, ProcedureId, Status}; use common_procedure_test::MockContextProvider; +use datatypes::prelude::ConcreteDataType; +use datatypes::schema::{ColumnSchema, Schema}; use store_api::metadata::ColumnMetadata; -use store_api::metric_engine_consts::{ALTER_PHYSICAL_EXTENSION_KEY, MANIFEST_INFO_EXTENSION_KEY}; +use store_api::metric_engine_consts::{ + ALTER_PHYSICAL_EXTENSION_KEY, DATA_SCHEMA_TABLE_ID_COLUMN_NAME, MANIFEST_INFO_EXTENSION_KEY, + is_metric_engine_internal_column, +}; use store_api::region_engine::RegionManifestInfo; use store_api::storage::RegionId; use store_api::storage::consts::ReservedColumnId; +use table::metadata::TableInfo; use tokio::sync::mpsc; use crate::ddl::TableMetadata; @@ -566,3 +572,136 @@ async fn test_on_submit_create_request() { }) ); } + +/// Appends a column to the table schema and returns its index. +fn push_column( + table_info: &mut TableInfo, + name: &str, + data_type: ConcreteDataType, + is_nullable: bool, +) -> usize { + let mut columns = table_info.meta.schema.column_schemas().to_vec(); + columns.push(ColumnSchema::new(name, data_type, is_nullable)); + table_info.meta.schema = Arc::new(Schema::new(columns)); + table_info.meta.schema.column_schemas().len() - 1 +} + +#[tokio::test] +async fn test_on_prepare_skips_internal_partition_columns() { + let (tx, mut rx) = mpsc::channel(8); + let node_manager = Arc::new(MockDatanodeManager::new(DatanodeWatcher::new(tx))); + let ddl_context = new_ddl_context(node_manager); + + // The physical table owns the engine identity column and has been repartitioned by + // both it and a user tag that no logical table declares. + let mut create_physical_table_task = test_create_physical_table_task("phy_table"); + let table_id_column_index = push_column( + &mut create_physical_table_task.table_info, + DATA_SCHEMA_TABLE_ID_COLUMN_NAME, + ConcreteDataType::uint32_datatype(), + false, + ); + let extra_tag_column_index = push_column( + &mut create_physical_table_task.table_info, + "extra_tag", + ConcreteDataType::string_datatype(), + true, + ); + create_physical_table_task + .table_info + .meta + .partition_key_indices = vec![extra_tag_column_index, table_id_column_index]; + + let TableMetadata { + table_id, + table_route, + .. + } = ddl_context + .table_metadata_allocator + .create(&create_physical_table_task) + .await + .unwrap(); + create_physical_table_task.set_table_id(table_id); + create_physical_table_metadata( + &ddl_context, + create_physical_table_task.table_info.clone(), + TableRouteValue::Physical(table_route), + ) + .await; + + let mut procedure = CreateLogicalTablesProcedure::new( + vec![test_create_logical_table_task("foo")], + table_id, + ddl_context.clone(), + ); + let status = procedure.on_prepare().await.unwrap(); + assert_matches!( + status, + Status::Executing { + persist: true, + clean_poisons: false + } + ); + + let ctx = ProcedureContext { + procedure_id: ProcedureId::random(), + provider: Arc::new(MockContextProvider::default()), + event_context: None, + }; + procedure.execute(&ctx).await.unwrap(); + let status = procedure.execute(&ctx).await.unwrap(); + let table_ids = status.downcast_output_ref::>().unwrap().clone(); + + // The engine rejects a request that occupies a reserved identity column. + let mut creates = None; + while let Ok((_, request)) = rx.try_recv() { + if let Some(region_request::Body::Creates(body)) = request.body { + creates = Some(body); + } + } + let creates = creates.unwrap(); + assert_eq!(creates.requests.len(), 1); + let requested_columns = creates.requests[0] + .column_defs + .iter() + .map(|def| def.column_def.as_ref().unwrap().name.as_str()) + .collect::>(); + assert!( + !requested_columns + .iter() + .any(|name| is_metric_engine_internal_column(name)), + "logical region request must not occupy internal columns: {requested_columns:?}" + ); + assert!(requested_columns.contains(&"extra_tag")); + + let logical_table_info = get_raw_table_info(&ddl_context, table_ids[0]).await; + let column_names = logical_table_info + .meta + .schema + .column_schemas() + .iter() + .map(|column| column.name.as_str()) + .collect::>(); + assert!( + !column_names.contains(&DATA_SCHEMA_TABLE_ID_COLUMN_NAME), + "internal identity column must not be merged into the logical schema: {column_names:?}" + ); + assert!(column_names.contains(&"extra_tag")); + + // The user partition column is still merged as a nullable string tag of the primary key. + let extra_tag_index = logical_table_info + .meta + .schema + .column_index_by_name("extra_tag") + .unwrap(); + assert!( + logical_table_info + .meta + .primary_key_indices + .contains(&extra_tag_index), + "merged partition column must be part of the primary key" + ); + let extra_tag = &logical_table_info.meta.schema.column_schemas()[extra_tag_index]; + assert_eq!(extra_tag.data_type, ConcreteDataType::string_datatype()); + assert!(extra_tag.is_nullable()); +}