fix(meta): skip metric engine identity columns when merging partition columns (#10368)

CreateLogicalTables copies every physical partition column into the logical
schema as a nullable String tag. After a physical metric table is repartitioned
by the internal `__table_id` column, that column reached the region create
request and the metric engine rejected it with "Internal column __table_id is
reserved", so no logical table could be created.

Filter engine-owned identity columns while merging, reusing
`is_metric_engine_internal_column`. Filtering at the merge point also protects
procedures that persisted an unfiltered `physical_partition_columns` list.
User partition columns keep the previous behavior.

Write routing for a physical table partitioned by `__table_id` is still
tracked by #10352; this change only unblocks logical table creation.

Signed-off-by: WenyXu <wenymedia@gmail.com>
This commit is contained in:
Weny Xu
2026-11-12 12:03:33 +00:00
committed by GitHub
parent c937e3997b
commit 6420082685
3 changed files with 154 additions and 2 deletions
@@ -97,7 +97,7 @@ impl CreateLogicalTablesProcedure {
self.check_input_tasks()?;
// Sets physical region numbers
self.fill_physical_table_info().await?;
// Add partition columns from physical table to logical table schemas
// Add the physical table's user partition columns to logical table schemas.
self.merge_partition_columns_into_logical_tables()?;
// Checks if the tables exist
self.check_tables_already_exist().await?;
@@ -325,6 +325,10 @@ pub struct CreateTablesData {
physical_table_id: TableId,
physical_region_numbers: Vec<RegionNumber>,
physical_columns: Vec<ColumnMetadata>,
/// Partition column names of the physical table.
///
/// Procedures persisted before engine identity columns were filtered may store them
/// here, so the merge re-checks the names instead of trusting this list.
physical_partition_columns: Vec<String>,
}
@@ -18,6 +18,7 @@ use std::sync::Arc;
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{ColumnSchema, Schema};
use snafu::OptionExt;
use store_api::metric_engine_consts::is_metric_engine_internal_column;
use crate::ddl::create_logical_tables::CreateLogicalTablesProcedure;
use crate::error::Result;
@@ -82,6 +83,14 @@ impl CreateLogicalTablesProcedure {
// Add missing partition columns
for partition_column in partition_columns {
// The engine owns its identity columns: they are derived from the logical
// table at write time, so a logical schema must never declare them. Older
// procedures may have persisted an unfiltered partition column list, hence
// filtering here rather than only while collecting it.
if is_metric_engine_internal_column(partition_column) {
continue;
}
if !existing_column_names.contains(partition_column) {
let new_column_index = column_schemas.len() + new_columns.len();
@@ -23,11 +23,17 @@ use common_error::ext::ErrorExt;
use common_error::status_code::StatusCode;
use common_procedure::{Context as ProcedureContext, Procedure, ProcedureId, Status};
use common_procedure_test::MockContextProvider;
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{ColumnSchema, Schema};
use store_api::metadata::ColumnMetadata;
use store_api::metric_engine_consts::{ALTER_PHYSICAL_EXTENSION_KEY, MANIFEST_INFO_EXTENSION_KEY};
use store_api::metric_engine_consts::{
ALTER_PHYSICAL_EXTENSION_KEY, DATA_SCHEMA_TABLE_ID_COLUMN_NAME, MANIFEST_INFO_EXTENSION_KEY,
is_metric_engine_internal_column,
};
use store_api::region_engine::RegionManifestInfo;
use store_api::storage::RegionId;
use store_api::storage::consts::ReservedColumnId;
use table::metadata::TableInfo;
use tokio::sync::mpsc;
use crate::ddl::TableMetadata;
@@ -566,3 +572,136 @@ async fn test_on_submit_create_request() {
})
);
}
/// Appends a column to the table schema and returns its index.
fn push_column(
table_info: &mut TableInfo,
name: &str,
data_type: ConcreteDataType,
is_nullable: bool,
) -> usize {
let mut columns = table_info.meta.schema.column_schemas().to_vec();
columns.push(ColumnSchema::new(name, data_type, is_nullable));
table_info.meta.schema = Arc::new(Schema::new(columns));
table_info.meta.schema.column_schemas().len() - 1
}
#[tokio::test]
async fn test_on_prepare_skips_internal_partition_columns() {
let (tx, mut rx) = mpsc::channel(8);
let node_manager = Arc::new(MockDatanodeManager::new(DatanodeWatcher::new(tx)));
let ddl_context = new_ddl_context(node_manager);
// The physical table owns the engine identity column and has been repartitioned by
// both it and a user tag that no logical table declares.
let mut create_physical_table_task = test_create_physical_table_task("phy_table");
let table_id_column_index = push_column(
&mut create_physical_table_task.table_info,
DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
ConcreteDataType::uint32_datatype(),
false,
);
let extra_tag_column_index = push_column(
&mut create_physical_table_task.table_info,
"extra_tag",
ConcreteDataType::string_datatype(),
true,
);
create_physical_table_task
.table_info
.meta
.partition_key_indices = vec![extra_tag_column_index, table_id_column_index];
let TableMetadata {
table_id,
table_route,
..
} = ddl_context
.table_metadata_allocator
.create(&create_physical_table_task)
.await
.unwrap();
create_physical_table_task.set_table_id(table_id);
create_physical_table_metadata(
&ddl_context,
create_physical_table_task.table_info.clone(),
TableRouteValue::Physical(table_route),
)
.await;
let mut procedure = CreateLogicalTablesProcedure::new(
vec![test_create_logical_table_task("foo")],
table_id,
ddl_context.clone(),
);
let status = procedure.on_prepare().await.unwrap();
assert_matches!(
status,
Status::Executing {
persist: true,
clean_poisons: false
}
);
let ctx = ProcedureContext {
procedure_id: ProcedureId::random(),
provider: Arc::new(MockContextProvider::default()),
event_context: None,
};
procedure.execute(&ctx).await.unwrap();
let status = procedure.execute(&ctx).await.unwrap();
let table_ids = status.downcast_output_ref::<Vec<u32>>().unwrap().clone();
// The engine rejects a request that occupies a reserved identity column.
let mut creates = None;
while let Ok((_, request)) = rx.try_recv() {
if let Some(region_request::Body::Creates(body)) = request.body {
creates = Some(body);
}
}
let creates = creates.unwrap();
assert_eq!(creates.requests.len(), 1);
let requested_columns = creates.requests[0]
.column_defs
.iter()
.map(|def| def.column_def.as_ref().unwrap().name.as_str())
.collect::<Vec<_>>();
assert!(
!requested_columns
.iter()
.any(|name| is_metric_engine_internal_column(name)),
"logical region request must not occupy internal columns: {requested_columns:?}"
);
assert!(requested_columns.contains(&"extra_tag"));
let logical_table_info = get_raw_table_info(&ddl_context, table_ids[0]).await;
let column_names = logical_table_info
.meta
.schema
.column_schemas()
.iter()
.map(|column| column.name.as_str())
.collect::<Vec<_>>();
assert!(
!column_names.contains(&DATA_SCHEMA_TABLE_ID_COLUMN_NAME),
"internal identity column must not be merged into the logical schema: {column_names:?}"
);
assert!(column_names.contains(&"extra_tag"));
// The user partition column is still merged as a nullable string tag of the primary key.
let extra_tag_index = logical_table_info
.meta
.schema
.column_index_by_name("extra_tag")
.unwrap();
assert!(
logical_table_info
.meta
.primary_key_indices
.contains(&extra_tag_index),
"merged partition column must be part of the primary key"
);
let extra_tag = &logical_table_info.meta.schema.column_schemas()[extra_tag_index];
assert_eq!(extra_tag.data_type, ConcreteDataType::string_datatype());
assert!(extra_tag.is_nullable());
}