fix!: enforce sparse primary key schema (#10020)

* fix: preserve required sorting for ordinary sparse scans

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* fix: enforce sparse primary key schema

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

* test: fix

Signed-off-by: shuiyisong <xixing.sys@gmail.com>

---------

Signed-off-by: shuiyisong <xixing.sys@gmail.com>
This commit is contained in:
shuiyisong
2026-10-28 17:09:33 +00:00
committed by GitHub
parent b21cd16f1d
commit efed847ca6
10 changed files with 294 additions and 96 deletions
+15 -1
View File
@@ -354,7 +354,11 @@ mod tests {
builder.primary_key_encoding(encoding);
for (id, name, ty) in columns {
builder.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new((*name).to_string(), ty.clone(), true),
column_schema: ColumnSchema::new(
(*name).to_string(),
ty.clone(),
!ReservedColumnId::is_reserved(*id),
),
semantic_type: SemanticType::Tag,
column_id: *id,
});
@@ -440,6 +444,16 @@ mod tests {
#[test]
fn test_decode_sparse_primary_key() {
let columns = vec![
(
ReservedColumnId::table_id(),
"__table_id",
ConcreteDataType::uint32_datatype(),
),
(
ReservedColumnId::tsid(),
"__tsid",
ConcreteDataType::uint64_datatype(),
),
(10, "k0", ConcreteDataType::string_datatype()),
(11, "k1", ConcreteDataType::string_datatype()),
];
+2 -1
View File
@@ -60,7 +60,8 @@ fn input(shape: &Shape) -> (RegionMetadataRef, HashSet<ColumnId>, RecordBatch) {
let mut pk = vec![ReservedColumnId::table_id(), ReservedColumnId::tsid()];
let mut inverted_columns = HashSet::new();
let mut add_tag = |id, name: String, data_type, indexed| {
let mut schema = ColumnSchema::new(name, data_type, true).with_inverted_index(false);
let mut schema = ColumnSchema::new(name, data_type, !ReservedColumnId::is_reserved(id))
.with_inverted_index(false);
if indexed && shape.inverted {
inverted_columns.insert(id);
}
+5 -1
View File
@@ -77,7 +77,11 @@ fn metadata(tags: u32) -> RegionMetadataRef {
primary_key.push(column_id);
builder.push_column_metadata(ColumnMetadata {
column_id,
column_schema: ColumnSchema::new(name, data_type, true),
column_schema: ColumnSchema::new(
name,
data_type,
!ReservedColumnId::is_reserved(column_id),
),
semantic_type: SemanticType::Tag,
});
}
+40
View File
@@ -117,6 +117,46 @@ async fn test_engine_column_compression_validation() {
assert_eq!(StatusCode::InvalidArguments, err.status_code());
}
#[tokio::test]
async fn test_engine_create_sparse_requires_reserved_columns() {
let mut env = TestEnv::new().await;
let engine = env.create_engine(MitoConfig::default()).await;
let region_id = RegionId::new(1, 1);
let mut request = CreateRequestBuilder::new()
.insert_option("primary_key_encoding", "sparse")
.insert_option("sst_format", "flat")
.build();
let error = engine
.handle_request(region_id, RegionRequest::Create(request.clone()))
.await
.unwrap_err();
assert_eq!(StatusCode::InvalidArguments, error.status_code());
assert!(
error
.output_msg()
.contains("sparse primary keys must start with (__table_id, __tsid)")
);
assert!(!engine.is_region_exists(region_id));
let metadata = sst_region_metadata_with_encoding(PrimaryKeyEncoding::Sparse);
request.column_metadatas = metadata.column_metadatas;
request.primary_key = metadata.primary_key;
engine
.handle_request(region_id, RegionRequest::Create(request.clone()))
.await
.unwrap();
let expected = engine.get_region(region_id).unwrap().metadata();
reopen_region(
&engine,
region_id,
request.table_dir,
false,
request.options,
)
.await;
assert_eq!(expected, engine.get_region(region_id).unwrap().metadata());
}
#[tokio::test]
async fn test_engine_create_new_region() {
test_engine_create_new_region_with_format(false).await;
+25 -77
View File
@@ -1858,19 +1858,17 @@ impl MultiBulkPart {
#[cfg(test)]
mod tests {
use api::v1::{Row, SemanticType, WriteHint};
use api::v1::{Row, WriteHint};
use datafusion_common::ScalarValue;
use datatypes::arrow::array::{
BinaryArray, DictionaryArray, Float64Array, TimestampMillisecondArray,
};
use datatypes::arrow::datatypes::UInt32Type;
use datatypes::prelude::{ConcreteDataType, Value};
use datatypes::schema::ColumnSchema;
use mito_codec::row_converter::build_primary_key_codec;
use rand::rngs::StdRng;
use rand::{Rng, SeedableRng};
use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder};
use store_api::storage::RegionId;
use store_api::metadata::RegionMetadataBuilder;
use store_api::storage::consts::ReservedColumnId;
use table::predicate::Predicate;
@@ -2594,47 +2592,30 @@ mod tests {
KeyValues::new(metadata.as_ref(), mutation).unwrap()
}
fn sparse_metadata_for_test() -> RegionMetadataRef {
let mut metadata = metadata_for_test().as_ref().clone();
metadata.column_metadatas[1].column_schema.data_type = ConcreteDataType::string_datatype();
let mut builder = RegionMetadataBuilder::from_existing(metadata);
let reserved = crate::test_util::sst_util::sst_region_metadata_with_encoding(
PrimaryKeyEncoding::Sparse,
);
for column in reserved.column_metadatas.into_iter().take(2) {
builder.push_column_metadata(column);
}
builder
.primary_key(vec![
ReservedColumnId::table_id(),
ReservedColumnId::tsid(),
0,
1,
])
.primary_key_encoding(PrimaryKeyEncoding::Sparse);
Arc::new(builder.build().unwrap())
}
#[test]
fn test_bulk_part_converter_sparse_primary_key_encoding() {
use api::v1::SemanticType;
use datatypes::schema::ColumnSchema;
use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder};
use store_api::storage::RegionId;
let mut builder = RegionMetadataBuilder::new(RegionId::new(123, 456));
builder
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new("k0", ConcreteDataType::string_datatype(), false),
semantic_type: SemanticType::Tag,
column_id: 0,
})
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new("k1", ConcreteDataType::string_datatype(), false),
semantic_type: SemanticType::Tag,
column_id: 1,
})
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new(
"ts",
ConcreteDataType::timestamp_millisecond_datatype(),
false,
),
semantic_type: SemanticType::Timestamp,
column_id: 2,
})
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new("v0", ConcreteDataType::int64_datatype(), true),
semantic_type: SemanticType::Field,
column_id: 3,
})
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new("v1", ConcreteDataType::float64_datatype(), true),
semantic_type: SemanticType::Field,
column_id: 4,
})
.primary_key(vec![0, 1])
.primary_key_encoding(PrimaryKeyEncoding::Sparse);
let metadata = Arc::new(builder.build().unwrap());
let metadata = sparse_metadata_for_test();
let primary_key_codec = build_primary_key_codec(&metadata);
let schema = to_flat_sst_arrow_schema(
@@ -2923,40 +2904,7 @@ mod tests {
#[test]
fn test_convert_bulk_part_sparse_encoding() {
let mut builder = RegionMetadataBuilder::new(RegionId::new(123, 456));
builder
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new("k0", ConcreteDataType::string_datatype(), false),
semantic_type: SemanticType::Tag,
column_id: 0,
})
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new("k1", ConcreteDataType::string_datatype(), false),
semantic_type: SemanticType::Tag,
column_id: 1,
})
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new(
"ts",
ConcreteDataType::timestamp_millisecond_datatype(),
false,
),
semantic_type: SemanticType::Timestamp,
column_id: 2,
})
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new("v0", ConcreteDataType::int64_datatype(), true),
semantic_type: SemanticType::Field,
column_id: 3,
})
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new("v1", ConcreteDataType::float64_datatype(), true),
semantic_type: SemanticType::Field,
column_id: 4,
})
.primary_key(vec![0, 1])
.primary_key_encoding(PrimaryKeyEncoding::Sparse);
let metadata = Arc::new(builder.build().unwrap());
let metadata = sparse_metadata_for_test();
let primary_key_codec = build_primary_key_codec(&metadata);
+51 -13
View File
@@ -1344,8 +1344,19 @@ mod tests {
],
&[],
);
let reserved = crate::test_util::sst_util::sst_region_metadata_with_encoding(
PrimaryKeyEncoding::Sparse,
);
actual_metadata.primary_key = reserved.primary_key.into_iter().take(2).collect();
actual_metadata
.column_metadatas
.extend(reserved.column_metadatas.into_iter().take(2));
actual_metadata.primary_key_encoding = PrimaryKeyEncoding::Sparse;
let actual_metadata = Arc::new(actual_metadata);
let actual_metadata = Arc::new(
RegionMetadataBuilder::from_existing(actual_metadata)
.build()
.unwrap(),
);
let mut expected_metadata = (*actual_metadata).clone();
for column in expected_metadata.column_metadatas.iter_mut() {
@@ -1373,7 +1384,10 @@ mod tests {
.unwrap()
.unwrap();
let sparse_k1 = encode_sparse_key(&[]);
let mut sparse_k1 = Vec::new();
SparsePrimaryKeyCodec::schemaless()
.encode_internal(42, 7, &mut sparse_k1)
.unwrap();
let input_columns: Vec<ArrayRef> = vec![
Arc::new(Int64Array::from(vec![100, 200])),
Arc::new(TimestampMillisecondArray::from_iter_values([1000, 2000])),
@@ -1384,8 +1398,10 @@ mod tests {
OpType::Put as u8,
])),
];
let input_schema =
to_flat_sst_arrow_schema(&actual_metadata, &FlatSchemaOptions::default());
let input_schema = to_flat_sst_arrow_schema(
&actual_metadata,
&FlatSchemaOptions::from_encoding(PrimaryKeyEncoding::Sparse),
);
let input_batch = RecordBatch::try_new(input_schema, input_columns).unwrap();
let result = compat_batch.compat(input_batch).unwrap();
@@ -1402,8 +1418,10 @@ mod tests {
OpType::Put as u8,
])),
];
let output_schema =
to_flat_sst_arrow_schema(&expected_metadata, &FlatSchemaOptions::default());
let output_schema = to_flat_sst_arrow_schema(
&expected_metadata,
&FlatSchemaOptions::from_encoding(PrimaryKeyEncoding::Sparse),
);
let expected_batch = RecordBatch::try_new(output_schema, expected_columns).unwrap();
assert_eq!(expected_batch, result);
@@ -1520,6 +1538,16 @@ mod tests {
columns[0] = json;
let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns)?;
for encoding in [PrimaryKeyEncoding::Dense, PrimaryKeyEncoding::Sparse] {
let batch = if encoding == PrimaryKeyEncoding::Sparse {
let mut key = Vec::new();
SparsePrimaryKeyCodec::schemaless().encode_internal(42, 7, &mut key)?;
let mut columns = batch.columns().to_vec();
columns[primary_key_column_index(batch.num_columns())] =
build_flat_test_pk_array(&[&key, &key]);
RecordBatch::try_new(batch.schema(), columns)?
} else {
batch.clone()
};
for missing_json in [false, true] {
let source = if missing_json {
new_metadata(
@@ -1533,12 +1561,12 @@ mod tests {
} else {
metadata.as_ref().clone()
};
let mut builder = RegionMetadataBuilder::from_existing(source);
builder.primary_key_encoding(encoding);
let actual = Arc::new(builder.build()?);
let mut builder = RegionMetadataBuilder::from_existing(metadata.as_ref().clone());
builder.primary_key_encoding(encoding);
builder.push_column_metadata(ColumnMetadata {
let mut actual_builder = RegionMetadataBuilder::from_existing(source);
actual_builder.primary_key_encoding(encoding);
let mut expected_builder =
RegionMetadataBuilder::from_existing(metadata.as_ref().clone());
expected_builder.primary_key_encoding(encoding);
expected_builder.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new(
"added",
ConcreteDataType::int64_datatype(),
@@ -1550,7 +1578,17 @@ mod tests {
semantic_type: SemanticType::Field,
column_id: 2,
});
let expected = Arc::new(builder.build()?);
if encoding == PrimaryKeyEncoding::Sparse {
let reserved = test_util::sst_util::sst_region_metadata_with_encoding(encoding);
for builder in [&mut actual_builder, &mut expected_builder] {
builder.primary_key(reserved.primary_key[..2].to_vec());
for column in reserved.column_metadatas.iter().take(2) {
builder.push_column_metadata(column.clone());
}
}
}
let actual = Arc::new(actual_builder.build()?);
let expected = Arc::new(expected_builder.build()?);
let read_columns = ReadColumns::new([0, 1, 2]);
let mapper = FlatProjectionMapper::new_with_json2_rewrite_targets(
&expected,
+8 -1
View File
@@ -1355,7 +1355,14 @@ mod tests {
column_id: col_id,
});
let primary_key: Vec<u32> = (0..num_tags as u32).collect();
let mut primary_key: Vec<u32> = (0..num_tags as u32).collect();
if encoding == PrimaryKeyEncoding::Sparse {
let reserved = crate::test_util::sst_util::sst_region_metadata_with_encoding(encoding);
primary_key.splice(0..0, reserved.primary_key.into_iter().take(2));
for column in reserved.column_metadatas.into_iter().take(2) {
builder.push_column_metadata(column);
}
}
builder.primary_key(primary_key);
builder.primary_key_encoding(encoding);
builder.build().unwrap()
+133 -1
View File
@@ -31,6 +31,7 @@ use common_macro::stack_trace_debug;
use datatypes::arrow;
use datatypes::arrow::datatypes::FieldRef;
use datatypes::extension::json::json2_metadata_with_updated_settings;
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{ColumnSchema, FulltextOptions, Schema, SchemaRef};
use datatypes::types::TimestampType;
use itertools::Itertools;
@@ -39,10 +40,11 @@ use serde::{Deserialize, Deserializer, Serialize};
use snafu::{Location, OptionExt, ResultExt, Snafu, ensure};
use crate::codec::PrimaryKeyEncoding;
use crate::metric_engine_consts::{DATA_SCHEMA_TABLE_ID_COLUMN_NAME, DATA_SCHEMA_TSID_COLUMN_NAME};
use crate::region_request::{
AddColumn, AddColumnLocation, AlterKind, ModifyColumnType, SetIndexOption, UnsetIndexOption,
};
use crate::storage::consts::is_internal_column;
use crate::storage::consts::{ReservedColumnId, is_internal_column};
use crate::storage::{ColumnId, RegionId};
pub type Result<T> = std::result::Result<T, MetadataError>;
@@ -476,6 +478,16 @@ impl RegionMetadata {
}
);
if self.primary_key_encoding == PrimaryKeyEncoding::Sparse {
ensure!(
self.primary_key
.starts_with(&[ReservedColumnId::table_id(), ReservedColumnId::tsid()]),
InvalidMetaSnafu {
reason: "sparse primary keys must start with (__table_id, __tsid)",
}
);
}
if !self.primary_key.is_empty() {
let mut pk_ids = HashSet::with_capacity(self.primary_key.len());
// Checks column ids in the primary key is valid.
@@ -523,6 +535,27 @@ impl RegionMetadata {
}
);
if self.primary_key_encoding == PrimaryKeyEncoding::Sparse {
let schema = &column.column_schema;
let valid = if *column_id == ReservedColumnId::table_id() {
schema.name == DATA_SCHEMA_TABLE_ID_COLUMN_NAME
&& schema.data_type == ConcreteDataType::uint32_datatype()
&& !schema.is_nullable()
} else if *column_id == ReservedColumnId::tsid() {
schema.name == DATA_SCHEMA_TSID_COLUMN_NAME
&& schema.data_type == ConcreteDataType::uint64_datatype()
&& !schema.is_nullable()
} else {
!ReservedColumnId::is_reserved(*column_id) && schema.data_type.is_string()
};
ensure!(
valid,
InvalidMetaSnafu {
reason: format!("invalid sparse primary key column {}", schema.name),
}
);
}
pk_ids.insert(column_id);
}
}
@@ -1388,6 +1421,105 @@ mod test {
builder.build().unwrap()
}
#[test]
fn test_sparse_primary_key_requires_reserved_columns() {
for primary_key in [vec![], vec![1]] {
let mut metadata = build_test_region_metadata();
if primary_key.is_empty() {
metadata.column_metadatas[0].semantic_type = SemanticType::Field;
}
metadata.primary_key = primary_key;
let mut builder = RegionMetadataBuilder::from_existing(metadata);
builder.primary_key_encoding(PrimaryKeyEncoding::Sparse);
let error = builder.build().unwrap_err();
assert_eq!(error.status_code(), StatusCode::InvalidArguments);
assert!(
error
.to_string()
.contains("sparse primary keys must start with (__table_id, __tsid)")
);
}
}
#[test]
fn test_sparse_reserved_primary_key_contract() {
let mut builder = RegionMetadataBuilder::from_existing(build_test_region_metadata());
builder.primary_key_encoding(PrimaryKeyEncoding::Sparse);
builder.column_metadatas[0].column_schema.data_type = ConcreteDataType::string_datatype();
for (name, column_id, data_type) in [
(
DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
ReservedColumnId::table_id(),
ConcreteDataType::uint32_datatype(),
),
(
DATA_SCHEMA_TSID_COLUMN_NAME,
ReservedColumnId::tsid(),
ConcreteDataType::uint64_datatype(),
),
] {
builder.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new(name, data_type, false),
semantic_type: SemanticType::Tag,
column_id,
});
}
builder.primary_key(vec![
ReservedColumnId::table_id(),
ReservedColumnId::tsid(),
1,
]);
let metadata = builder.build().unwrap();
let json = serde_json::to_string(&metadata).unwrap();
assert_eq!(
metadata,
serde_json::from_str::<RegionMetadata>(&json).unwrap()
);
for case in 0..12 {
let mut builder = RegionMetadataBuilder::from_existing(metadata.clone());
match case {
0 => builder.primary_key.swap(0, 1),
1 => {
builder.primary_key.remove(0);
builder.column_metadatas.remove(3);
}
2 => builder.column_metadatas[3].column_schema.set_nullable(),
3 => {
builder.column_metadatas[3].column_schema.data_type =
ConcreteDataType::uint64_datatype()
}
4 => builder.column_metadatas[4].column_schema.name = "wrong_name".into(),
5 => {
builder.column_metadatas[0].column_schema.data_type =
ConcreteDataType::int64_datatype()
}
6 => {
builder.primary_key.remove(1);
builder.column_metadatas.remove(4);
}
7 => builder.column_metadatas[4].column_schema.set_nullable(),
8 => {
builder.column_metadatas[4].column_schema.data_type =
ConcreteDataType::uint32_datatype()
}
9 => builder.column_metadatas[3].semantic_type = SemanticType::Field,
10 => builder.column_metadatas[4].semantic_type = SemanticType::Field,
11 => {
builder.primary_key[0] = 10;
builder.primary_key[1] = 11;
builder.column_metadatas[3].column_id = 10;
builder.column_metadatas[4].column_id = 11;
}
_ => unreachable!(),
}
assert_eq!(
builder.build().unwrap_err().status_code(),
StatusCode::InvalidArguments
);
}
}
#[test]
fn test_region_metadata() {
let region_metadata = build_test_region_metadata();
+14
View File
@@ -717,6 +717,7 @@ mod test {
use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
use datatypes::vectors::{Int32Vector, TimestampMillisecondVector, VectorRef};
use futures::TryStreamExt;
use store_api::codec::PrimaryKeyEncoding;
use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder};
use store_api::metric_engine_consts::DATA_SCHEMA_TABLE_ID_COLUMN_NAME;
use store_api::region_engine::{
@@ -1108,7 +1109,14 @@ mod test {
/// order or hash partition columns.
#[test]
fn test_metric_projection_claims_only_projected_row_key_prefix() {
for encoding in [PrimaryKeyEncoding::Dense, PrimaryKeyEncoding::Sparse] {
check_metric_projection_ordering(encoding);
}
}
fn check_metric_projection_ordering(encoding: PrimaryKeyEncoding) {
let mut builder = RegionMetadataBuilder::new(RegionId::new(1234, 5691));
builder.primary_key_encoding(encoding);
for (name, column_id) in [
(
DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
@@ -1139,6 +1147,12 @@ mod test {
let metadata = Arc::new(builder.build().unwrap());
for (projection, partitioned_by_tsid, claimed, unclaimed) in [
(
vec![DATA_SCHEMA_TABLE_ID_COLUMN_NAME, "A", "ts"],
false,
vec![vec![DATA_SCHEMA_TABLE_ID_COLUMN_NAME]],
vec![vec!["A"], vec![DATA_SCHEMA_TABLE_ID_COLUMN_NAME, "ts"]],
),
// The logical projection drops the row key prefix, so `A` is not a prefix of the
// row key and neither `A` nor `ts` is ordered.
(
@@ -109,7 +109,7 @@ CREATE TABLE bad (
host STRING PRIMARY KEY COMPRESSION WITH (level = 3)
) WITH ('primary_key_encoding' = 'sparse');
Error: 1004(InvalidArguments), Invalid region options, COMPRESSION on tag column host requires flat SST format and dense primary key encoding
Error: 1004(InvalidArguments), Invalid metadata, sparse primary keys must start with (__table_id, __tsid)
CREATE TABLE compressed_tags (
ts TIMESTAMP TIME INDEX,