mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-17 16:24:07 +00:00
fix!: enforce sparse primary key schema (#10020)
* fix: preserve required sorting for ordinary sparse scans Signed-off-by: shuiyisong <xixing.sys@gmail.com> * fix: enforce sparse primary key schema Signed-off-by: shuiyisong <xixing.sys@gmail.com> * test: fix Signed-off-by: shuiyisong <xixing.sys@gmail.com> --------- Signed-off-by: shuiyisong <xixing.sys@gmail.com>
This commit is contained in:
@@ -354,7 +354,11 @@ mod tests {
|
||||
builder.primary_key_encoding(encoding);
|
||||
for (id, name, ty) in columns {
|
||||
builder.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new((*name).to_string(), ty.clone(), true),
|
||||
column_schema: ColumnSchema::new(
|
||||
(*name).to_string(),
|
||||
ty.clone(),
|
||||
!ReservedColumnId::is_reserved(*id),
|
||||
),
|
||||
semantic_type: SemanticType::Tag,
|
||||
column_id: *id,
|
||||
});
|
||||
@@ -440,6 +444,16 @@ mod tests {
|
||||
#[test]
|
||||
fn test_decode_sparse_primary_key() {
|
||||
let columns = vec![
|
||||
(
|
||||
ReservedColumnId::table_id(),
|
||||
"__table_id",
|
||||
ConcreteDataType::uint32_datatype(),
|
||||
),
|
||||
(
|
||||
ReservedColumnId::tsid(),
|
||||
"__tsid",
|
||||
ConcreteDataType::uint64_datatype(),
|
||||
),
|
||||
(10, "k0", ConcreteDataType::string_datatype()),
|
||||
(11, "k1", ConcreteDataType::string_datatype()),
|
||||
];
|
||||
|
||||
@@ -60,7 +60,8 @@ fn input(shape: &Shape) -> (RegionMetadataRef, HashSet<ColumnId>, RecordBatch) {
|
||||
let mut pk = vec![ReservedColumnId::table_id(), ReservedColumnId::tsid()];
|
||||
let mut inverted_columns = HashSet::new();
|
||||
let mut add_tag = |id, name: String, data_type, indexed| {
|
||||
let mut schema = ColumnSchema::new(name, data_type, true).with_inverted_index(false);
|
||||
let mut schema = ColumnSchema::new(name, data_type, !ReservedColumnId::is_reserved(id))
|
||||
.with_inverted_index(false);
|
||||
if indexed && shape.inverted {
|
||||
inverted_columns.insert(id);
|
||||
}
|
||||
|
||||
@@ -77,7 +77,11 @@ fn metadata(tags: u32) -> RegionMetadataRef {
|
||||
primary_key.push(column_id);
|
||||
builder.push_column_metadata(ColumnMetadata {
|
||||
column_id,
|
||||
column_schema: ColumnSchema::new(name, data_type, true),
|
||||
column_schema: ColumnSchema::new(
|
||||
name,
|
||||
data_type,
|
||||
!ReservedColumnId::is_reserved(column_id),
|
||||
),
|
||||
semantic_type: SemanticType::Tag,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -117,6 +117,46 @@ async fn test_engine_column_compression_validation() {
|
||||
assert_eq!(StatusCode::InvalidArguments, err.status_code());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_engine_create_sparse_requires_reserved_columns() {
|
||||
let mut env = TestEnv::new().await;
|
||||
let engine = env.create_engine(MitoConfig::default()).await;
|
||||
let region_id = RegionId::new(1, 1);
|
||||
let mut request = CreateRequestBuilder::new()
|
||||
.insert_option("primary_key_encoding", "sparse")
|
||||
.insert_option("sst_format", "flat")
|
||||
.build();
|
||||
let error = engine
|
||||
.handle_request(region_id, RegionRequest::Create(request.clone()))
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(StatusCode::InvalidArguments, error.status_code());
|
||||
assert!(
|
||||
error
|
||||
.output_msg()
|
||||
.contains("sparse primary keys must start with (__table_id, __tsid)")
|
||||
);
|
||||
assert!(!engine.is_region_exists(region_id));
|
||||
|
||||
let metadata = sst_region_metadata_with_encoding(PrimaryKeyEncoding::Sparse);
|
||||
request.column_metadatas = metadata.column_metadatas;
|
||||
request.primary_key = metadata.primary_key;
|
||||
engine
|
||||
.handle_request(region_id, RegionRequest::Create(request.clone()))
|
||||
.await
|
||||
.unwrap();
|
||||
let expected = engine.get_region(region_id).unwrap().metadata();
|
||||
reopen_region(
|
||||
&engine,
|
||||
region_id,
|
||||
request.table_dir,
|
||||
false,
|
||||
request.options,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(expected, engine.get_region(region_id).unwrap().metadata());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_engine_create_new_region() {
|
||||
test_engine_create_new_region_with_format(false).await;
|
||||
|
||||
@@ -1858,19 +1858,17 @@ impl MultiBulkPart {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use api::v1::{Row, SemanticType, WriteHint};
|
||||
use api::v1::{Row, WriteHint};
|
||||
use datafusion_common::ScalarValue;
|
||||
use datatypes::arrow::array::{
|
||||
BinaryArray, DictionaryArray, Float64Array, TimestampMillisecondArray,
|
||||
};
|
||||
use datatypes::arrow::datatypes::UInt32Type;
|
||||
use datatypes::prelude::{ConcreteDataType, Value};
|
||||
use datatypes::schema::ColumnSchema;
|
||||
use mito_codec::row_converter::build_primary_key_codec;
|
||||
use rand::rngs::StdRng;
|
||||
use rand::{Rng, SeedableRng};
|
||||
use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder};
|
||||
use store_api::storage::RegionId;
|
||||
use store_api::metadata::RegionMetadataBuilder;
|
||||
use store_api::storage::consts::ReservedColumnId;
|
||||
use table::predicate::Predicate;
|
||||
|
||||
@@ -2594,47 +2592,30 @@ mod tests {
|
||||
KeyValues::new(metadata.as_ref(), mutation).unwrap()
|
||||
}
|
||||
|
||||
fn sparse_metadata_for_test() -> RegionMetadataRef {
|
||||
let mut metadata = metadata_for_test().as_ref().clone();
|
||||
metadata.column_metadatas[1].column_schema.data_type = ConcreteDataType::string_datatype();
|
||||
let mut builder = RegionMetadataBuilder::from_existing(metadata);
|
||||
let reserved = crate::test_util::sst_util::sst_region_metadata_with_encoding(
|
||||
PrimaryKeyEncoding::Sparse,
|
||||
);
|
||||
for column in reserved.column_metadatas.into_iter().take(2) {
|
||||
builder.push_column_metadata(column);
|
||||
}
|
||||
builder
|
||||
.primary_key(vec![
|
||||
ReservedColumnId::table_id(),
|
||||
ReservedColumnId::tsid(),
|
||||
0,
|
||||
1,
|
||||
])
|
||||
.primary_key_encoding(PrimaryKeyEncoding::Sparse);
|
||||
Arc::new(builder.build().unwrap())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bulk_part_converter_sparse_primary_key_encoding() {
|
||||
use api::v1::SemanticType;
|
||||
use datatypes::schema::ColumnSchema;
|
||||
use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder};
|
||||
use store_api::storage::RegionId;
|
||||
|
||||
let mut builder = RegionMetadataBuilder::new(RegionId::new(123, 456));
|
||||
builder
|
||||
.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new("k0", ConcreteDataType::string_datatype(), false),
|
||||
semantic_type: SemanticType::Tag,
|
||||
column_id: 0,
|
||||
})
|
||||
.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new("k1", ConcreteDataType::string_datatype(), false),
|
||||
semantic_type: SemanticType::Tag,
|
||||
column_id: 1,
|
||||
})
|
||||
.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new(
|
||||
"ts",
|
||||
ConcreteDataType::timestamp_millisecond_datatype(),
|
||||
false,
|
||||
),
|
||||
semantic_type: SemanticType::Timestamp,
|
||||
column_id: 2,
|
||||
})
|
||||
.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new("v0", ConcreteDataType::int64_datatype(), true),
|
||||
semantic_type: SemanticType::Field,
|
||||
column_id: 3,
|
||||
})
|
||||
.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new("v1", ConcreteDataType::float64_datatype(), true),
|
||||
semantic_type: SemanticType::Field,
|
||||
column_id: 4,
|
||||
})
|
||||
.primary_key(vec![0, 1])
|
||||
.primary_key_encoding(PrimaryKeyEncoding::Sparse);
|
||||
let metadata = Arc::new(builder.build().unwrap());
|
||||
let metadata = sparse_metadata_for_test();
|
||||
|
||||
let primary_key_codec = build_primary_key_codec(&metadata);
|
||||
let schema = to_flat_sst_arrow_schema(
|
||||
@@ -2923,40 +2904,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_convert_bulk_part_sparse_encoding() {
|
||||
let mut builder = RegionMetadataBuilder::new(RegionId::new(123, 456));
|
||||
builder
|
||||
.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new("k0", ConcreteDataType::string_datatype(), false),
|
||||
semantic_type: SemanticType::Tag,
|
||||
column_id: 0,
|
||||
})
|
||||
.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new("k1", ConcreteDataType::string_datatype(), false),
|
||||
semantic_type: SemanticType::Tag,
|
||||
column_id: 1,
|
||||
})
|
||||
.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new(
|
||||
"ts",
|
||||
ConcreteDataType::timestamp_millisecond_datatype(),
|
||||
false,
|
||||
),
|
||||
semantic_type: SemanticType::Timestamp,
|
||||
column_id: 2,
|
||||
})
|
||||
.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new("v0", ConcreteDataType::int64_datatype(), true),
|
||||
semantic_type: SemanticType::Field,
|
||||
column_id: 3,
|
||||
})
|
||||
.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new("v1", ConcreteDataType::float64_datatype(), true),
|
||||
semantic_type: SemanticType::Field,
|
||||
column_id: 4,
|
||||
})
|
||||
.primary_key(vec![0, 1])
|
||||
.primary_key_encoding(PrimaryKeyEncoding::Sparse);
|
||||
let metadata = Arc::new(builder.build().unwrap());
|
||||
let metadata = sparse_metadata_for_test();
|
||||
|
||||
let primary_key_codec = build_primary_key_codec(&metadata);
|
||||
|
||||
|
||||
@@ -1344,8 +1344,19 @@ mod tests {
|
||||
],
|
||||
&[],
|
||||
);
|
||||
let reserved = crate::test_util::sst_util::sst_region_metadata_with_encoding(
|
||||
PrimaryKeyEncoding::Sparse,
|
||||
);
|
||||
actual_metadata.primary_key = reserved.primary_key.into_iter().take(2).collect();
|
||||
actual_metadata
|
||||
.column_metadatas
|
||||
.extend(reserved.column_metadatas.into_iter().take(2));
|
||||
actual_metadata.primary_key_encoding = PrimaryKeyEncoding::Sparse;
|
||||
let actual_metadata = Arc::new(actual_metadata);
|
||||
let actual_metadata = Arc::new(
|
||||
RegionMetadataBuilder::from_existing(actual_metadata)
|
||||
.build()
|
||||
.unwrap(),
|
||||
);
|
||||
|
||||
let mut expected_metadata = (*actual_metadata).clone();
|
||||
for column in expected_metadata.column_metadatas.iter_mut() {
|
||||
@@ -1373,7 +1384,10 @@ mod tests {
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
let sparse_k1 = encode_sparse_key(&[]);
|
||||
let mut sparse_k1 = Vec::new();
|
||||
SparsePrimaryKeyCodec::schemaless()
|
||||
.encode_internal(42, 7, &mut sparse_k1)
|
||||
.unwrap();
|
||||
let input_columns: Vec<ArrayRef> = vec![
|
||||
Arc::new(Int64Array::from(vec![100, 200])),
|
||||
Arc::new(TimestampMillisecondArray::from_iter_values([1000, 2000])),
|
||||
@@ -1384,8 +1398,10 @@ mod tests {
|
||||
OpType::Put as u8,
|
||||
])),
|
||||
];
|
||||
let input_schema =
|
||||
to_flat_sst_arrow_schema(&actual_metadata, &FlatSchemaOptions::default());
|
||||
let input_schema = to_flat_sst_arrow_schema(
|
||||
&actual_metadata,
|
||||
&FlatSchemaOptions::from_encoding(PrimaryKeyEncoding::Sparse),
|
||||
);
|
||||
let input_batch = RecordBatch::try_new(input_schema, input_columns).unwrap();
|
||||
|
||||
let result = compat_batch.compat(input_batch).unwrap();
|
||||
@@ -1402,8 +1418,10 @@ mod tests {
|
||||
OpType::Put as u8,
|
||||
])),
|
||||
];
|
||||
let output_schema =
|
||||
to_flat_sst_arrow_schema(&expected_metadata, &FlatSchemaOptions::default());
|
||||
let output_schema = to_flat_sst_arrow_schema(
|
||||
&expected_metadata,
|
||||
&FlatSchemaOptions::from_encoding(PrimaryKeyEncoding::Sparse),
|
||||
);
|
||||
let expected_batch = RecordBatch::try_new(output_schema, expected_columns).unwrap();
|
||||
|
||||
assert_eq!(expected_batch, result);
|
||||
@@ -1520,6 +1538,16 @@ mod tests {
|
||||
columns[0] = json;
|
||||
let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns)?;
|
||||
for encoding in [PrimaryKeyEncoding::Dense, PrimaryKeyEncoding::Sparse] {
|
||||
let batch = if encoding == PrimaryKeyEncoding::Sparse {
|
||||
let mut key = Vec::new();
|
||||
SparsePrimaryKeyCodec::schemaless().encode_internal(42, 7, &mut key)?;
|
||||
let mut columns = batch.columns().to_vec();
|
||||
columns[primary_key_column_index(batch.num_columns())] =
|
||||
build_flat_test_pk_array(&[&key, &key]);
|
||||
RecordBatch::try_new(batch.schema(), columns)?
|
||||
} else {
|
||||
batch.clone()
|
||||
};
|
||||
for missing_json in [false, true] {
|
||||
let source = if missing_json {
|
||||
new_metadata(
|
||||
@@ -1533,12 +1561,12 @@ mod tests {
|
||||
} else {
|
||||
metadata.as_ref().clone()
|
||||
};
|
||||
let mut builder = RegionMetadataBuilder::from_existing(source);
|
||||
builder.primary_key_encoding(encoding);
|
||||
let actual = Arc::new(builder.build()?);
|
||||
let mut builder = RegionMetadataBuilder::from_existing(metadata.as_ref().clone());
|
||||
builder.primary_key_encoding(encoding);
|
||||
builder.push_column_metadata(ColumnMetadata {
|
||||
let mut actual_builder = RegionMetadataBuilder::from_existing(source);
|
||||
actual_builder.primary_key_encoding(encoding);
|
||||
let mut expected_builder =
|
||||
RegionMetadataBuilder::from_existing(metadata.as_ref().clone());
|
||||
expected_builder.primary_key_encoding(encoding);
|
||||
expected_builder.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new(
|
||||
"added",
|
||||
ConcreteDataType::int64_datatype(),
|
||||
@@ -1550,7 +1578,17 @@ mod tests {
|
||||
semantic_type: SemanticType::Field,
|
||||
column_id: 2,
|
||||
});
|
||||
let expected = Arc::new(builder.build()?);
|
||||
if encoding == PrimaryKeyEncoding::Sparse {
|
||||
let reserved = test_util::sst_util::sst_region_metadata_with_encoding(encoding);
|
||||
for builder in [&mut actual_builder, &mut expected_builder] {
|
||||
builder.primary_key(reserved.primary_key[..2].to_vec());
|
||||
for column in reserved.column_metadatas.iter().take(2) {
|
||||
builder.push_column_metadata(column.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
let actual = Arc::new(actual_builder.build()?);
|
||||
let expected = Arc::new(expected_builder.build()?);
|
||||
let read_columns = ReadColumns::new([0, 1, 2]);
|
||||
let mapper = FlatProjectionMapper::new_with_json2_rewrite_targets(
|
||||
&expected,
|
||||
|
||||
@@ -1355,7 +1355,14 @@ mod tests {
|
||||
column_id: col_id,
|
||||
});
|
||||
|
||||
let primary_key: Vec<u32> = (0..num_tags as u32).collect();
|
||||
let mut primary_key: Vec<u32> = (0..num_tags as u32).collect();
|
||||
if encoding == PrimaryKeyEncoding::Sparse {
|
||||
let reserved = crate::test_util::sst_util::sst_region_metadata_with_encoding(encoding);
|
||||
primary_key.splice(0..0, reserved.primary_key.into_iter().take(2));
|
||||
for column in reserved.column_metadatas.into_iter().take(2) {
|
||||
builder.push_column_metadata(column);
|
||||
}
|
||||
}
|
||||
builder.primary_key(primary_key);
|
||||
builder.primary_key_encoding(encoding);
|
||||
builder.build().unwrap()
|
||||
|
||||
@@ -31,6 +31,7 @@ use common_macro::stack_trace_debug;
|
||||
use datatypes::arrow;
|
||||
use datatypes::arrow::datatypes::FieldRef;
|
||||
use datatypes::extension::json::json2_metadata_with_updated_settings;
|
||||
use datatypes::prelude::ConcreteDataType;
|
||||
use datatypes::schema::{ColumnSchema, FulltextOptions, Schema, SchemaRef};
|
||||
use datatypes::types::TimestampType;
|
||||
use itertools::Itertools;
|
||||
@@ -39,10 +40,11 @@ use serde::{Deserialize, Deserializer, Serialize};
|
||||
use snafu::{Location, OptionExt, ResultExt, Snafu, ensure};
|
||||
|
||||
use crate::codec::PrimaryKeyEncoding;
|
||||
use crate::metric_engine_consts::{DATA_SCHEMA_TABLE_ID_COLUMN_NAME, DATA_SCHEMA_TSID_COLUMN_NAME};
|
||||
use crate::region_request::{
|
||||
AddColumn, AddColumnLocation, AlterKind, ModifyColumnType, SetIndexOption, UnsetIndexOption,
|
||||
};
|
||||
use crate::storage::consts::is_internal_column;
|
||||
use crate::storage::consts::{ReservedColumnId, is_internal_column};
|
||||
use crate::storage::{ColumnId, RegionId};
|
||||
|
||||
pub type Result<T> = std::result::Result<T, MetadataError>;
|
||||
@@ -476,6 +478,16 @@ impl RegionMetadata {
|
||||
}
|
||||
);
|
||||
|
||||
if self.primary_key_encoding == PrimaryKeyEncoding::Sparse {
|
||||
ensure!(
|
||||
self.primary_key
|
||||
.starts_with(&[ReservedColumnId::table_id(), ReservedColumnId::tsid()]),
|
||||
InvalidMetaSnafu {
|
||||
reason: "sparse primary keys must start with (__table_id, __tsid)",
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
if !self.primary_key.is_empty() {
|
||||
let mut pk_ids = HashSet::with_capacity(self.primary_key.len());
|
||||
// Checks column ids in the primary key is valid.
|
||||
@@ -523,6 +535,27 @@ impl RegionMetadata {
|
||||
}
|
||||
);
|
||||
|
||||
if self.primary_key_encoding == PrimaryKeyEncoding::Sparse {
|
||||
let schema = &column.column_schema;
|
||||
let valid = if *column_id == ReservedColumnId::table_id() {
|
||||
schema.name == DATA_SCHEMA_TABLE_ID_COLUMN_NAME
|
||||
&& schema.data_type == ConcreteDataType::uint32_datatype()
|
||||
&& !schema.is_nullable()
|
||||
} else if *column_id == ReservedColumnId::tsid() {
|
||||
schema.name == DATA_SCHEMA_TSID_COLUMN_NAME
|
||||
&& schema.data_type == ConcreteDataType::uint64_datatype()
|
||||
&& !schema.is_nullable()
|
||||
} else {
|
||||
!ReservedColumnId::is_reserved(*column_id) && schema.data_type.is_string()
|
||||
};
|
||||
ensure!(
|
||||
valid,
|
||||
InvalidMetaSnafu {
|
||||
reason: format!("invalid sparse primary key column {}", schema.name),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
pk_ids.insert(column_id);
|
||||
}
|
||||
}
|
||||
@@ -1388,6 +1421,105 @@ mod test {
|
||||
builder.build().unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sparse_primary_key_requires_reserved_columns() {
|
||||
for primary_key in [vec![], vec![1]] {
|
||||
let mut metadata = build_test_region_metadata();
|
||||
if primary_key.is_empty() {
|
||||
metadata.column_metadatas[0].semantic_type = SemanticType::Field;
|
||||
}
|
||||
metadata.primary_key = primary_key;
|
||||
let mut builder = RegionMetadataBuilder::from_existing(metadata);
|
||||
builder.primary_key_encoding(PrimaryKeyEncoding::Sparse);
|
||||
let error = builder.build().unwrap_err();
|
||||
assert_eq!(error.status_code(), StatusCode::InvalidArguments);
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("sparse primary keys must start with (__table_id, __tsid)")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sparse_reserved_primary_key_contract() {
|
||||
let mut builder = RegionMetadataBuilder::from_existing(build_test_region_metadata());
|
||||
builder.primary_key_encoding(PrimaryKeyEncoding::Sparse);
|
||||
builder.column_metadatas[0].column_schema.data_type = ConcreteDataType::string_datatype();
|
||||
for (name, column_id, data_type) in [
|
||||
(
|
||||
DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
|
||||
ReservedColumnId::table_id(),
|
||||
ConcreteDataType::uint32_datatype(),
|
||||
),
|
||||
(
|
||||
DATA_SCHEMA_TSID_COLUMN_NAME,
|
||||
ReservedColumnId::tsid(),
|
||||
ConcreteDataType::uint64_datatype(),
|
||||
),
|
||||
] {
|
||||
builder.push_column_metadata(ColumnMetadata {
|
||||
column_schema: ColumnSchema::new(name, data_type, false),
|
||||
semantic_type: SemanticType::Tag,
|
||||
column_id,
|
||||
});
|
||||
}
|
||||
builder.primary_key(vec![
|
||||
ReservedColumnId::table_id(),
|
||||
ReservedColumnId::tsid(),
|
||||
1,
|
||||
]);
|
||||
let metadata = builder.build().unwrap();
|
||||
let json = serde_json::to_string(&metadata).unwrap();
|
||||
assert_eq!(
|
||||
metadata,
|
||||
serde_json::from_str::<RegionMetadata>(&json).unwrap()
|
||||
);
|
||||
|
||||
for case in 0..12 {
|
||||
let mut builder = RegionMetadataBuilder::from_existing(metadata.clone());
|
||||
match case {
|
||||
0 => builder.primary_key.swap(0, 1),
|
||||
1 => {
|
||||
builder.primary_key.remove(0);
|
||||
builder.column_metadatas.remove(3);
|
||||
}
|
||||
2 => builder.column_metadatas[3].column_schema.set_nullable(),
|
||||
3 => {
|
||||
builder.column_metadatas[3].column_schema.data_type =
|
||||
ConcreteDataType::uint64_datatype()
|
||||
}
|
||||
4 => builder.column_metadatas[4].column_schema.name = "wrong_name".into(),
|
||||
5 => {
|
||||
builder.column_metadatas[0].column_schema.data_type =
|
||||
ConcreteDataType::int64_datatype()
|
||||
}
|
||||
6 => {
|
||||
builder.primary_key.remove(1);
|
||||
builder.column_metadatas.remove(4);
|
||||
}
|
||||
7 => builder.column_metadatas[4].column_schema.set_nullable(),
|
||||
8 => {
|
||||
builder.column_metadatas[4].column_schema.data_type =
|
||||
ConcreteDataType::uint32_datatype()
|
||||
}
|
||||
9 => builder.column_metadatas[3].semantic_type = SemanticType::Field,
|
||||
10 => builder.column_metadatas[4].semantic_type = SemanticType::Field,
|
||||
11 => {
|
||||
builder.primary_key[0] = 10;
|
||||
builder.primary_key[1] = 11;
|
||||
builder.column_metadatas[3].column_id = 10;
|
||||
builder.column_metadatas[4].column_id = 11;
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
assert_eq!(
|
||||
builder.build().unwrap_err().status_code(),
|
||||
StatusCode::InvalidArguments
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_region_metadata() {
|
||||
let region_metadata = build_test_region_metadata();
|
||||
|
||||
@@ -717,6 +717,7 @@ mod test {
|
||||
use datatypes::schema::{ColumnSchema, Schema, SchemaRef};
|
||||
use datatypes::vectors::{Int32Vector, TimestampMillisecondVector, VectorRef};
|
||||
use futures::TryStreamExt;
|
||||
use store_api::codec::PrimaryKeyEncoding;
|
||||
use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder};
|
||||
use store_api::metric_engine_consts::DATA_SCHEMA_TABLE_ID_COLUMN_NAME;
|
||||
use store_api::region_engine::{
|
||||
@@ -1108,7 +1109,14 @@ mod test {
|
||||
/// order or hash partition columns.
|
||||
#[test]
|
||||
fn test_metric_projection_claims_only_projected_row_key_prefix() {
|
||||
for encoding in [PrimaryKeyEncoding::Dense, PrimaryKeyEncoding::Sparse] {
|
||||
check_metric_projection_ordering(encoding);
|
||||
}
|
||||
}
|
||||
|
||||
fn check_metric_projection_ordering(encoding: PrimaryKeyEncoding) {
|
||||
let mut builder = RegionMetadataBuilder::new(RegionId::new(1234, 5691));
|
||||
builder.primary_key_encoding(encoding);
|
||||
for (name, column_id) in [
|
||||
(
|
||||
DATA_SCHEMA_TABLE_ID_COLUMN_NAME,
|
||||
@@ -1139,6 +1147,12 @@ mod test {
|
||||
let metadata = Arc::new(builder.build().unwrap());
|
||||
|
||||
for (projection, partitioned_by_tsid, claimed, unclaimed) in [
|
||||
(
|
||||
vec![DATA_SCHEMA_TABLE_ID_COLUMN_NAME, "A", "ts"],
|
||||
false,
|
||||
vec![vec![DATA_SCHEMA_TABLE_ID_COLUMN_NAME]],
|
||||
vec![vec!["A"], vec![DATA_SCHEMA_TABLE_ID_COLUMN_NAME, "ts"]],
|
||||
),
|
||||
// The logical projection drops the row key prefix, so `A` is not a prefix of the
|
||||
// row key and neither `A` nor `ts` is ordered.
|
||||
(
|
||||
|
||||
@@ -109,7 +109,7 @@ CREATE TABLE bad (
|
||||
host STRING PRIMARY KEY COMPRESSION WITH (level = 3)
|
||||
) WITH ('primary_key_encoding' = 'sparse');
|
||||
|
||||
Error: 1004(InvalidArguments), Invalid region options, COMPRESSION on tag column host requires flat SST format and dense primary key encoding
|
||||
Error: 1004(InvalidArguments), Invalid metadata, sparse primary keys must start with (__table_id, __tsid)
|
||||
|
||||
CREATE TABLE compressed_tags (
|
||||
ts TIMESTAMP TIME INDEX,
|
||||
|
||||
Reference in New Issue
Block a user