feat(mito2): expose scan-bound raw series candidates

Adapt the existing series reader through an optional scanner entrypoint with
owned snapshot-bound label metadata and complete encoded series keys. Preserve
selected field order and Arrow segment views without eager label decoding.

Keep ordinary storage pushdown and MVCC semantics. Inexact field predicates
can leave false-positive candidates; callers must retain residual columns and
evaluate the ordinary residual filter before final projection. This adds no
public native query consumer or replacement filter evaluator.

Exercise both internal batch forms, stream finality/time/error/drop boundaries,
real Mito candidate parity, and logical metric routing with immutable binding.

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
This commit is contained in:
discord9
2026-10-15 15:56:52 +08:00
parent 4f0cc3244a
commit 0f8e5330f7
6 changed files with 1570 additions and 3 deletions
+303 -1
View File
@@ -308,14 +308,19 @@ impl MetricEngineInner {
mod test {
use std::fmt;
use api::v1::Rows;
use api::v1::value::ValueData;
use api::v1::{Row, Rows, Value};
use common_error::ext::ErrorExt;
use common_error::status_code::StatusCode;
use datafusion::physical_plan::DisplayFormatType;
use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
use datatypes::series::raw::RawTimeAxis;
use datatypes::series::samples::SampleRef;
use futures_util::TryStreamExt;
use futures_util::future::try_join_all;
use mito_codec::row_converter::{SparseOffsetsCache, SparsePrimaryKeyCodec};
use mito2::config::MitoConfig;
use store_api::region_engine::{BoundTag, RawLabelSource};
use store_api::region_engine::{PrepareRequest, QueryScanContext};
use store_api::region_request::{RegionFlushRequest, RegionPutRequest, RegionRequest};
use store_api::storage::{TimeSeriesDistribution, TimeSeriesRowSelector};
@@ -552,6 +557,303 @@ mod test {
assert!(explain.contains("\"mode\":\"two_phase\""), "{explain}");
}
#[tokio::test]
async fn test_raw_series_scan_uses_logical_metric_route_and_bound_metadata() {
let env = TestEnv::with_mito_config(
"test_logical_raw_series_scan",
MitoConfig {
experimental_series_scan_v2: true,
..Default::default()
},
EngineConfig::default(),
)
.await;
env.init_metric_region().await;
let physical_region_id = env.default_physical_region_id();
let logical_a = env.default_logical_region_id();
let logical_b = RegionId::new(1024, logical_a.region_number());
let mut create_b = create_logical_region_request(
&["job", "private"],
physical_region_id,
"test_logical_raw_series_scan_b",
);
// Deliberately request ids different from A's assigned physical ids.
for (index, column) in create_b.column_metadatas.iter_mut().enumerate().skip(2) {
column.column_id = 100 + index as u32;
}
env.metric()
.handle_request(logical_b, RegionRequest::Create(create_b))
.await
.unwrap();
let write = |schema: Vec<api::v1::ColumnSchema>, tag_values: &[&[&str]], start: i64| {
let rows = tag_values
.iter()
.enumerate()
.map(|(offset, tags)| {
let mut values = vec![
Value {
value_data: Some(ValueData::TimestampMillisecondValue(
start + offset as i64,
)),
},
Value {
value_data: Some(ValueData::F64Value((start + offset as i64) as f64)),
},
];
values.extend(tags.iter().map(|tag| Value {
value_data: Some(ValueData::StringValue((*tag).to_string())),
}));
Row { values }
})
.collect();
RegionRequest::Put(RegionPutRequest {
skip_wal: false,
rows: Rows { schema, rows },
hint: None,
partition_expr_version: None,
})
};
env.metric()
.handle_request(
logical_a,
write(
test_util::row_schema_with_tags(&["job"]),
&[&["a-one"], &["a-two"]],
10,
),
)
.await
.unwrap();
env.metric()
.handle_request(
logical_b,
write(
test_util::row_schema_with_tags(&["job", "private"]),
&[&["b-one", "private-one"], &["b-two", "private-two"]],
100,
),
)
.await
.unwrap();
env.metric()
.handle_request(
physical_region_id,
RegionRequest::Flush(RegionFlushRequest::default()),
)
.await
.unwrap();
let logical_metadata = env
.metric()
.inner
.logical_region_metadata(physical_region_id, logical_a)
.await
.unwrap();
let metric = env.metric();
let mut scanner = metric
.handle_query(
logical_a,
ScanRequest {
distribution: Some(TimeSeriesDistribution::PerSeries),
projection: Some(vec![0, 1]), // job is bound although not projected.
..Default::default()
},
)
.await
.unwrap();
let physical_metadata = scanner.metadata();
assert_eq!(logical_metadata.region_id, logical_a);
assert_eq!(
scanner
.schema()
.column_schemas()
.iter()
.map(|column| column.name.as_str())
.collect::<Vec<_>>(),
vec!["greptime_timestamp", "greptime_value"]
);
let logical_tags = logical_metadata
.column_metadatas
.iter()
.filter(|column| column.semantic_type == SemanticType::Tag)
.map(|column| {
let physical = physical_metadata
.column_by_name(&column.column_schema.name)
.expect("logical tag must exist in scan metadata");
assert_eq!(physical.semantic_type, SemanticType::Tag);
assert_eq!(
physical.column_schema.data_type,
column.column_schema.data_type
);
BoundTag {
column_id: physical.column_id,
name: Arc::from(column.column_schema.name.as_str()),
}
})
.collect::<Vec<_>>();
assert_eq!(logical_tags.len(), 1);
assert_eq!(logical_tags[0].name.as_ref(), "job");
assert!(
!logical_tags
.iter()
.any(|tag| tag.name.as_ref() == "private")
);
let assigned_job_id = physical_metadata.column_by_name("job").unwrap().column_id;
assert_ne!(assigned_job_id, 102);
let logical_tags: Arc<[BoundTag]> = logical_tags.into();
let source = RawLabelSource::PrimaryKey {
metric_name: Arc::from("fixture_metric"),
physical_metadata: physical_metadata.clone(),
logical_tags: logical_tags.clone(),
};
// The acquired scanner and source retain their immutable schema snapshot across an alter.
env.metric()
.handle_request(
logical_b,
RegionRequest::Alter(alter_logical_region_add_tag_columns(
200,
&["later_private"],
)),
)
.await
.unwrap();
let fresh_scanner = metric
.handle_query(
logical_a,
ScanRequest {
distribution: Some(TimeSeriesDistribution::PerSeries),
projection: Some(vec![0, 1]),
..Default::default()
},
)
.await
.unwrap();
let fresh_metadata = fresh_scanner.metadata();
assert!(!Arc::ptr_eq(&physical_metadata, &fresh_metadata));
assert!(fresh_metadata.schema_version > physical_metadata.schema_version);
assert!(fresh_metadata.column_by_name("later_private").is_some());
assert!(physical_metadata.column_by_name("later_private").is_none());
assert!(Arc::ptr_eq(
match &source {
RawLabelSource::PrimaryKey {
physical_metadata, ..
} => physical_metadata,
RawLabelSource::Explicit => unreachable!(),
},
&scanner.metadata()
));
let mismatched_source = RawLabelSource::PrimaryKey {
metric_name: Arc::from("fixture_metric"),
physical_metadata: fresh_metadata,
logical_tags: logical_tags.clone(),
};
assert!(
scanner
.scan_series_partition(
&QueryScanContext::default(),
&ExecutionPlanMetricsSet::default(),
0,
&mismatched_source
)
.is_err()
);
drop(fresh_scanner);
let ranges = scanner
.properties()
.partitions
.iter()
.flatten()
.copied()
.collect::<Vec<_>>();
let partitions = scanner.properties().partitions.len().max(1);
let mut prepared_ranges = vec![Vec::new(); partitions];
prepared_ranges[0] = ranges;
scanner
.prepare(
PrepareRequest::default()
.with_ranges(prepared_ranges)
.with_target_partitions(partitions),
)
.unwrap();
let scan_context = QueryScanContext::default();
let scan_metrics = ExecutionPlanMetricsSet::default();
let mut samples = Vec::new();
let mut keys = std::collections::HashSet::new();
let mut decoded_jobs = std::collections::HashSet::new();
let mut last_markers = 0;
for partition in 0..partitions {
let stream = scanner
.scan_series_partition(&scan_context, &scan_metrics, partition, &source)
.unwrap()
.expect("metric scanner should support raw series");
assert_eq!(
stream
.schema()
.fields
.iter()
.map(|field| field.name.as_ref())
.collect::<Vec<_>>(),
vec!["greptime_value"]
);
match stream.label_source() {
RawLabelSource::PrimaryKey {
metric_name,
physical_metadata: bound_metadata,
logical_tags: bound_tags,
} => {
assert_eq!(metric_name.as_ref(), "fixture_metric");
assert!(Arc::ptr_eq(bound_metadata, &physical_metadata));
assert_eq!(bound_tags.len(), 1);
assert_eq!(bound_tags[0].name.as_ref(), "job");
assert_eq!(bound_tags[0].column_id, assigned_job_id);
}
RawLabelSource::Explicit => panic!("expected scan-bound primary-key source"),
}
let chunks = stream.try_collect::<Vec<_>>().await.unwrap();
for chunk in chunks {
assert!(chunk.metric.is_none());
assert!(!chunk.read_key.is_empty());
let key_bytes = chunk.read_key.as_slice();
let key_codec = SparsePrimaryKeyCodec::new(&physical_metadata);
let (table_id, _) = key_codec.decode_ids(key_bytes).unwrap();
assert_eq!(table_id, logical_a.table_id());
let mut offsets = SparseOffsetsCache::default();
let job_offset = key_codec.has_column(key_bytes, &mut offsets, assigned_job_id);
assert!(
job_offset.is_some(),
"complete identity must include job tag"
);
let job_value = key_codec
.decode_value_at(key_bytes, job_offset.unwrap(), assigned_job_id)
.unwrap();
decoded_jobs.insert(job_value.to_string());
keys.insert(key_bytes.to_vec());
last_markers += usize::from(chunk.is_last);
let RawTimeAxis::Explicit(times) = chunk.time;
samples.extend(times.iter().enumerate().map(|(index, time)| {
let value = match chunk.fields.get(0).unwrap().sample(index) {
SampleRef::Float(value) => value,
_ => panic!("expected a float sample"),
};
(*time, value)
}));
}
}
samples.sort_by(|left, right| left.0.cmp(&right.0));
assert_eq!(samples, vec![(10, 10.0), (11, 11.0)]);
assert_eq!(keys.len(), 2);
assert_eq!(
decoded_jobs,
["a-one".to_string(), "a-two".to_string()].into()
);
assert_eq!(last_markers, 2);
}
#[tokio::test]
async fn test_series_scan_v2_flag_disables_two_phase_mode() {
let env = TestEnv::with_mito_config(
+415
View File
@@ -4759,3 +4759,418 @@ fn scan_field_values(batches: &[common_recordbatch::RecordBatch]) -> Vec<f64> {
})
.collect()
}
/// Raw Mito output contains candidates after projection, filtering, and MVCC read processing.
#[tokio::test]
async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() {
use datatypes::arrow::array::Array;
use datatypes::series::raw::{RawSeriesOrdering, RawTimeAxis};
use datatypes::series::samples::SampleRef;
use datatypes::value::Value as DatumValue;
use mito_codec::row_converter::build_primary_key_codec;
use store_api::region_engine::{BoundTag, QueryScanContext, RawLabelSource};
let mut env =
TestEnv::with_prefix("test_mito_handle_query_raw_series_candidate_projection_and_mvcc")
.await;
let engine = env.create_engine(MitoConfig::default()).await;
let region_id = RegionId::new(1, 1);
let request = CreateRequestBuilder::new().field_num(3).build();
let schema = test_util::rows_schema(&request);
engine
.handle_request(region_id, RegionRequest::Create(request))
.await
.unwrap();
let row = |tag: &str, f0: Option<f64>, f1: Option<f64>, gate: f64, ts: i64| api::v1::Row {
values: vec![
api::v1::Value {
value_data: Some(ValueData::StringValue(tag.to_string())),
},
api::v1::Value {
value_data: f0.map(ValueData::F64Value),
},
api::v1::Value {
value_data: f1.map(ValueData::F64Value),
},
api::v1::Value {
value_data: Some(ValueData::F64Value(gate)),
},
api::v1::Value {
value_data: Some(ValueData::TimestampMillisecondValue(ts)),
},
],
};
let put = |rows| {
RegionRequest::Put(RegionPutRequest {
rows: Rows {
schema: schema.clone(),
rows,
},
hint: None,
skip_wal: false,
partition_expr_version: None,
})
};
engine
.handle_request(
region_id,
put(vec![
row("A", Some(1.0), Some(10.0), 1.0, 1000),
row("A", Some(2.0), None, 1.0, 2000),
row("B", Some(5.0), Some(-0.0), 1.0, 1000),
row("B", Some(6.0), Some(60.0), 0.0, 2000),
row("B", None, Some(70.0), 1.0, 3000),
]),
)
.await
.unwrap();
test_util::flush_region(&engine, region_id, None).await;
// Newer write supersedes A@1000; A@2000 is deleted. B@2000 fails field_2, but the raw producer emits candidates before the caller's residual filter.
engine
.handle_request(
region_id,
put(vec![row("A", Some(3.0), Some(30.0), 1.0, 1000)]),
)
.await
.unwrap();
engine
.handle_request(
region_id,
RegionRequest::Delete(store_api::region_request::RegionDeleteRequest {
rows: Rows {
schema: test_util::delete_rows_schema(
&CreateRequestBuilder::new().field_num(3).build(),
),
rows: vec![api::v1::Row {
values: vec![
api::v1::Value {
value_data: Some(ValueData::StringValue("A".to_string())),
},
api::v1::Value {
value_data: Some(ValueData::TimestampMillisecondValue(2000)),
},
],
}],
},
hint: None,
partition_expr_version: None,
}),
)
.await
.unwrap();
let request = ScanRequest {
projection: Some(vec![2, 1, 4]),
filters: vec![col("field_2").gt(lit(0.0_f64))],
distribution: Some(TimeSeriesDistribution::PerSeries),
preserve_pk_dictionary_encoding: true,
..Default::default()
};
let mut scanner = engine
.handle_query(region_id, request.clone())
.await
.unwrap();
let metadata = scanner.metadata();
let tags = metadata
.column_metadatas
.iter()
.filter(|c| c.semantic_type == SemanticType::Tag.into())
.map(|c| BoundTag {
column_id: c.column_id,
name: Arc::from(c.column_schema.name.as_str()),
})
.collect::<Vec<_>>();
let source = RawLabelSource::PrimaryKey {
metric_name: Arc::from("fixture_metric"),
physical_metadata: metadata.clone(),
logical_tags: tags.into(),
};
let ranges = scanner.properties().partitions.clone();
scanner
.prepare(PrepareRequest {
ranges: Some(ranges.clone()),
target_partitions: Some(ranges.len()),
..Default::default()
})
.unwrap();
let metrics = ExecutionPlanMetricsSet::default();
let mut raw = Vec::new();
for partition in 0..ranges.len() {
let mut previous_times = HashMap::<Vec<u8>, i64>::new();
let mut stream = scanner
.scan_series_partition(&QueryScanContext::default(), &metrics, partition, &source)
.unwrap()
.expect("supported millisecond PerSeries scan");
assert_eq!(RawSeriesOrdering::Contiguous, stream.ordering());
assert!(stream.schema().eval_grid.is_none());
assert_eq!(
vec!["field_1", "field_0"],
stream
.schema()
.fields
.iter()
.map(|f| f.name.as_ref())
.collect::<Vec<_>>()
);
let mut closed_keys = std::collections::HashSet::<Vec<u8>>::new();
while let Some(chunk) = stream.try_next().await.unwrap() {
assert!(chunk.metric.is_none());
assert!(
!closed_keys.contains(chunk.read_key.as_slice()),
"no chunk may follow a key's is_last chunk"
);
let RawTimeAxis::Explicit(times) = chunk.time;
assert_eq!(2, chunk.fields.len());
assert_eq!(times.len(), chunk.fields.get(0).unwrap().len());
assert_eq!(times.len(), chunk.fields.get(1).unwrap().len());
for (index, time) in times.iter().enumerate() {
let SampleRef::Float(f1) = chunk.fields.get(0).unwrap().sample(index) else {
panic!("f1 float")
};
let f0 = match chunk.fields.get(1).unwrap().sample(index) {
SampleRef::Float(v) => Some(v),
SampleRef::Missing => None,
_ => panic!("f0 type"),
};
let key = chunk.read_key.to_vec();
if let Some(previous) = previous_times.insert(key.clone(), *time) {
assert!(
previous < *time,
"raw time must increase strictly across chunks"
);
}
if chunk.is_last {
closed_keys.insert(chunk.read_key.to_vec());
}
raw.push((
chunk.read_key.clone(),
*time,
f1,
f0,
chunk.is_last && index + 1 == times.len(),
));
}
}
}
raw.sort_by(|a, b| a.2.total_cmp(&b.2));
assert!(
raw.iter()
.any(|(_, _, value, _, _)| value.to_bits() == (-0.0_f64).to_bits())
);
let codec = build_primary_key_codec(&metadata);
let mut expected_keys = Vec::new();
for tag in ["A", "B"] {
let mut key = Vec::new();
codec
.encode_values(
&[(metadata.primary_key[0], DatumValue::String(tag.into()))],
&mut key,
)
.unwrap();
expected_keys.push(key);
}
expected_keys.sort();
let mut actual_keys = raw
.iter()
.map(|(key, _, _, _, _)| key.to_vec())
.collect::<Vec<_>>();
actual_keys.sort();
actual_keys.dedup();
assert_eq!(expected_keys, actual_keys);
for key in &actual_keys {
assert_eq!(
1,
raw.iter()
.filter(|(k, _, _, _, last)| k.as_ref() == key && *last)
.count()
);
}
// Compare keyed visible tuples through a separately opened legacy scanner.
// A partition cannot be consumed through both paths.
let mut legacy_request = request;
legacy_request.projection = Some(vec![0, 2, 1, 4]);
legacy_request.preserve_pk_dictionary_encoding = false;
let legacy = engine
.handle_query(region_id, legacy_request)
.await
.unwrap();
let mut batches = Vec::new();
for partition in 0..legacy.properties().num_partitions() {
batches.extend(
legacy
.scan_partition(&QueryScanContext::default(), &metrics, partition)
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap(),
);
}
let visible = batches
.iter()
.flat_map(|batch| {
(0..batch.num_rows()).map(move |r| {
let f1 = batch
.column(1)
.as_any()
.downcast_ref::<Float64Array>()
.unwrap();
let f0 = batch
.column(2)
.as_any()
.downcast_ref::<Float64Array>()
.unwrap();
(
batch
.column(0)
.as_any()
.downcast_ref::<StringArray>()
.unwrap()
.value(r)
.to_string(),
batch
.column(3)
.as_any()
.downcast_ref::<TimestampMillisecondArray>()
.unwrap()
.value(r),
f1.value(r),
if f0.is_null(r) {
None
} else {
Some(f0.value(r))
},
)
})
})
.collect::<Vec<_>>();
assert_eq!(
vec![
(1000, -0.0, Some(5.0)),
(1000, 30.0, Some(3.0)),
(2000, 60.0, Some(6.0)),
(3000, 70.0, None)
],
raw.iter()
.map(|(_, t, a, b, _)| (*t, *a, *b))
.collect::<Vec<_>>()
);
let mut visible = visible;
visible.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1)));
let mut raw_visible = raw
.iter()
.map(|(key, time, f1, f0, _)| {
let tag = if key.as_ref() == expected_keys[0].as_slice() {
"A"
} else {
"B"
};
(tag.to_string(), *time, *f1, *f0)
})
.collect::<Vec<_>>();
raw_visible.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1)));
assert_eq!(raw_visible, visible);
// field_2's static predicate is residual work for the caller; B@2000 remains a candidate here.
// Applying that predicate later should remove B@2000 and leave three rows.
assert_eq!(
vec![
("A".to_string(), 1000, 30.0, Some(3.0)),
("B".to_string(), 1000, -0.0, Some(5.0)),
("B".to_string(), 2000, 60.0, Some(6.0)),
("B".to_string(), 3000, 70.0, None)
],
visible
);
}
/// Invalid raw bindings fail before consumption; explicit labels decline and legacy still reads.
#[tokio::test]
async fn test_mito_raw_series_invalid_binding_before_legacy_read() {
use store_api::region_engine::{BoundTag, QueryScanContext, RawLabelSource};
let mut env =
TestEnv::with_prefix("test_mito_raw_series_invalid_binding_before_legacy_read").await;
let engine = env.create_engine(MitoConfig::default()).await;
let region_id = RegionId::new(1, 1);
let request = CreateRequestBuilder::new().build();
let schema = test_util::rows_schema(&request);
engine
.handle_request(region_id, RegionRequest::Create(request))
.await
.unwrap();
test_util::put_rows(
&engine,
region_id,
Rows {
schema,
rows: test_util::build_rows(0, 2),
},
)
.await;
let scanner = engine
.handle_query(
region_id,
ScanRequest {
distribution: Some(TimeSeriesDistribution::PerSeries),
preserve_pk_dictionary_encoding: false,
..Default::default()
},
)
.await
.unwrap();
let metadata = scanner.metadata();
let metrics = ExecutionPlanMetricsSet::default();
let ctx = QueryScanContext::default();
assert!(
scanner
.scan_series_partition(&ctx, &metrics, 0, &RawLabelSource::Explicit)
.unwrap()
.is_none()
);
let valid_shape_bad_tag = RawLabelSource::PrimaryKey {
metric_name: Arc::from("fixture_metric"),
physical_metadata: metadata.clone(),
logical_tags: Arc::from([BoundTag {
column_id: metadata.primary_key[0],
name: Arc::from("wrong_name"),
}]),
};
assert!(
scanner
.scan_series_partition(&ctx, &metrics, 0, &valid_shape_bad_tag)
.is_err()
);
let batches = scanner.scan_partition(&ctx, &metrics, 0).unwrap();
let batches = RecordBatches::try_collect(batches).await.unwrap();
let rows = batches
.iter()
.flat_map(|batch| {
(0..batch.num_rows()).map(move |row| {
let tag = batch
.column_by_name("tag_0")
.unwrap()
.as_any()
.downcast_ref::<StringArray>()
.unwrap()
.value(row);
let field = batch
.column_by_name("field_0")
.unwrap()
.as_any()
.downcast_ref::<Float64Array>()
.unwrap()
.value(row);
let time = batch
.column_by_name("ts")
.unwrap()
.as_any()
.downcast_ref::<TimestampMillisecondArray>()
.unwrap()
.value(row);
(tag.to_string(), field, time)
})
})
.collect::<Vec<_>>();
assert_eq!(
vec![("0".to_string(), 0.0, 0), ("1".to_string(), 1.0, 1000)],
rows
);
}
+128
View File
@@ -14,6 +14,8 @@
//! Per-series scan implementation.
mod raw;
use std::fmt;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
@@ -564,6 +566,132 @@ impl RegionScanner for SeriesScan {
.map_err(BoxedError::new)
}
fn scan_series_partition(
&self,
ctx: &QueryScanContext,
metrics_set: &ExecutionPlanMetricsSet,
partition: usize,
label_source: &store_api::region_engine::RawLabelSource,
) -> Result<Option<store_api::region_engine::SendableRawSeriesStream>, BoxedError> {
use api::v1::SemanticType;
use common_query::native_histogram::is_native_histogram_value_type;
use datatypes::series::samples::{FieldSpec, SeriesSchema};
use store_api::metric_engine_consts::is_metric_engine_internal_column;
use store_api::region_engine::RawLabelSource;
use store_api::storage::consts::{ReservedColumnId, is_internal_column};
let input = &self.stream_ctx.input;
let metadata = input.mapper.metadata();
match label_source {
RawLabelSource::PrimaryKey {
physical_metadata,
logical_tags,
..
} => {
if !Arc::ptr_eq(metadata, physical_metadata) {
return Err(BoxedError::new(
crate::error::InvalidRecordBatchSnafu {
reason: "raw label source metadata does not match scanner snapshot"
.to_string(),
}
.build(),
));
}
let mut ids = std::collections::HashSet::new();
let mut names = std::collections::HashSet::new();
for tag in logical_tags.iter() {
let column = metadata.column_by_id(tag.column_id).filter(|column| {
column.semantic_type == SemanticType::Tag
&& metadata.primary_key.contains(&tag.column_id)
});
let valid = column.is_some_and(|column| {
column.column_schema.name == tag.name.as_ref()
&& tag.name.as_ref() != "__name__"
&& !is_internal_column(tag.name.as_ref())
&& !is_metric_engine_internal_column(tag.name.as_ref())
&& !ReservedColumnId::is_reserved(tag.column_id)
});
if !valid || !ids.insert(tag.column_id) || !names.insert(tag.name.as_ref()) {
return Err(BoxedError::new(
crate::error::InvalidRecordBatchSnafu {
reason: "invalid raw label binding".to_string(),
}
.build(),
));
}
}
}
RawLabelSource::Explicit => return Ok(None),
}
let selected = input.mapper.output_schema();
let mut fields = Vec::new();
let mut specs = Vec::new();
for output in selected.column_schemas() {
let Some(column) = metadata.column_metadatas.iter().find(|column| {
column.column_schema.name == output.name
&& column.semantic_type == SemanticType::Field
}) else {
continue;
};
let Some((index, (_, data_type))) = input
.mapper
.batch_schema()
.iter()
.enumerate()
.find(|(_, (id, _))| *id == column.column_id)
else {
return Err(BoxedError::new(
crate::error::InvalidRecordBatchSnafu {
reason: format!(
"selected field {} is missing from mapper batch schema",
output.name
),
}
.build(),
));
};
if !data_type.is_numeric() && !is_native_histogram_value_type(data_type) {
return Ok(None);
}
fields.push((index, data_type.clone()));
specs.push(FieldSpec {
name: Arc::from(output.name.as_str()),
});
}
let time_is_millis = matches!(
metadata.time_index_type(),
datatypes::types::TimestampType::Millisecond(_)
);
if fields.is_empty()
|| input.append_mode
|| input.series_row_selector.is_some()
|| !time_is_millis
{
return Ok(None);
}
let schema = Arc::new(SeriesSchema {
fields: specs.into(),
eval_grid: None,
});
let metrics = new_partition_metrics(
&self.stream_ctx,
ctx.explain_verbose,
metrics_set,
partition,
&self.metrics_list,
);
let batches = self
.scan_batch_in_partition(ctx, partition, metrics.clone(), metrics_set)
.map_err(BoxedError::new)?;
Ok(Some(raw::make_stream(
batches,
schema,
label_source.clone(),
fields,
metrics,
)))
}
fn prepare(&mut self, request: PrepareRequest) -> Result<(), BoxedError> {
self.properties.prepare(request);
+234
View File
@@ -0,0 +1,234 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#[cfg(test)]
mod tests;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::Instant;
use common_error::ext::BoxedError;
use common_query::native_histogram::{
histogram_samples_from_arrow, is_native_histogram_value_type,
};
use datatypes::arrow::array::{
Array, BinaryArray, DictionaryArray, Float64Array, StructArray, TimestampMillisecondArray,
};
use datatypes::arrow::buffer::Buffer;
use datatypes::arrow::compute::cast;
use datatypes::arrow::datatypes::UInt32Type;
use datatypes::arrow::record_batch::RecordBatch;
use datatypes::prelude::ConcreteDataType;
use datatypes::series::raw::{RawSeriesChunk, RawSeriesOrdering, RawTimeAxis};
use datatypes::series::samples::{Samples, SeriesFields, SeriesSchema};
use futures::{Stream, StreamExt};
use snafu::{IntoError, ResultExt};
use store_api::region_engine::{RawLabelSource, RawSeriesStream, SendableRawSeriesStream};
use crate::error::{ComputeArrowSnafu, DataTypeMismatchSnafu, InvalidRecordBatchSnafu};
use crate::read::scan_util::PartitionMetrics;
use crate::read::series_scan::SeriesBatch;
use crate::read::stream::{ScanBatch, ScanBatchStream};
use crate::sst::parquet::flat_format::{primary_key_column_index, time_index_column_index};
fn boxed(error: crate::error::Error) -> BoxedError {
BoxedError::new(error)
}
fn invalid(reason: impl Into<String>) -> BoxedError {
boxed(
InvalidRecordBatchSnafu {
reason: reason.into(),
}
.build(),
)
}
fn read_key(batch: &RecordBatch, row: usize) -> Result<&[u8], BoxedError> {
let column = batch.column(primary_key_column_index(batch.num_columns()));
if let Some(binary) = column.as_any().downcast_ref::<BinaryArray>() {
return binary
.is_valid(row)
.then(|| binary.value(row))
.ok_or_else(|| invalid("null primary key"));
}
let dictionary = column
.as_any()
.downcast_ref::<DictionaryArray<UInt32Type>>()
.ok_or_else(|| invalid("invalid primary-key array"))?;
let values = dictionary
.values()
.as_any()
.downcast_ref::<BinaryArray>()
.ok_or_else(|| invalid("invalid primary-key dictionary"))?;
if !dictionary.keys().is_valid(row) {
return Err(invalid("null primary key"));
}
let index = dictionary.keys().value(row) as usize;
if index >= values.len() || !values.is_valid(index) {
return Err(invalid("invalid primary-key dictionary value"));
}
Ok(values.value(index))
}
fn chunk(
batch: &RecordBatch,
start: usize,
end: usize,
schema: &SeriesSchema,
fields: &[(usize, ConcreteDataType)],
) -> Result<RawSeriesChunk, BoxedError> {
let key = read_key(batch, start)?;
let timestamp = batch
.column(time_index_column_index(batch.num_columns()))
.as_any()
.downcast_ref::<TimestampMillisecondArray>()
.ok_or_else(|| invalid("non-millisecond timestamp array"))?;
if (start..end).any(|row| !timestamp.is_valid(row)) {
return Err(invalid("null timestamp"));
}
let time = RawTimeAxis::try_new(timestamp.values().slice(start, end - start))
.context(DataTypeMismatchSnafu)
.map_err(BoxedError::new)?;
let mut samples = Vec::with_capacity(fields.len());
for (index, data_type) in fields {
let array = batch.column(*index).slice(start, end - start);
if is_native_histogram_value_type(data_type) {
let hist = array
.as_any()
.downcast_ref::<StructArray>()
.ok_or_else(|| invalid("invalid native histogram array"))?;
samples.push(histogram_samples_from_arrow(hist).map_err(|error| {
let source = common_query::error::Error::GeneralDataFusion {
error,
location: snafu::Location::new(file!(), line!(), column!()),
};
BoxedError::new(
crate::error::ExternalSnafu {
context: "native histogram conversion".to_string(),
}
.into_error(BoxedError::new(source)),
)
})?);
} else {
let values = cast(&array, &datatypes::arrow::datatypes::DataType::Float64)
.context(ComputeArrowSnafu)
.map_err(boxed)?;
let values = values
.as_any()
.downcast_ref::<Float64Array>()
.ok_or_else(|| invalid("invalid numeric field array"))?;
samples.push(Samples::from_float(values.clone()));
}
}
let fields = SeriesFields::try_from_vec(samples)
.context(DataTypeMismatchSnafu)
.map_err(BoxedError::new)?;
RawSeriesChunk::try_new(
schema,
Buffer::from(key.to_vec()),
None,
time,
fields,
false,
)
.context(DataTypeMismatchSnafu)
.map_err(BoxedError::new)
}
struct RawStream {
schema: Arc<SeriesSchema>,
label_source: RawLabelSource,
inner: futures::stream::BoxStream<'static, Result<RawSeriesChunk, BoxedError>>,
}
impl Stream for RawStream {
type Item = Result<RawSeriesChunk, BoxedError>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
self.inner.as_mut().poll_next(cx)
}
}
impl RawSeriesStream for RawStream {
fn schema(&self) -> &Arc<SeriesSchema> {
&self.schema
}
fn label_source(&self) -> &RawLabelSource {
&self.label_source
}
fn ordering(&self) -> RawSeriesOrdering {
RawSeriesOrdering::Contiguous
}
}
pub(crate) fn make_stream(
inner: ScanBatchStream,
schema: Arc<SeriesSchema>,
label_source: RawLabelSource,
fields: Vec<(usize, ConcreteDataType)>,
metrics: PartitionMetrics,
) -> SendableRawSeriesStream {
let output_schema = schema.clone();
let stream = async_stream::stream! {
let mut input = inner;
let mut pending: Option<RawSeriesChunk> = None;
while let Some(item) = input.next().await {
let records = match item {
Ok(ScanBatch::Series(SeriesBatch::Flat(flat))) => flat.batches.into_vec(),
Ok(ScanBatch::RecordBatch(batch)) => vec![batch],
Err(error) => { yield Err(BoxedError::new(error)); return; }
};
for record in records {
let mut start = 0;
while start < record.num_rows() {
let key = match read_key(&record, start) { Ok(key) => key, Err(error) => { yield Err(error); return; } };
let mut end = start + 1;
while end < record.num_rows() {
match read_key(&record, end) {
Ok(next) if next == key => end += 1,
Ok(_) => break,
Err(error) => { yield Err(error); return; }
}
}
let begun = Instant::now();
let current = match chunk(&record, start, end, &output_schema, &fields) {
Ok(chunk) => chunk, Err(error) => { yield Err(error); return; }
};
metrics.inc_convert_batch_cost(begun.elapsed());
if let Some(old) = pending.as_ref() {
if old.read_key.as_slice() == current.read_key.as_slice()
&& current.time.timestamp(0).zip(old.time.timestamp(old.time.len() - 1))
.is_some_and(|(next, previous)| next <= previous) {
yield Err(invalid("raw timestamps are not strictly increasing")); return;
}
}
if let Some(mut old) = pending.take() {
if old.read_key.as_slice() != current.read_key.as_slice() { old.is_last = true; }
yield Ok(old);
}
pending = Some(current);
start = end;
}
}
}
if let Some(mut last) = pending { last.is_last = true; yield Ok(last); }
};
Box::pin(RawStream {
schema,
label_source,
inner: Box::pin(stream),
})
}
+481
View File
@@ -0,0 +1,481 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::pin::Pin;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::task::{Context, Poll};
use std::time::Instant;
use common_error::ext::ErrorExt;
use common_query::native_histogram::{
CounterResetHint, NativeHistogram, Span, build_histogram_array,
};
use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet;
use datatypes::arrow::array::{
ArrayRef, BinaryArray, Int64Array, StructArray, TimestampMillisecondArray, UInt8Array,
UInt64Array,
};
use datatypes::arrow::datatypes::{DataType, Field, Schema};
use datatypes::arrow::record_batch::RecordBatch;
use datatypes::data_type::DataType as _;
use datatypes::series::raw::RawSeriesChunk;
use datatypes::series::samples::SampleRef;
use datatypes::series::samples::{FieldSpec, SeriesSchema};
use futures::{StreamExt, stream};
use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder};
use store_api::region_engine::{BoundTag, RawLabelSource, SendableRawSeriesStream};
use store_api::storage::RegionId;
use super::make_stream;
use crate::read::scan_util::PartitionMetrics;
use crate::read::series_scan::{FlatSeriesBatch, SeriesBatch};
use crate::read::stream::{ScanBatch, ScanBatchStream};
fn metrics() -> PartitionMetrics {
PartitionMetrics::new(
store_api::storage::RegionId::new(1, 0),
0,
"raw-test",
Instant::now(),
false,
&ExecutionPlanMetricsSet::new(),
)
}
fn schema() -> Arc<SeriesSchema> {
Arc::new(SeriesSchema {
fields: vec![FieldSpec {
name: Arc::from("value"),
}]
.into(),
eval_grid: None,
})
}
fn batch(keys: &[&[u8]], times: &[i64], values: &[Option<i64>]) -> RecordBatch {
assert_eq!(keys.len(), times.len());
assert_eq!(keys.len(), values.len());
let columns: Vec<ArrayRef> = vec![
Arc::new(Int64Array::from(values.to_vec())),
Arc::new(TimestampMillisecondArray::from(times.to_vec())),
Arc::new(BinaryArray::from_iter_values(keys.iter().copied())),
Arc::new(UInt64Array::from(vec![1; keys.len()])),
Arc::new(UInt8Array::from(vec![0; keys.len()])),
];
let schema = Arc::new(Schema::new(vec![
Field::new("value", DataType::Int64, true),
Field::new(
"ts",
DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None),
false,
),
Field::new("__primary_key", DataType::Binary, false),
Field::new("__sequence", DataType::UInt64, false),
Field::new("__op_type", DataType::UInt8, false),
]));
RecordBatch::try_new(schema, columns).unwrap()
}
fn histogram_input() -> (RecordBatch, Vec<NativeHistogram>) {
let expected = vec![NativeHistogram {
schema: 3,
zero_threshold: 0.25,
sum: 42.5,
reset_hint: CounterResetHint::Unrecognized(77),
start_timestamp: Some(1234),
custom_values: vec![1.5, 2.5],
positive_spans: vec![Span {
offset: 2,
length: 2,
}],
negative_spans: vec![Span {
offset: -1,
length: 1,
}],
count: 8.0,
zero_count: 2.0,
positive_buckets: vec![3.0, 4.0],
negative_buckets: vec![1.0],
}];
let array = build_histogram_array(&[Some(expected[0].clone())]);
let array = array
.as_any()
.downcast_ref::<StructArray>()
.unwrap()
.clone();
let columns: Vec<ArrayRef> = vec![
Arc::new(array),
Arc::new(TimestampMillisecondArray::from(vec![Some(1)])),
Arc::new(BinaryArray::from_iter_values([b"hist-key".as_slice()])),
Arc::new(UInt64Array::from(vec![1])),
Arc::new(UInt8Array::from(vec![0])),
];
let fields = vec![
Field::new(
"histogram",
common_query::native_histogram::native_histogram_value_type().as_arrow_type(),
true,
),
Field::new(
"ts",
DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None),
false,
),
Field::new("__primary_key", DataType::Binary, false),
Field::new("__sequence", DataType::UInt64, false),
Field::new("__op_type", DataType::UInt8, false),
];
(
RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap(),
expected,
)
}
fn input(items: Vec<crate::error::Result<ScanBatch>>) -> ScanBatchStream {
Box::pin(stream::iter(items))
}
fn metadata() -> Arc<store_api::metadata::RegionMetadata> {
metadata_with_field_type(datatypes::prelude::ConcreteDataType::int64_datatype())
}
fn metadata_with_field_type(
field_type: datatypes::prelude::ConcreteDataType,
) -> Arc<store_api::metadata::RegionMetadata> {
let mut builder = RegionMetadataBuilder::new(RegionId::new(1, 0));
builder.push_column_metadata(ColumnMetadata {
column_schema: datatypes::schema::ColumnSchema::new(
"tag",
datatypes::prelude::ConcreteDataType::string_datatype(),
true,
),
semantic_type: api::v1::SemanticType::Tag,
column_id: 0,
});
builder.push_column_metadata(ColumnMetadata {
column_schema: datatypes::schema::ColumnSchema::new(
"ts",
datatypes::prelude::ConcreteDataType::timestamp_millisecond_datatype(),
false,
),
semantic_type: api::v1::SemanticType::Timestamp,
column_id: 1,
});
builder.push_column_metadata(ColumnMetadata {
column_schema: datatypes::schema::ColumnSchema::new("value", field_type, true),
semantic_type: api::v1::SemanticType::Field,
column_id: 2,
});
builder.primary_key(vec![0]);
Arc::new(builder.build().unwrap())
}
fn convert_with_source(
inner: ScanBatchStream,
schema: Arc<SeriesSchema>,
source: RawLabelSource,
field_type: datatypes::prelude::ConcreteDataType,
) -> SendableRawSeriesStream {
make_stream(inner, schema, source, vec![(0, field_type)], metrics())
}
fn primary_key_source(metadata: Arc<store_api::metadata::RegionMetadata>) -> RawLabelSource {
RawLabelSource::PrimaryKey {
metric_name: Arc::from("metric"),
physical_metadata: metadata,
logical_tags: Arc::from([BoundTag {
column_id: 0,
name: Arc::from("tag"),
}]),
}
}
fn convert(inner: ScanBatchStream, schema: Arc<SeriesSchema>) -> SendableRawSeriesStream {
convert_with_source(
inner,
schema,
primary_key_source(metadata()),
datatypes::prelude::ConcreteDataType::int64_datatype(),
)
}
struct DropSpy {
inner: ScanBatchStream,
drops: Arc<AtomicUsize>,
}
impl futures::Stream for DropSpy {
type Item = crate::error::Result<ScanBatch>;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
self.inner.as_mut().poll_next(cx)
}
}
impl Drop for DropSpy {
fn drop(&mut self) {
self.drops.fetch_add(1, Ordering::SeqCst);
}
}
fn values(chunk: &RawSeriesChunk) -> Vec<Option<f64>> {
(0..chunk.time.len())
.map(|i| match chunk.fields.get(0).unwrap().sample(i) {
datatypes::series::samples::SampleRef::Missing => None,
datatypes::series::samples::SampleRef::Float(v) => Some(v),
_ => unreachable!(),
})
.collect()
}
#[tokio::test]
async fn chunks_preserve_keys_values_times_and_final_flags_for_both_input_shapes() {
let schema = schema();
let first = batch(
&[b"prefix-full-key-a", b"prefix-full-key-a"],
&[1, 2],
&[Some(1), None],
);
let second = batch(
&[
b"prefix-full-key-a",
b"prefix-full-key-a",
b"prefix-full-key-b",
b"prefix-full-key-b",
],
&[3, 4, 1, 2],
&[Some(3), Some(4), Some(5), Some(6)],
);
for flat in [true, false] {
let items = if flat {
let mut batches = smallvec::SmallVec::new();
batches.extend([first.clone(), second.clone()]);
vec![Ok(ScanBatch::Series(SeriesBatch::Flat(FlatSeriesBatch {
batches,
})))]
} else {
vec![
Ok(ScanBatch::RecordBatch(first.clone())),
Ok(ScanBatch::RecordBatch(second.clone())),
]
};
let mut output = convert(input(items), schema.clone());
let mut chunks = Vec::new();
while let Some(chunk) = output.next().await {
chunks.push(chunk.unwrap());
}
assert_eq!(
chunks
.iter()
.map(|c| c.read_key.as_slice())
.collect::<Vec<_>>(),
vec![
b"prefix-full-key-a".as_slice(),
b"prefix-full-key-a",
b"prefix-full-key-b"
]
);
assert_eq!(
chunks.iter().map(|c| c.is_last).collect::<Vec<_>>(),
vec![false, true, true]
);
assert_eq!(
chunks
.iter()
.map(|c| (0..c.time.len())
.map(|i| c.time.timestamp(i).unwrap())
.collect::<Vec<_>>())
.collect::<Vec<_>>(),
vec![vec![1, 2], vec![3, 4], vec![1, 2]]
);
assert_eq!(
chunks.iter().map(values).collect::<Vec<_>>(),
vec![
vec![Some(1.0), None],
vec![Some(3.0), Some(4.0)],
vec![Some(5.0), Some(6.0)]
]
);
}
}
#[tokio::test]
async fn upstream_error_is_terminal_after_yielded_prefix_without_flushing_pending() {
let upstream_error = crate::error::InvalidRecordBatchSnafu {
reason: "upstream injected failure".to_string(),
}
.build();
let mut output = convert(
input(vec![
Ok(ScanBatch::RecordBatch(batch(&[b"a"], &[1], &[Some(1)]))),
Ok(ScanBatch::RecordBatch(batch(&[b"a"], &[2], &[Some(2)]))),
Ok(ScanBatch::RecordBatch(batch(&[b"a"], &[3], &[Some(3)]))),
Err(upstream_error),
]),
schema(),
);
let first = output.next().await.unwrap().unwrap();
assert_eq!(first.read_key.as_slice(), b"a");
assert_eq!(first.time.timestamp(0), Some(1));
assert_eq!(values(&first), vec![Some(1.0)]);
assert!(first.metric.is_none());
assert!(!first.is_last);
let second = output.next().await.unwrap().unwrap();
assert_eq!(second.read_key.as_slice(), b"a");
assert_eq!(second.time.timestamp(0), Some(2));
assert_eq!(values(&second), vec![Some(2.0)]);
assert!(second.metric.is_none());
assert!(!second.is_last);
let error = output.next().await.unwrap().unwrap_err();
let Some(crate::error::Error::InvalidRecordBatch { reason, .. }) =
error.as_any().downcast_ref::<crate::error::Error>()
else {
panic!("unexpected upstream error: {error}");
};
assert_eq!(reason, "upstream injected failure");
assert!(output.next().await.is_none());
}
#[tokio::test]
async fn dropping_raw_stream_drops_unfinished_input() {
let drops = Arc::new(AtomicUsize::new(0));
let upstream = input(vec![
Ok(ScanBatch::RecordBatch(batch(&[b"a"], &[1], &[Some(1)]))),
Ok(ScanBatch::RecordBatch(batch(&[b"b"], &[1], &[Some(2)]))),
Ok(ScanBatch::RecordBatch(batch(&[b"c"], &[1], &[Some(3)]))),
]);
let spy = DropSpy {
inner: upstream,
drops: drops.clone(),
};
let mut raw = convert(Box::pin(spy), schema());
assert_eq!(raw.next().await.unwrap().unwrap().read_key.as_slice(), b"a");
drop(raw);
assert_eq!(drops.load(Ordering::SeqCst), 1);
}
#[tokio::test]
async fn raw_stream_owns_schema_and_bound_source_metadata() {
let schema = schema();
let schema_weak = Arc::downgrade(&schema);
let metadata = metadata();
let metadata_weak = Arc::downgrade(&metadata);
let mut raw = convert_with_source(
input(vec![]),
schema.clone(),
primary_key_source(metadata.clone()),
datatypes::prelude::ConcreteDataType::int64_datatype(),
);
drop(schema);
drop(metadata);
assert!(schema_weak.upgrade().is_some());
assert!(metadata_weak.upgrade().is_some());
assert_eq!(raw.schema().fields[0].name.as_ref(), "value");
match raw.label_source() {
RawLabelSource::PrimaryKey {
metric_name,
physical_metadata,
logical_tags,
} => {
assert_eq!(metric_name.as_ref(), "metric");
assert_eq!(physical_metadata.region_id, RegionId::new(1, 0));
assert_eq!(
physical_metadata.column_metadatas[0].column_schema.name,
"tag"
);
assert_eq!(logical_tags[0].column_id, 0);
assert_eq!(logical_tags[0].name.as_ref(), "tag");
assert!(Arc::ptr_eq(
physical_metadata,
&metadata_weak.upgrade().unwrap()
));
}
RawLabelSource::Explicit => panic!("expected bound primary-key labels"),
}
assert!(raw.next().await.is_none());
drop(raw);
assert!(schema_weak.upgrade().is_none());
assert!(metadata_weak.upgrade().is_none());
}
#[tokio::test]
async fn empty_input_is_a_successful_empty_stream() {
let mut output = convert(input(vec![]), schema());
assert!(output.next().await.is_none());
}
#[tokio::test]
async fn native_histogram_adapter_preserves_complete_payload() {
let (record, expected) = histogram_input();
let mut fields = schema().fields.to_vec();
fields[0].name = Arc::from("histogram");
let series_schema = Arc::new(SeriesSchema {
fields: fields.into(),
eval_grid: None,
});
let mut raw = convert_with_source(
input(vec![Ok(ScanBatch::RecordBatch(record))]),
series_schema,
primary_key_source(metadata_with_field_type(
common_query::native_histogram::native_histogram_value_type().clone(),
)),
common_query::native_histogram::native_histogram_value_type().clone(),
);
assert!(matches!(
raw.label_source(),
RawLabelSource::PrimaryKey { .. }
));
let chunk = raw.next().await.unwrap().unwrap();
assert!(chunk.metric.is_none());
assert!(chunk.is_last);
let SampleRef::Histogram(actual) = chunk.fields.get(0).unwrap().sample(0) else {
panic!("expected native histogram sample");
};
let expected = &expected[0];
assert_eq!(actual.schema, expected.schema);
assert_eq!(actual.zero_threshold, expected.zero_threshold);
assert_eq!(actual.sum, expected.sum);
assert_eq!(actual.count, expected.count);
assert_eq!(actual.zero_count, expected.zero_count);
assert_eq!(actual.reset_hint, expected.reset_hint);
assert_eq!(actual.start_timestamp, expected.start_timestamp);
assert_eq!(actual.custom_values, expected.custom_values);
assert_eq!(actual.positive.span_offsets, vec![2]);
assert_eq!(actual.positive.span_lengths, vec![2]);
assert_eq!(actual.positive.buckets, expected.positive_buckets);
assert_eq!(actual.negative.span_offsets, vec![-1]);
assert_eq!(actual.negative.span_lengths, vec![1]);
assert_eq!(actual.negative.buckets, expected.negative_buckets);
assert!(raw.next().await.is_none());
}
#[tokio::test]
async fn duplicate_or_descending_cross_batch_time_is_terminal() {
for next in [2, 1] {
let mut output = convert(
input(vec![
Ok(ScanBatch::RecordBatch(batch(
&[b"k", b"k"],
&[1, 2],
&[Some(1), Some(2)],
))),
Ok(ScanBatch::RecordBatch(batch(&[b"k"], &[next], &[Some(3)]))),
]),
schema(),
);
assert!(output.next().await.unwrap().is_err());
assert!(output.next().await.is_none());
}
}
+9 -2
View File
@@ -551,12 +551,19 @@ pub trait RegionScanner: Debug + DisplayAs + Send {
/// `None` means raw capability is absent before execution and permits a legacy scan instead.
/// `Some` selects this entrypoint, including for an empty successful stream. Errors propagate;
/// callers must not replay a failed raw scan through the legacy path. Use only one entrypoint
/// for a given scanner partition per execution.
/// for a given scanner partition per execution. The explicit label source is owned by the
/// returned stream and must be bound to this scanner's metadata snapshot.
///
/// Output preserves the ordinary storage reader's pushdown and MVCC semantics. Inexact
/// pushed-down predicates can leave false-positive candidates, just as in `scan_partition`.
/// Callers must retain residual predicate columns until those predicates are evaluated and
/// apply the final projection afterward; raw capability does not mean filtering is complete.
fn scan_series_partition(
&self,
_ctx: &QueryScanContext,
_metrics_set: &ExecutionPlanMetricsSet,
_partition: usize,
_label_source: &RawLabelSource,
) -> Result<Option<SendableRawSeriesStream>, BoxedError> {
Ok(None)
}
@@ -849,7 +856,7 @@ mod tests {
assert!(
scanner
.scan_series_partition(&ctx, &metrics, 0)
.scan_series_partition(&ctx, &metrics, 0, &RawLabelSource::Explicit)
.unwrap()
.is_none()
);