diff --git a/src/metric-engine/src/engine/read.rs b/src/metric-engine/src/engine/read.rs index 8d8d32fcc6c..1e10f7b9eb1 100644 --- a/src/metric-engine/src/engine/read.rs +++ b/src/metric-engine/src/engine/read.rs @@ -308,14 +308,19 @@ impl MetricEngineInner { mod test { use std::fmt; - use api::v1::Rows; + use api::v1::value::ValueData; + use api::v1::{Row, Rows, Value}; use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; use datafusion::physical_plan::DisplayFormatType; use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; + use datatypes::series::raw::RawTimeAxis; + use datatypes::series::samples::SampleRef; use futures_util::TryStreamExt; use futures_util::future::try_join_all; + use mito_codec::row_converter::{SparseOffsetsCache, SparsePrimaryKeyCodec}; use mito2::config::MitoConfig; + use store_api::region_engine::{BoundTag, RawLabelSource}; use store_api::region_engine::{PrepareRequest, QueryScanContext}; use store_api::region_request::{RegionFlushRequest, RegionPutRequest, RegionRequest}; use store_api::storage::{TimeSeriesDistribution, TimeSeriesRowSelector}; @@ -552,6 +557,303 @@ mod test { assert!(explain.contains("\"mode\":\"two_phase\""), "{explain}"); } + #[tokio::test] + async fn test_raw_series_scan_uses_logical_metric_route_and_bound_metadata() { + let env = TestEnv::with_mito_config( + "test_logical_raw_series_scan", + MitoConfig { + experimental_series_scan_v2: true, + ..Default::default() + }, + EngineConfig::default(), + ) + .await; + env.init_metric_region().await; + + let physical_region_id = env.default_physical_region_id(); + let logical_a = env.default_logical_region_id(); + let logical_b = RegionId::new(1024, logical_a.region_number()); + let mut create_b = create_logical_region_request( + &["job", "private"], + physical_region_id, + "test_logical_raw_series_scan_b", + ); + // Deliberately request ids different from A's assigned physical ids. + for (index, column) in create_b.column_metadatas.iter_mut().enumerate().skip(2) { + column.column_id = 100 + index as u32; + } + env.metric() + .handle_request(logical_b, RegionRequest::Create(create_b)) + .await + .unwrap(); + + let write = |schema: Vec, tag_values: &[&[&str]], start: i64| { + let rows = tag_values + .iter() + .enumerate() + .map(|(offset, tags)| { + let mut values = vec![ + Value { + value_data: Some(ValueData::TimestampMillisecondValue( + start + offset as i64, + )), + }, + Value { + value_data: Some(ValueData::F64Value((start + offset as i64) as f64)), + }, + ]; + values.extend(tags.iter().map(|tag| Value { + value_data: Some(ValueData::StringValue((*tag).to_string())), + })); + Row { values } + }) + .collect(); + RegionRequest::Put(RegionPutRequest { + skip_wal: false, + rows: Rows { schema, rows }, + hint: None, + partition_expr_version: None, + }) + }; + env.metric() + .handle_request( + logical_a, + write( + test_util::row_schema_with_tags(&["job"]), + &[&["a-one"], &["a-two"]], + 10, + ), + ) + .await + .unwrap(); + env.metric() + .handle_request( + logical_b, + write( + test_util::row_schema_with_tags(&["job", "private"]), + &[&["b-one", "private-one"], &["b-two", "private-two"]], + 100, + ), + ) + .await + .unwrap(); + env.metric() + .handle_request( + physical_region_id, + RegionRequest::Flush(RegionFlushRequest::default()), + ) + .await + .unwrap(); + + let logical_metadata = env + .metric() + .inner + .logical_region_metadata(physical_region_id, logical_a) + .await + .unwrap(); + let metric = env.metric(); + let mut scanner = metric + .handle_query( + logical_a, + ScanRequest { + distribution: Some(TimeSeriesDistribution::PerSeries), + projection: Some(vec![0, 1]), // job is bound although not projected. + ..Default::default() + }, + ) + .await + .unwrap(); + let physical_metadata = scanner.metadata(); + assert_eq!(logical_metadata.region_id, logical_a); + assert_eq!( + scanner + .schema() + .column_schemas() + .iter() + .map(|column| column.name.as_str()) + .collect::>(), + vec!["greptime_timestamp", "greptime_value"] + ); + + let logical_tags = logical_metadata + .column_metadatas + .iter() + .filter(|column| column.semantic_type == SemanticType::Tag) + .map(|column| { + let physical = physical_metadata + .column_by_name(&column.column_schema.name) + .expect("logical tag must exist in scan metadata"); + assert_eq!(physical.semantic_type, SemanticType::Tag); + assert_eq!( + physical.column_schema.data_type, + column.column_schema.data_type + ); + BoundTag { + column_id: physical.column_id, + name: Arc::from(column.column_schema.name.as_str()), + } + }) + .collect::>(); + assert_eq!(logical_tags.len(), 1); + assert_eq!(logical_tags[0].name.as_ref(), "job"); + assert!( + !logical_tags + .iter() + .any(|tag| tag.name.as_ref() == "private") + ); + let assigned_job_id = physical_metadata.column_by_name("job").unwrap().column_id; + assert_ne!(assigned_job_id, 102); + let logical_tags: Arc<[BoundTag]> = logical_tags.into(); + let source = RawLabelSource::PrimaryKey { + metric_name: Arc::from("fixture_metric"), + physical_metadata: physical_metadata.clone(), + logical_tags: logical_tags.clone(), + }; + + // The acquired scanner and source retain their immutable schema snapshot across an alter. + env.metric() + .handle_request( + logical_b, + RegionRequest::Alter(alter_logical_region_add_tag_columns( + 200, + &["later_private"], + )), + ) + .await + .unwrap(); + let fresh_scanner = metric + .handle_query( + logical_a, + ScanRequest { + distribution: Some(TimeSeriesDistribution::PerSeries), + projection: Some(vec![0, 1]), + ..Default::default() + }, + ) + .await + .unwrap(); + let fresh_metadata = fresh_scanner.metadata(); + assert!(!Arc::ptr_eq(&physical_metadata, &fresh_metadata)); + assert!(fresh_metadata.schema_version > physical_metadata.schema_version); + assert!(fresh_metadata.column_by_name("later_private").is_some()); + assert!(physical_metadata.column_by_name("later_private").is_none()); + assert!(Arc::ptr_eq( + match &source { + RawLabelSource::PrimaryKey { + physical_metadata, .. + } => physical_metadata, + RawLabelSource::Explicit => unreachable!(), + }, + &scanner.metadata() + )); + let mismatched_source = RawLabelSource::PrimaryKey { + metric_name: Arc::from("fixture_metric"), + physical_metadata: fresh_metadata, + logical_tags: logical_tags.clone(), + }; + assert!( + scanner + .scan_series_partition( + &QueryScanContext::default(), + &ExecutionPlanMetricsSet::default(), + 0, + &mismatched_source + ) + .is_err() + ); + drop(fresh_scanner); + + let ranges = scanner + .properties() + .partitions + .iter() + .flatten() + .copied() + .collect::>(); + let partitions = scanner.properties().partitions.len().max(1); + let mut prepared_ranges = vec![Vec::new(); partitions]; + prepared_ranges[0] = ranges; + scanner + .prepare( + PrepareRequest::default() + .with_ranges(prepared_ranges) + .with_target_partitions(partitions), + ) + .unwrap(); + let scan_context = QueryScanContext::default(); + let scan_metrics = ExecutionPlanMetricsSet::default(); + let mut samples = Vec::new(); + let mut keys = std::collections::HashSet::new(); + let mut decoded_jobs = std::collections::HashSet::new(); + let mut last_markers = 0; + for partition in 0..partitions { + let stream = scanner + .scan_series_partition(&scan_context, &scan_metrics, partition, &source) + .unwrap() + .expect("metric scanner should support raw series"); + assert_eq!( + stream + .schema() + .fields + .iter() + .map(|field| field.name.as_ref()) + .collect::>(), + vec!["greptime_value"] + ); + match stream.label_source() { + RawLabelSource::PrimaryKey { + metric_name, + physical_metadata: bound_metadata, + logical_tags: bound_tags, + } => { + assert_eq!(metric_name.as_ref(), "fixture_metric"); + assert!(Arc::ptr_eq(bound_metadata, &physical_metadata)); + assert_eq!(bound_tags.len(), 1); + assert_eq!(bound_tags[0].name.as_ref(), "job"); + assert_eq!(bound_tags[0].column_id, assigned_job_id); + } + RawLabelSource::Explicit => panic!("expected scan-bound primary-key source"), + } + let chunks = stream.try_collect::>().await.unwrap(); + for chunk in chunks { + assert!(chunk.metric.is_none()); + assert!(!chunk.read_key.is_empty()); + let key_bytes = chunk.read_key.as_slice(); + let key_codec = SparsePrimaryKeyCodec::new(&physical_metadata); + let (table_id, _) = key_codec.decode_ids(key_bytes).unwrap(); + assert_eq!(table_id, logical_a.table_id()); + let mut offsets = SparseOffsetsCache::default(); + let job_offset = key_codec.has_column(key_bytes, &mut offsets, assigned_job_id); + assert!( + job_offset.is_some(), + "complete identity must include job tag" + ); + let job_value = key_codec + .decode_value_at(key_bytes, job_offset.unwrap(), assigned_job_id) + .unwrap(); + decoded_jobs.insert(job_value.to_string()); + keys.insert(key_bytes.to_vec()); + last_markers += usize::from(chunk.is_last); + let RawTimeAxis::Explicit(times) = chunk.time; + samples.extend(times.iter().enumerate().map(|(index, time)| { + let value = match chunk.fields.get(0).unwrap().sample(index) { + SampleRef::Float(value) => value, + _ => panic!("expected a float sample"), + }; + (*time, value) + })); + } + } + samples.sort_by(|left, right| left.0.cmp(&right.0)); + assert_eq!(samples, vec![(10, 10.0), (11, 11.0)]); + assert_eq!(keys.len(), 2); + assert_eq!( + decoded_jobs, + ["a-one".to_string(), "a-two".to_string()].into() + ); + assert_eq!(last_markers, 2); + } + #[tokio::test] async fn test_series_scan_v2_flag_disables_two_phase_mode() { let env = TestEnv::with_mito_config( diff --git a/src/mito2/src/engine/scan_test.rs b/src/mito2/src/engine/scan_test.rs index f55f084ebd7..9d72b1a3e8e 100644 --- a/src/mito2/src/engine/scan_test.rs +++ b/src/mito2/src/engine/scan_test.rs @@ -4759,3 +4759,418 @@ fn scan_field_values(batches: &[common_recordbatch::RecordBatch]) -> Vec { }) .collect() } + +/// Raw Mito output contains candidates after projection, filtering, and MVCC read processing. +#[tokio::test] +async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() { + use datatypes::arrow::array::Array; + use datatypes::series::raw::{RawSeriesOrdering, RawTimeAxis}; + use datatypes::series::samples::SampleRef; + use datatypes::value::Value as DatumValue; + use mito_codec::row_converter::build_primary_key_codec; + use store_api::region_engine::{BoundTag, QueryScanContext, RawLabelSource}; + + let mut env = + TestEnv::with_prefix("test_mito_handle_query_raw_series_candidate_projection_and_mvcc") + .await; + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + let request = CreateRequestBuilder::new().field_num(3).build(); + let schema = test_util::rows_schema(&request); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + let row = |tag: &str, f0: Option, f1: Option, gate: f64, ts: i64| api::v1::Row { + values: vec![ + api::v1::Value { + value_data: Some(ValueData::StringValue(tag.to_string())), + }, + api::v1::Value { + value_data: f0.map(ValueData::F64Value), + }, + api::v1::Value { + value_data: f1.map(ValueData::F64Value), + }, + api::v1::Value { + value_data: Some(ValueData::F64Value(gate)), + }, + api::v1::Value { + value_data: Some(ValueData::TimestampMillisecondValue(ts)), + }, + ], + }; + let put = |rows| { + RegionRequest::Put(RegionPutRequest { + rows: Rows { + schema: schema.clone(), + rows, + }, + hint: None, + skip_wal: false, + partition_expr_version: None, + }) + }; + engine + .handle_request( + region_id, + put(vec![ + row("A", Some(1.0), Some(10.0), 1.0, 1000), + row("A", Some(2.0), None, 1.0, 2000), + row("B", Some(5.0), Some(-0.0), 1.0, 1000), + row("B", Some(6.0), Some(60.0), 0.0, 2000), + row("B", None, Some(70.0), 1.0, 3000), + ]), + ) + .await + .unwrap(); + test_util::flush_region(&engine, region_id, None).await; + // Newer write supersedes A@1000; A@2000 is deleted. B@2000 fails field_2, but the raw producer emits candidates before the caller's residual filter. + engine + .handle_request( + region_id, + put(vec![row("A", Some(3.0), Some(30.0), 1.0, 1000)]), + ) + .await + .unwrap(); + engine + .handle_request( + region_id, + RegionRequest::Delete(store_api::region_request::RegionDeleteRequest { + rows: Rows { + schema: test_util::delete_rows_schema( + &CreateRequestBuilder::new().field_num(3).build(), + ), + rows: vec![api::v1::Row { + values: vec![ + api::v1::Value { + value_data: Some(ValueData::StringValue("A".to_string())), + }, + api::v1::Value { + value_data: Some(ValueData::TimestampMillisecondValue(2000)), + }, + ], + }], + }, + hint: None, + partition_expr_version: None, + }), + ) + .await + .unwrap(); + + let request = ScanRequest { + projection: Some(vec![2, 1, 4]), + filters: vec![col("field_2").gt(lit(0.0_f64))], + distribution: Some(TimeSeriesDistribution::PerSeries), + preserve_pk_dictionary_encoding: true, + ..Default::default() + }; + let mut scanner = engine + .handle_query(region_id, request.clone()) + .await + .unwrap(); + let metadata = scanner.metadata(); + let tags = metadata + .column_metadatas + .iter() + .filter(|c| c.semantic_type == SemanticType::Tag.into()) + .map(|c| BoundTag { + column_id: c.column_id, + name: Arc::from(c.column_schema.name.as_str()), + }) + .collect::>(); + let source = RawLabelSource::PrimaryKey { + metric_name: Arc::from("fixture_metric"), + physical_metadata: metadata.clone(), + logical_tags: tags.into(), + }; + let ranges = scanner.properties().partitions.clone(); + scanner + .prepare(PrepareRequest { + ranges: Some(ranges.clone()), + target_partitions: Some(ranges.len()), + ..Default::default() + }) + .unwrap(); + let metrics = ExecutionPlanMetricsSet::default(); + let mut raw = Vec::new(); + for partition in 0..ranges.len() { + let mut previous_times = HashMap::, i64>::new(); + let mut stream = scanner + .scan_series_partition(&QueryScanContext::default(), &metrics, partition, &source) + .unwrap() + .expect("supported millisecond PerSeries scan"); + assert_eq!(RawSeriesOrdering::Contiguous, stream.ordering()); + assert!(stream.schema().eval_grid.is_none()); + assert_eq!( + vec!["field_1", "field_0"], + stream + .schema() + .fields + .iter() + .map(|f| f.name.as_ref()) + .collect::>() + ); + let mut closed_keys = std::collections::HashSet::>::new(); + while let Some(chunk) = stream.try_next().await.unwrap() { + assert!(chunk.metric.is_none()); + assert!( + !closed_keys.contains(chunk.read_key.as_slice()), + "no chunk may follow a key's is_last chunk" + ); + let RawTimeAxis::Explicit(times) = chunk.time; + assert_eq!(2, chunk.fields.len()); + assert_eq!(times.len(), chunk.fields.get(0).unwrap().len()); + assert_eq!(times.len(), chunk.fields.get(1).unwrap().len()); + for (index, time) in times.iter().enumerate() { + let SampleRef::Float(f1) = chunk.fields.get(0).unwrap().sample(index) else { + panic!("f1 float") + }; + let f0 = match chunk.fields.get(1).unwrap().sample(index) { + SampleRef::Float(v) => Some(v), + SampleRef::Missing => None, + _ => panic!("f0 type"), + }; + let key = chunk.read_key.to_vec(); + if let Some(previous) = previous_times.insert(key.clone(), *time) { + assert!( + previous < *time, + "raw time must increase strictly across chunks" + ); + } + if chunk.is_last { + closed_keys.insert(chunk.read_key.to_vec()); + } + raw.push(( + chunk.read_key.clone(), + *time, + f1, + f0, + chunk.is_last && index + 1 == times.len(), + )); + } + } + } + raw.sort_by(|a, b| a.2.total_cmp(&b.2)); + assert!( + raw.iter() + .any(|(_, _, value, _, _)| value.to_bits() == (-0.0_f64).to_bits()) + ); + let codec = build_primary_key_codec(&metadata); + let mut expected_keys = Vec::new(); + for tag in ["A", "B"] { + let mut key = Vec::new(); + codec + .encode_values( + &[(metadata.primary_key[0], DatumValue::String(tag.into()))], + &mut key, + ) + .unwrap(); + expected_keys.push(key); + } + expected_keys.sort(); + let mut actual_keys = raw + .iter() + .map(|(key, _, _, _, _)| key.to_vec()) + .collect::>(); + actual_keys.sort(); + actual_keys.dedup(); + assert_eq!(expected_keys, actual_keys); + for key in &actual_keys { + assert_eq!( + 1, + raw.iter() + .filter(|(k, _, _, _, last)| k.as_ref() == key && *last) + .count() + ); + } + // Compare keyed visible tuples through a separately opened legacy scanner. + // A partition cannot be consumed through both paths. + let mut legacy_request = request; + legacy_request.projection = Some(vec![0, 2, 1, 4]); + legacy_request.preserve_pk_dictionary_encoding = false; + let legacy = engine + .handle_query(region_id, legacy_request) + .await + .unwrap(); + let mut batches = Vec::new(); + for partition in 0..legacy.properties().num_partitions() { + batches.extend( + legacy + .scan_partition(&QueryScanContext::default(), &metrics, partition) + .unwrap() + .try_collect::>() + .await + .unwrap(), + ); + } + let visible = batches + .iter() + .flat_map(|batch| { + (0..batch.num_rows()).map(move |r| { + let f1 = batch + .column(1) + .as_any() + .downcast_ref::() + .unwrap(); + let f0 = batch + .column(2) + .as_any() + .downcast_ref::() + .unwrap(); + ( + batch + .column(0) + .as_any() + .downcast_ref::() + .unwrap() + .value(r) + .to_string(), + batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap() + .value(r), + f1.value(r), + if f0.is_null(r) { + None + } else { + Some(f0.value(r)) + }, + ) + }) + }) + .collect::>(); + assert_eq!( + vec![ + (1000, -0.0, Some(5.0)), + (1000, 30.0, Some(3.0)), + (2000, 60.0, Some(6.0)), + (3000, 70.0, None) + ], + raw.iter() + .map(|(_, t, a, b, _)| (*t, *a, *b)) + .collect::>() + ); + let mut visible = visible; + visible.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1))); + let mut raw_visible = raw + .iter() + .map(|(key, time, f1, f0, _)| { + let tag = if key.as_ref() == expected_keys[0].as_slice() { + "A" + } else { + "B" + }; + (tag.to_string(), *time, *f1, *f0) + }) + .collect::>(); + raw_visible.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.cmp(&b.1))); + assert_eq!(raw_visible, visible); + // field_2's static predicate is residual work for the caller; B@2000 remains a candidate here. + // Applying that predicate later should remove B@2000 and leave three rows. + assert_eq!( + vec![ + ("A".to_string(), 1000, 30.0, Some(3.0)), + ("B".to_string(), 1000, -0.0, Some(5.0)), + ("B".to_string(), 2000, 60.0, Some(6.0)), + ("B".to_string(), 3000, 70.0, None) + ], + visible + ); +} + +/// Invalid raw bindings fail before consumption; explicit labels decline and legacy still reads. +#[tokio::test] +async fn test_mito_raw_series_invalid_binding_before_legacy_read() { + use store_api::region_engine::{BoundTag, QueryScanContext, RawLabelSource}; + let mut env = + TestEnv::with_prefix("test_mito_raw_series_invalid_binding_before_legacy_read").await; + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + let request = CreateRequestBuilder::new().build(); + let schema = test_util::rows_schema(&request); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + test_util::put_rows( + &engine, + region_id, + Rows { + schema, + rows: test_util::build_rows(0, 2), + }, + ) + .await; + let scanner = engine + .handle_query( + region_id, + ScanRequest { + distribution: Some(TimeSeriesDistribution::PerSeries), + preserve_pk_dictionary_encoding: false, + ..Default::default() + }, + ) + .await + .unwrap(); + let metadata = scanner.metadata(); + let metrics = ExecutionPlanMetricsSet::default(); + let ctx = QueryScanContext::default(); + assert!( + scanner + .scan_series_partition(&ctx, &metrics, 0, &RawLabelSource::Explicit) + .unwrap() + .is_none() + ); + let valid_shape_bad_tag = RawLabelSource::PrimaryKey { + metric_name: Arc::from("fixture_metric"), + physical_metadata: metadata.clone(), + logical_tags: Arc::from([BoundTag { + column_id: metadata.primary_key[0], + name: Arc::from("wrong_name"), + }]), + }; + assert!( + scanner + .scan_series_partition(&ctx, &metrics, 0, &valid_shape_bad_tag) + .is_err() + ); + let batches = scanner.scan_partition(&ctx, &metrics, 0).unwrap(); + let batches = RecordBatches::try_collect(batches).await.unwrap(); + let rows = batches + .iter() + .flat_map(|batch| { + (0..batch.num_rows()).map(move |row| { + let tag = batch + .column_by_name("tag_0") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(row); + let field = batch + .column_by_name("field_0") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(row); + let time = batch + .column_by_name("ts") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap() + .value(row); + (tag.to_string(), field, time) + }) + }) + .collect::>(); + assert_eq!( + vec![("0".to_string(), 0.0, 0), ("1".to_string(), 1.0, 1000)], + rows + ); +} diff --git a/src/mito2/src/read/series_scan.rs b/src/mito2/src/read/series_scan.rs index 4328a20f59c..7238f8b3577 100644 --- a/src/mito2/src/read/series_scan.rs +++ b/src/mito2/src/read/series_scan.rs @@ -14,6 +14,8 @@ //! Per-series scan implementation. +mod raw; + use std::fmt; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; @@ -564,6 +566,132 @@ impl RegionScanner for SeriesScan { .map_err(BoxedError::new) } + fn scan_series_partition( + &self, + ctx: &QueryScanContext, + metrics_set: &ExecutionPlanMetricsSet, + partition: usize, + label_source: &store_api::region_engine::RawLabelSource, + ) -> Result, BoxedError> { + use api::v1::SemanticType; + use common_query::native_histogram::is_native_histogram_value_type; + use datatypes::series::samples::{FieldSpec, SeriesSchema}; + use store_api::metric_engine_consts::is_metric_engine_internal_column; + use store_api::region_engine::RawLabelSource; + use store_api::storage::consts::{ReservedColumnId, is_internal_column}; + + let input = &self.stream_ctx.input; + let metadata = input.mapper.metadata(); + match label_source { + RawLabelSource::PrimaryKey { + physical_metadata, + logical_tags, + .. + } => { + if !Arc::ptr_eq(metadata, physical_metadata) { + return Err(BoxedError::new( + crate::error::InvalidRecordBatchSnafu { + reason: "raw label source metadata does not match scanner snapshot" + .to_string(), + } + .build(), + )); + } + let mut ids = std::collections::HashSet::new(); + let mut names = std::collections::HashSet::new(); + for tag in logical_tags.iter() { + let column = metadata.column_by_id(tag.column_id).filter(|column| { + column.semantic_type == SemanticType::Tag + && metadata.primary_key.contains(&tag.column_id) + }); + let valid = column.is_some_and(|column| { + column.column_schema.name == tag.name.as_ref() + && tag.name.as_ref() != "__name__" + && !is_internal_column(tag.name.as_ref()) + && !is_metric_engine_internal_column(tag.name.as_ref()) + && !ReservedColumnId::is_reserved(tag.column_id) + }); + if !valid || !ids.insert(tag.column_id) || !names.insert(tag.name.as_ref()) { + return Err(BoxedError::new( + crate::error::InvalidRecordBatchSnafu { + reason: "invalid raw label binding".to_string(), + } + .build(), + )); + } + } + } + RawLabelSource::Explicit => return Ok(None), + } + let selected = input.mapper.output_schema(); + let mut fields = Vec::new(); + let mut specs = Vec::new(); + for output in selected.column_schemas() { + let Some(column) = metadata.column_metadatas.iter().find(|column| { + column.column_schema.name == output.name + && column.semantic_type == SemanticType::Field + }) else { + continue; + }; + let Some((index, (_, data_type))) = input + .mapper + .batch_schema() + .iter() + .enumerate() + .find(|(_, (id, _))| *id == column.column_id) + else { + return Err(BoxedError::new( + crate::error::InvalidRecordBatchSnafu { + reason: format!( + "selected field {} is missing from mapper batch schema", + output.name + ), + } + .build(), + )); + }; + if !data_type.is_numeric() && !is_native_histogram_value_type(data_type) { + return Ok(None); + } + fields.push((index, data_type.clone())); + specs.push(FieldSpec { + name: Arc::from(output.name.as_str()), + }); + } + let time_is_millis = matches!( + metadata.time_index_type(), + datatypes::types::TimestampType::Millisecond(_) + ); + if fields.is_empty() + || input.append_mode + || input.series_row_selector.is_some() + || !time_is_millis + { + return Ok(None); + } + let schema = Arc::new(SeriesSchema { + fields: specs.into(), + eval_grid: None, + }); + let metrics = new_partition_metrics( + &self.stream_ctx, + ctx.explain_verbose, + metrics_set, + partition, + &self.metrics_list, + ); + let batches = self + .scan_batch_in_partition(ctx, partition, metrics.clone(), metrics_set) + .map_err(BoxedError::new)?; + Ok(Some(raw::make_stream( + batches, + schema, + label_source.clone(), + fields, + metrics, + ))) + } + fn prepare(&mut self, request: PrepareRequest) -> Result<(), BoxedError> { self.properties.prepare(request); diff --git a/src/mito2/src/read/series_scan/raw.rs b/src/mito2/src/read/series_scan/raw.rs new file mode 100644 index 00000000000..8df41d68aed --- /dev/null +++ b/src/mito2/src/read/series_scan/raw.rs @@ -0,0 +1,234 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#[cfg(test)] +mod tests; + +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Instant; + +use common_error::ext::BoxedError; +use common_query::native_histogram::{ + histogram_samples_from_arrow, is_native_histogram_value_type, +}; +use datatypes::arrow::array::{ + Array, BinaryArray, DictionaryArray, Float64Array, StructArray, TimestampMillisecondArray, +}; +use datatypes::arrow::buffer::Buffer; +use datatypes::arrow::compute::cast; +use datatypes::arrow::datatypes::UInt32Type; +use datatypes::arrow::record_batch::RecordBatch; +use datatypes::prelude::ConcreteDataType; +use datatypes::series::raw::{RawSeriesChunk, RawSeriesOrdering, RawTimeAxis}; +use datatypes::series::samples::{Samples, SeriesFields, SeriesSchema}; +use futures::{Stream, StreamExt}; +use snafu::{IntoError, ResultExt}; +use store_api::region_engine::{RawLabelSource, RawSeriesStream, SendableRawSeriesStream}; + +use crate::error::{ComputeArrowSnafu, DataTypeMismatchSnafu, InvalidRecordBatchSnafu}; +use crate::read::scan_util::PartitionMetrics; +use crate::read::series_scan::SeriesBatch; +use crate::read::stream::{ScanBatch, ScanBatchStream}; +use crate::sst::parquet::flat_format::{primary_key_column_index, time_index_column_index}; + +fn boxed(error: crate::error::Error) -> BoxedError { + BoxedError::new(error) +} + +fn invalid(reason: impl Into) -> BoxedError { + boxed( + InvalidRecordBatchSnafu { + reason: reason.into(), + } + .build(), + ) +} + +fn read_key(batch: &RecordBatch, row: usize) -> Result<&[u8], BoxedError> { + let column = batch.column(primary_key_column_index(batch.num_columns())); + if let Some(binary) = column.as_any().downcast_ref::() { + return binary + .is_valid(row) + .then(|| binary.value(row)) + .ok_or_else(|| invalid("null primary key")); + } + let dictionary = column + .as_any() + .downcast_ref::>() + .ok_or_else(|| invalid("invalid primary-key array"))?; + let values = dictionary + .values() + .as_any() + .downcast_ref::() + .ok_or_else(|| invalid("invalid primary-key dictionary"))?; + if !dictionary.keys().is_valid(row) { + return Err(invalid("null primary key")); + } + let index = dictionary.keys().value(row) as usize; + if index >= values.len() || !values.is_valid(index) { + return Err(invalid("invalid primary-key dictionary value")); + } + Ok(values.value(index)) +} + +fn chunk( + batch: &RecordBatch, + start: usize, + end: usize, + schema: &SeriesSchema, + fields: &[(usize, ConcreteDataType)], +) -> Result { + let key = read_key(batch, start)?; + let timestamp = batch + .column(time_index_column_index(batch.num_columns())) + .as_any() + .downcast_ref::() + .ok_or_else(|| invalid("non-millisecond timestamp array"))?; + if (start..end).any(|row| !timestamp.is_valid(row)) { + return Err(invalid("null timestamp")); + } + let time = RawTimeAxis::try_new(timestamp.values().slice(start, end - start)) + .context(DataTypeMismatchSnafu) + .map_err(BoxedError::new)?; + let mut samples = Vec::with_capacity(fields.len()); + for (index, data_type) in fields { + let array = batch.column(*index).slice(start, end - start); + if is_native_histogram_value_type(data_type) { + let hist = array + .as_any() + .downcast_ref::() + .ok_or_else(|| invalid("invalid native histogram array"))?; + samples.push(histogram_samples_from_arrow(hist).map_err(|error| { + let source = common_query::error::Error::GeneralDataFusion { + error, + location: snafu::Location::new(file!(), line!(), column!()), + }; + BoxedError::new( + crate::error::ExternalSnafu { + context: "native histogram conversion".to_string(), + } + .into_error(BoxedError::new(source)), + ) + })?); + } else { + let values = cast(&array, &datatypes::arrow::datatypes::DataType::Float64) + .context(ComputeArrowSnafu) + .map_err(boxed)?; + let values = values + .as_any() + .downcast_ref::() + .ok_or_else(|| invalid("invalid numeric field array"))?; + samples.push(Samples::from_float(values.clone())); + } + } + let fields = SeriesFields::try_from_vec(samples) + .context(DataTypeMismatchSnafu) + .map_err(BoxedError::new)?; + RawSeriesChunk::try_new( + schema, + Buffer::from(key.to_vec()), + None, + time, + fields, + false, + ) + .context(DataTypeMismatchSnafu) + .map_err(BoxedError::new) +} + +struct RawStream { + schema: Arc, + label_source: RawLabelSource, + inner: futures::stream::BoxStream<'static, Result>, +} + +impl Stream for RawStream { + type Item = Result; + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.inner.as_mut().poll_next(cx) + } +} + +impl RawSeriesStream for RawStream { + fn schema(&self) -> &Arc { + &self.schema + } + fn label_source(&self) -> &RawLabelSource { + &self.label_source + } + fn ordering(&self) -> RawSeriesOrdering { + RawSeriesOrdering::Contiguous + } +} + +pub(crate) fn make_stream( + inner: ScanBatchStream, + schema: Arc, + label_source: RawLabelSource, + fields: Vec<(usize, ConcreteDataType)>, + metrics: PartitionMetrics, +) -> SendableRawSeriesStream { + let output_schema = schema.clone(); + let stream = async_stream::stream! { + let mut input = inner; + let mut pending: Option = None; + while let Some(item) = input.next().await { + let records = match item { + Ok(ScanBatch::Series(SeriesBatch::Flat(flat))) => flat.batches.into_vec(), + Ok(ScanBatch::RecordBatch(batch)) => vec![batch], + Err(error) => { yield Err(BoxedError::new(error)); return; } + }; + for record in records { + let mut start = 0; + while start < record.num_rows() { + let key = match read_key(&record, start) { Ok(key) => key, Err(error) => { yield Err(error); return; } }; + let mut end = start + 1; + while end < record.num_rows() { + match read_key(&record, end) { + Ok(next) if next == key => end += 1, + Ok(_) => break, + Err(error) => { yield Err(error); return; } + } + } + let begun = Instant::now(); + let current = match chunk(&record, start, end, &output_schema, &fields) { + Ok(chunk) => chunk, Err(error) => { yield Err(error); return; } + }; + metrics.inc_convert_batch_cost(begun.elapsed()); + if let Some(old) = pending.as_ref() { + if old.read_key.as_slice() == current.read_key.as_slice() + && current.time.timestamp(0).zip(old.time.timestamp(old.time.len() - 1)) + .is_some_and(|(next, previous)| next <= previous) { + yield Err(invalid("raw timestamps are not strictly increasing")); return; + } + } + if let Some(mut old) = pending.take() { + if old.read_key.as_slice() != current.read_key.as_slice() { old.is_last = true; } + yield Ok(old); + } + pending = Some(current); + start = end; + } + } + } + if let Some(mut last) = pending { last.is_last = true; yield Ok(last); } + }; + Box::pin(RawStream { + schema, + label_source, + inner: Box::pin(stream), + }) +} diff --git a/src/mito2/src/read/series_scan/raw/tests.rs b/src/mito2/src/read/series_scan/raw/tests.rs new file mode 100644 index 00000000000..57619b5fb22 --- /dev/null +++ b/src/mito2/src/read/series_scan/raw/tests.rs @@ -0,0 +1,481 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::pin::Pin; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::task::{Context, Poll}; +use std::time::Instant; + +use common_error::ext::ErrorExt; +use common_query::native_histogram::{ + CounterResetHint, NativeHistogram, Span, build_histogram_array, +}; +use datafusion::physical_plan::metrics::ExecutionPlanMetricsSet; +use datatypes::arrow::array::{ + ArrayRef, BinaryArray, Int64Array, StructArray, TimestampMillisecondArray, UInt8Array, + UInt64Array, +}; +use datatypes::arrow::datatypes::{DataType, Field, Schema}; +use datatypes::arrow::record_batch::RecordBatch; +use datatypes::data_type::DataType as _; +use datatypes::series::raw::RawSeriesChunk; +use datatypes::series::samples::SampleRef; +use datatypes::series::samples::{FieldSpec, SeriesSchema}; +use futures::{StreamExt, stream}; +use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder}; +use store_api::region_engine::{BoundTag, RawLabelSource, SendableRawSeriesStream}; +use store_api::storage::RegionId; + +use super::make_stream; +use crate::read::scan_util::PartitionMetrics; +use crate::read::series_scan::{FlatSeriesBatch, SeriesBatch}; +use crate::read::stream::{ScanBatch, ScanBatchStream}; + +fn metrics() -> PartitionMetrics { + PartitionMetrics::new( + store_api::storage::RegionId::new(1, 0), + 0, + "raw-test", + Instant::now(), + false, + &ExecutionPlanMetricsSet::new(), + ) +} + +fn schema() -> Arc { + Arc::new(SeriesSchema { + fields: vec![FieldSpec { + name: Arc::from("value"), + }] + .into(), + eval_grid: None, + }) +} + +fn batch(keys: &[&[u8]], times: &[i64], values: &[Option]) -> RecordBatch { + assert_eq!(keys.len(), times.len()); + assert_eq!(keys.len(), values.len()); + let columns: Vec = vec![ + Arc::new(Int64Array::from(values.to_vec())), + Arc::new(TimestampMillisecondArray::from(times.to_vec())), + Arc::new(BinaryArray::from_iter_values(keys.iter().copied())), + Arc::new(UInt64Array::from(vec![1; keys.len()])), + Arc::new(UInt8Array::from(vec![0; keys.len()])), + ]; + let schema = Arc::new(Schema::new(vec![ + Field::new("value", DataType::Int64, true), + Field::new( + "ts", + DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None), + false, + ), + Field::new("__primary_key", DataType::Binary, false), + Field::new("__sequence", DataType::UInt64, false), + Field::new("__op_type", DataType::UInt8, false), + ])); + RecordBatch::try_new(schema, columns).unwrap() +} + +fn histogram_input() -> (RecordBatch, Vec) { + let expected = vec![NativeHistogram { + schema: 3, + zero_threshold: 0.25, + sum: 42.5, + reset_hint: CounterResetHint::Unrecognized(77), + start_timestamp: Some(1234), + custom_values: vec![1.5, 2.5], + positive_spans: vec![Span { + offset: 2, + length: 2, + }], + negative_spans: vec![Span { + offset: -1, + length: 1, + }], + count: 8.0, + zero_count: 2.0, + positive_buckets: vec![3.0, 4.0], + negative_buckets: vec![1.0], + }]; + let array = build_histogram_array(&[Some(expected[0].clone())]); + let array = array + .as_any() + .downcast_ref::() + .unwrap() + .clone(); + let columns: Vec = vec![ + Arc::new(array), + Arc::new(TimestampMillisecondArray::from(vec![Some(1)])), + Arc::new(BinaryArray::from_iter_values([b"hist-key".as_slice()])), + Arc::new(UInt64Array::from(vec![1])), + Arc::new(UInt8Array::from(vec![0])), + ]; + let fields = vec![ + Field::new( + "histogram", + common_query::native_histogram::native_histogram_value_type().as_arrow_type(), + true, + ), + Field::new( + "ts", + DataType::Timestamp(datatypes::arrow::datatypes::TimeUnit::Millisecond, None), + false, + ), + Field::new("__primary_key", DataType::Binary, false), + Field::new("__sequence", DataType::UInt64, false), + Field::new("__op_type", DataType::UInt8, false), + ]; + ( + RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap(), + expected, + ) +} + +fn input(items: Vec>) -> ScanBatchStream { + Box::pin(stream::iter(items)) +} + +fn metadata() -> Arc { + metadata_with_field_type(datatypes::prelude::ConcreteDataType::int64_datatype()) +} + +fn metadata_with_field_type( + field_type: datatypes::prelude::ConcreteDataType, +) -> Arc { + let mut builder = RegionMetadataBuilder::new(RegionId::new(1, 0)); + builder.push_column_metadata(ColumnMetadata { + column_schema: datatypes::schema::ColumnSchema::new( + "tag", + datatypes::prelude::ConcreteDataType::string_datatype(), + true, + ), + semantic_type: api::v1::SemanticType::Tag, + column_id: 0, + }); + builder.push_column_metadata(ColumnMetadata { + column_schema: datatypes::schema::ColumnSchema::new( + "ts", + datatypes::prelude::ConcreteDataType::timestamp_millisecond_datatype(), + false, + ), + semantic_type: api::v1::SemanticType::Timestamp, + column_id: 1, + }); + builder.push_column_metadata(ColumnMetadata { + column_schema: datatypes::schema::ColumnSchema::new("value", field_type, true), + semantic_type: api::v1::SemanticType::Field, + column_id: 2, + }); + builder.primary_key(vec![0]); + Arc::new(builder.build().unwrap()) +} + +fn convert_with_source( + inner: ScanBatchStream, + schema: Arc, + source: RawLabelSource, + field_type: datatypes::prelude::ConcreteDataType, +) -> SendableRawSeriesStream { + make_stream(inner, schema, source, vec![(0, field_type)], metrics()) +} + +fn primary_key_source(metadata: Arc) -> RawLabelSource { + RawLabelSource::PrimaryKey { + metric_name: Arc::from("metric"), + physical_metadata: metadata, + logical_tags: Arc::from([BoundTag { + column_id: 0, + name: Arc::from("tag"), + }]), + } +} + +fn convert(inner: ScanBatchStream, schema: Arc) -> SendableRawSeriesStream { + convert_with_source( + inner, + schema, + primary_key_source(metadata()), + datatypes::prelude::ConcreteDataType::int64_datatype(), + ) +} + +struct DropSpy { + inner: ScanBatchStream, + drops: Arc, +} + +impl futures::Stream for DropSpy { + type Item = crate::error::Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.inner.as_mut().poll_next(cx) + } +} + +impl Drop for DropSpy { + fn drop(&mut self) { + self.drops.fetch_add(1, Ordering::SeqCst); + } +} + +fn values(chunk: &RawSeriesChunk) -> Vec> { + (0..chunk.time.len()) + .map(|i| match chunk.fields.get(0).unwrap().sample(i) { + datatypes::series::samples::SampleRef::Missing => None, + datatypes::series::samples::SampleRef::Float(v) => Some(v), + _ => unreachable!(), + }) + .collect() +} + +#[tokio::test] +async fn chunks_preserve_keys_values_times_and_final_flags_for_both_input_shapes() { + let schema = schema(); + let first = batch( + &[b"prefix-full-key-a", b"prefix-full-key-a"], + &[1, 2], + &[Some(1), None], + ); + let second = batch( + &[ + b"prefix-full-key-a", + b"prefix-full-key-a", + b"prefix-full-key-b", + b"prefix-full-key-b", + ], + &[3, 4, 1, 2], + &[Some(3), Some(4), Some(5), Some(6)], + ); + for flat in [true, false] { + let items = if flat { + let mut batches = smallvec::SmallVec::new(); + batches.extend([first.clone(), second.clone()]); + vec![Ok(ScanBatch::Series(SeriesBatch::Flat(FlatSeriesBatch { + batches, + })))] + } else { + vec![ + Ok(ScanBatch::RecordBatch(first.clone())), + Ok(ScanBatch::RecordBatch(second.clone())), + ] + }; + let mut output = convert(input(items), schema.clone()); + let mut chunks = Vec::new(); + while let Some(chunk) = output.next().await { + chunks.push(chunk.unwrap()); + } + assert_eq!( + chunks + .iter() + .map(|c| c.read_key.as_slice()) + .collect::>(), + vec![ + b"prefix-full-key-a".as_slice(), + b"prefix-full-key-a", + b"prefix-full-key-b" + ] + ); + assert_eq!( + chunks.iter().map(|c| c.is_last).collect::>(), + vec![false, true, true] + ); + assert_eq!( + chunks + .iter() + .map(|c| (0..c.time.len()) + .map(|i| c.time.timestamp(i).unwrap()) + .collect::>()) + .collect::>(), + vec![vec![1, 2], vec![3, 4], vec![1, 2]] + ); + assert_eq!( + chunks.iter().map(values).collect::>(), + vec![ + vec![Some(1.0), None], + vec![Some(3.0), Some(4.0)], + vec![Some(5.0), Some(6.0)] + ] + ); + } +} + +#[tokio::test] +async fn upstream_error_is_terminal_after_yielded_prefix_without_flushing_pending() { + let upstream_error = crate::error::InvalidRecordBatchSnafu { + reason: "upstream injected failure".to_string(), + } + .build(); + let mut output = convert( + input(vec![ + Ok(ScanBatch::RecordBatch(batch(&[b"a"], &[1], &[Some(1)]))), + Ok(ScanBatch::RecordBatch(batch(&[b"a"], &[2], &[Some(2)]))), + Ok(ScanBatch::RecordBatch(batch(&[b"a"], &[3], &[Some(3)]))), + Err(upstream_error), + ]), + schema(), + ); + let first = output.next().await.unwrap().unwrap(); + assert_eq!(first.read_key.as_slice(), b"a"); + assert_eq!(first.time.timestamp(0), Some(1)); + assert_eq!(values(&first), vec![Some(1.0)]); + assert!(first.metric.is_none()); + assert!(!first.is_last); + let second = output.next().await.unwrap().unwrap(); + assert_eq!(second.read_key.as_slice(), b"a"); + assert_eq!(second.time.timestamp(0), Some(2)); + assert_eq!(values(&second), vec![Some(2.0)]); + assert!(second.metric.is_none()); + assert!(!second.is_last); + let error = output.next().await.unwrap().unwrap_err(); + let Some(crate::error::Error::InvalidRecordBatch { reason, .. }) = + error.as_any().downcast_ref::() + else { + panic!("unexpected upstream error: {error}"); + }; + assert_eq!(reason, "upstream injected failure"); + assert!(output.next().await.is_none()); +} + +#[tokio::test] +async fn dropping_raw_stream_drops_unfinished_input() { + let drops = Arc::new(AtomicUsize::new(0)); + let upstream = input(vec![ + Ok(ScanBatch::RecordBatch(batch(&[b"a"], &[1], &[Some(1)]))), + Ok(ScanBatch::RecordBatch(batch(&[b"b"], &[1], &[Some(2)]))), + Ok(ScanBatch::RecordBatch(batch(&[b"c"], &[1], &[Some(3)]))), + ]); + let spy = DropSpy { + inner: upstream, + drops: drops.clone(), + }; + let mut raw = convert(Box::pin(spy), schema()); + assert_eq!(raw.next().await.unwrap().unwrap().read_key.as_slice(), b"a"); + drop(raw); + assert_eq!(drops.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn raw_stream_owns_schema_and_bound_source_metadata() { + let schema = schema(); + let schema_weak = Arc::downgrade(&schema); + let metadata = metadata(); + let metadata_weak = Arc::downgrade(&metadata); + let mut raw = convert_with_source( + input(vec![]), + schema.clone(), + primary_key_source(metadata.clone()), + datatypes::prelude::ConcreteDataType::int64_datatype(), + ); + drop(schema); + drop(metadata); + assert!(schema_weak.upgrade().is_some()); + assert!(metadata_weak.upgrade().is_some()); + assert_eq!(raw.schema().fields[0].name.as_ref(), "value"); + match raw.label_source() { + RawLabelSource::PrimaryKey { + metric_name, + physical_metadata, + logical_tags, + } => { + assert_eq!(metric_name.as_ref(), "metric"); + assert_eq!(physical_metadata.region_id, RegionId::new(1, 0)); + assert_eq!( + physical_metadata.column_metadatas[0].column_schema.name, + "tag" + ); + assert_eq!(logical_tags[0].column_id, 0); + assert_eq!(logical_tags[0].name.as_ref(), "tag"); + assert!(Arc::ptr_eq( + physical_metadata, + &metadata_weak.upgrade().unwrap() + )); + } + RawLabelSource::Explicit => panic!("expected bound primary-key labels"), + } + assert!(raw.next().await.is_none()); + drop(raw); + assert!(schema_weak.upgrade().is_none()); + assert!(metadata_weak.upgrade().is_none()); +} + +#[tokio::test] +async fn empty_input_is_a_successful_empty_stream() { + let mut output = convert(input(vec![]), schema()); + assert!(output.next().await.is_none()); +} + +#[tokio::test] +async fn native_histogram_adapter_preserves_complete_payload() { + let (record, expected) = histogram_input(); + let mut fields = schema().fields.to_vec(); + fields[0].name = Arc::from("histogram"); + let series_schema = Arc::new(SeriesSchema { + fields: fields.into(), + eval_grid: None, + }); + let mut raw = convert_with_source( + input(vec![Ok(ScanBatch::RecordBatch(record))]), + series_schema, + primary_key_source(metadata_with_field_type( + common_query::native_histogram::native_histogram_value_type().clone(), + )), + common_query::native_histogram::native_histogram_value_type().clone(), + ); + assert!(matches!( + raw.label_source(), + RawLabelSource::PrimaryKey { .. } + )); + let chunk = raw.next().await.unwrap().unwrap(); + assert!(chunk.metric.is_none()); + assert!(chunk.is_last); + let SampleRef::Histogram(actual) = chunk.fields.get(0).unwrap().sample(0) else { + panic!("expected native histogram sample"); + }; + let expected = &expected[0]; + assert_eq!(actual.schema, expected.schema); + assert_eq!(actual.zero_threshold, expected.zero_threshold); + assert_eq!(actual.sum, expected.sum); + assert_eq!(actual.count, expected.count); + assert_eq!(actual.zero_count, expected.zero_count); + assert_eq!(actual.reset_hint, expected.reset_hint); + assert_eq!(actual.start_timestamp, expected.start_timestamp); + assert_eq!(actual.custom_values, expected.custom_values); + assert_eq!(actual.positive.span_offsets, vec![2]); + assert_eq!(actual.positive.span_lengths, vec![2]); + assert_eq!(actual.positive.buckets, expected.positive_buckets); + assert_eq!(actual.negative.span_offsets, vec![-1]); + assert_eq!(actual.negative.span_lengths, vec![1]); + assert_eq!(actual.negative.buckets, expected.negative_buckets); + assert!(raw.next().await.is_none()); +} + +#[tokio::test] +async fn duplicate_or_descending_cross_batch_time_is_terminal() { + for next in [2, 1] { + let mut output = convert( + input(vec![ + Ok(ScanBatch::RecordBatch(batch( + &[b"k", b"k"], + &[1, 2], + &[Some(1), Some(2)], + ))), + Ok(ScanBatch::RecordBatch(batch(&[b"k"], &[next], &[Some(3)]))), + ]), + schema(), + ); + assert!(output.next().await.unwrap().is_err()); + assert!(output.next().await.is_none()); + } +} diff --git a/src/store-api/src/region_engine.rs b/src/store-api/src/region_engine.rs index 82183f77ec3..c73c757b826 100644 --- a/src/store-api/src/region_engine.rs +++ b/src/store-api/src/region_engine.rs @@ -551,12 +551,19 @@ pub trait RegionScanner: Debug + DisplayAs + Send { /// `None` means raw capability is absent before execution and permits a legacy scan instead. /// `Some` selects this entrypoint, including for an empty successful stream. Errors propagate; /// callers must not replay a failed raw scan through the legacy path. Use only one entrypoint - /// for a given scanner partition per execution. + /// for a given scanner partition per execution. The explicit label source is owned by the + /// returned stream and must be bound to this scanner's metadata snapshot. + /// + /// Output preserves the ordinary storage reader's pushdown and MVCC semantics. Inexact + /// pushed-down predicates can leave false-positive candidates, just as in `scan_partition`. + /// Callers must retain residual predicate columns until those predicates are evaluated and + /// apply the final projection afterward; raw capability does not mean filtering is complete. fn scan_series_partition( &self, _ctx: &QueryScanContext, _metrics_set: &ExecutionPlanMetricsSet, _partition: usize, + _label_source: &RawLabelSource, ) -> Result, BoxedError> { Ok(None) } @@ -849,7 +856,7 @@ mod tests { assert!( scanner - .scan_series_partition(&ctx, &metrics, 0) + .scan_series_partition(&ctx, &metrics, 0, &RawLabelSource::Explicit) .unwrap() .is_none() );