fix(mito2): adapt raw readers to shared series contracts

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
This commit is contained in:
discord9
2026-10-16 15:53:57 +08:00
parent b0af8b3eaf
commit 3ed6b5df67
4 changed files with 38 additions and 34 deletions
+3 -3
View File
@@ -817,8 +817,8 @@ mod test {
let chunks = stream.try_collect::<Vec<_>>().await.unwrap();
for chunk in chunks {
assert!(chunk.metric.is_none());
assert!(!chunk.read_key.is_empty());
let key_bytes = chunk.read_key.as_slice();
assert!(!chunk.primary_key.is_empty());
let key_bytes = chunk.primary_key.as_slice();
let key_codec = SparsePrimaryKeyCodec::new(&physical_metadata);
let (table_id, _) = key_codec.decode_ids(key_bytes).unwrap();
assert_eq!(table_id, logical_a.table_id());
@@ -834,7 +834,7 @@ mod test {
decoded_jobs.insert(job_value.to_string());
keys.insert(key_bytes.to_vec());
last_markers += usize::from(chunk.is_last);
let times = chunk.time.as_explicit().unwrap();
let times = chunk.time.as_buffer().as_ref();
samples.extend(times.iter().enumerate().map(|(index, time)| {
let value = match chunk.fields.get(0).unwrap().sample(index) {
SampleRef::Float(value) => value,
+9 -9
View File
@@ -4768,7 +4768,7 @@ async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() {
use api::v1::{Row, Value};
use common_recordbatch::filter;
use datatypes::arrow::array::Array;
use datatypes::series::raw::RawSeriesOrdering;
use datatypes::series::SeriesOrdering;
use datatypes::series::samples::SampleRef;
use datatypes::value::Value as DatumValue;
use mito_codec::row_converter::build_primary_key_codec;
@@ -4907,7 +4907,7 @@ async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() {
.scan_series_partition(&QueryScanContext::default(), &metrics, partition, &source)
.unwrap()
.expect("supported millisecond PerSeries scan");
assert_eq!(RawSeriesOrdering::Contiguous, stream.ordering());
assert_eq!(SeriesOrdering::Contiguous, stream.ordering());
assert!(stream.schema().eval_grid.is_none());
assert_eq!(
vec!["field_1", "field_0"],
@@ -4922,10 +4922,10 @@ async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() {
while let Some(chunk) = stream.try_next().await.unwrap() {
assert!(chunk.metric.is_none());
assert!(
!closed_keys.contains(chunk.read_key.as_slice()),
!closed_keys.contains(chunk.primary_key.as_slice()),
"no chunk may follow a key's is_last chunk"
);
let times = chunk.time.as_explicit().unwrap();
let times = chunk.time.as_buffer().as_ref();
assert_eq!(2, chunk.fields.len());
assert_eq!(times.len(), chunk.fields.get(0).unwrap().len());
assert_eq!(times.len(), chunk.fields.get(1).unwrap().len());
@@ -4938,7 +4938,7 @@ async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() {
SampleRef::Missing => None,
_ => panic!("f0 type"),
};
let key = chunk.read_key.to_vec();
let key = chunk.primary_key.to_vec();
if let Some(previous) = previous_times.insert(key.clone(), *time) {
assert!(
previous < *time,
@@ -4946,10 +4946,10 @@ async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() {
);
}
if chunk.is_last {
closed_keys.insert(chunk.read_key.to_vec());
closed_keys.insert(chunk.primary_key.to_vec());
}
raw.push((
chunk.read_key.clone(),
chunk.primary_key.clone(),
*time,
f1,
f0,
@@ -5199,7 +5199,7 @@ async fn test_mito_raw_series_int64_predicate_uses_original_precision() {
);
let chunks = stream.try_collect::<Vec<_>>().await.unwrap();
for chunk in chunks {
let axis = chunk.time.as_explicit().unwrap();
let axis = chunk.time.as_buffer().as_ref();
times.extend(axis.iter().copied());
for index in 0..axis.len() {
output_values.push((
@@ -5313,7 +5313,7 @@ async fn test_mito_raw_series_invalid_binding_before_legacy_read() {
let tuples = chunks
.iter()
.map(|chunk| {
let times = chunk.time.as_explicit().unwrap();
let times = chunk.time.as_buffer().as_ref();
let SampleRef::Float(value) = chunk.fields.get(0).unwrap().sample(0) else {
panic!("expected raw float sample");
};
+9 -8
View File
@@ -36,7 +36,8 @@ use datatypes::arrow::datatypes::ArrowNativeType;
use datatypes::arrow::datatypes::DataType;
use datatypes::arrow::record_batch::RecordBatch;
use datatypes::prelude::ConcreteDataType;
use datatypes::series::raw::{RawSeriesChunk, RawSeriesOrdering, RawTimeAxis};
use datatypes::series::SeriesOrdering;
use datatypes::series::raw::{RawSeriesChunk, RawTimestamps};
use datatypes::series::samples::{Samples, SeriesFields, SeriesSchema};
use futures::stream::BoxStream;
use futures::{Stream, StreamExt};
@@ -82,7 +83,7 @@ fn chunk(
if (start..end).any(|row| !timestamp.is_valid(row)) {
return Err(invalid("null timestamp"));
}
let time = RawTimeAxis::try_new(timestamp.values().slice(start, end - start))
let time = RawTimestamps::try_new(timestamp.values().slice(start, end - start))
.context(DataTypeMismatchSnafu)
.map_err(BoxedError::new)?;
let mut samples = Vec::with_capacity(fields.len());
@@ -144,8 +145,8 @@ impl RawSeriesStream for RawStream {
fn label_source(&self) -> &RawLabelSource {
&self.label_source
}
fn ordering(&self) -> RawSeriesOrdering {
RawSeriesOrdering::Contiguous
fn ordering(&self) -> SeriesOrdering {
SeriesOrdering::Contiguous
}
}
@@ -238,20 +239,20 @@ pub(crate) fn make_stream(
metrics.inc_convert_batch_cost(begun.elapsed());
if let Some(old) = pending.as_ref()
&& is_legacy
&& old.read_key.as_slice() != current.read_key.as_slice()
&& current.read_key.as_slice() <= old.read_key.as_slice()
&& old.primary_key.as_slice() != current.primary_key.as_slice()
&& current.primary_key.as_slice() <= old.primary_key.as_slice()
{
yield Err(invalid("legacy series primary keys are not strictly increasing")); return;
}
if let Some(old) = pending.as_ref()
&& old.read_key.as_slice() == current.read_key.as_slice()
&& old.primary_key.as_slice() == current.primary_key.as_slice()
&& current.time.timestamp(0).zip(old.time.timestamp(old.time.len() - 1))
.is_some_and(|(next, previous)| next <= previous)
{
yield Err(invalid("raw timestamps are not strictly increasing")); return;
}
if let Some(mut old) = pending.take() {
if old.read_key.as_slice() != current.read_key.as_slice() { old.is_last = true; }
if old.primary_key.as_slice() != current.primary_key.as_slice() { old.is_last = true; }
yield Ok(old);
}
pending = Some(current);
+17 -14
View File
@@ -338,7 +338,7 @@ async fn chunks_preserve_keys_values_times_and_final_flags_for_both_input_shapes
assert_eq!(
chunks
.iter()
.map(|c| c.read_key.as_slice())
.map(|c| c.primary_key.as_slice())
.collect::<Vec<_>>(),
vec![
b"prefix-full-key-a".as_slice(),
@@ -398,12 +398,12 @@ async fn typed_predicate_filters_before_numeric_samples_are_cast() {
);
let chunk = output.next().await.unwrap().unwrap();
assert!(chunk.is_last);
assert_eq!(chunk.read_key.as_slice(), b"a");
assert_eq!(chunk.primary_key.as_slice(), b"a");
assert_eq!(chunk.time.len(), 1);
assert_eq!(chunk.time.timestamp(0), Some(3));
assert_eq!(values(&chunk), vec![Some((threshold + 1) as f64)]);
let chunk = output.next().await.unwrap().unwrap();
assert_eq!(chunk.read_key.as_slice(), b"c");
assert_eq!(chunk.primary_key.as_slice(), b"c");
assert!(chunk.is_last);
assert_eq!(chunk.time.timestamp(0), Some(1));
assert_eq!(values(&chunk), vec![Some((threshold + 1) as f64)]);
@@ -457,13 +457,13 @@ async fn upstream_error_is_terminal_after_yielded_prefix_without_flushing_pendin
schema(),
);
let first = output.next().await.unwrap().unwrap();
assert_eq!(first.read_key.as_slice(), b"a");
assert_eq!(first.primary_key.as_slice(), b"a");
assert_eq!(first.time.timestamp(0), Some(1));
assert_eq!(values(&first), vec![Some(1.0)]);
assert!(first.metric.is_none());
assert!(!first.is_last);
let second = output.next().await.unwrap().unwrap();
assert_eq!(second.read_key.as_slice(), b"a");
assert_eq!(second.primary_key.as_slice(), b"a");
assert_eq!(second.time.timestamp(0), Some(2));
assert_eq!(values(&second), vec![Some(2.0)]);
assert!(second.metric.is_none());
@@ -502,7 +502,10 @@ async fn dropping_raw_stream_drops_unfinished_input() {
drops: drops.clone(),
};
let mut raw = convert(Box::pin(spy), schema());
assert_eq!(raw.next().await.unwrap().unwrap().read_key.as_slice(), b"a");
assert_eq!(
raw.next().await.unwrap().unwrap().primary_key.as_slice(),
b"a"
);
drop(raw);
assert_eq!(drops.load(Ordering::SeqCst), 1);
}
@@ -532,7 +535,7 @@ async fn legacy_series_requires_increasing_keys_but_record_batches_do_not() {
schema(),
);
let prefix = output.next().await.unwrap().unwrap();
assert_eq!(prefix.read_key.as_slice(), b"a");
assert_eq!(prefix.primary_key.as_slice(), b"a");
assert_eq!(values(&prefix), vec![Some(1.0)]);
assert_eq!(prefix.time.timestamp(0), Some(1));
assert!(prefix.is_last);
@@ -549,12 +552,12 @@ async fn legacy_series_requires_increasing_keys_but_record_batches_do_not() {
schema(),
);
let first = output.next().await.unwrap().unwrap();
assert_eq!(first.read_key.as_slice(), b"a");
assert_eq!(first.primary_key.as_slice(), b"a");
assert_eq!(values(&first), vec![Some(1.0)]);
assert_eq!(first.time.timestamp(0), Some(1));
assert!(first.is_last);
let second = output.next().await.unwrap().unwrap();
assert_eq!(second.read_key.as_slice(), b"b");
assert_eq!(second.primary_key.as_slice(), b"b");
assert_eq!(values(&second), vec![Some(2.0)]);
assert_eq!(second.time.timestamp(0), Some(1));
assert!(second.is_last);
@@ -576,12 +579,12 @@ async fn legacy_series_requires_increasing_keys_but_record_batches_do_not() {
schema(),
);
let first = output.next().await.unwrap().unwrap();
assert_eq!(first.read_key.as_slice(), b"b");
assert_eq!(first.primary_key.as_slice(), b"b");
assert_eq!(values(&first), vec![Some(1.0)]);
assert_eq!(first.time.timestamp(0), Some(1));
assert!(first.is_last);
let second = output.next().await.unwrap().unwrap();
assert_eq!(second.read_key.as_slice(), b"a");
assert_eq!(second.primary_key.as_slice(), b"a");
assert_eq!(values(&second), vec![Some(2.0)]);
assert_eq!(second.time.timestamp(0), Some(1));
assert!(second.is_last);
@@ -605,10 +608,10 @@ async fn raw_key_shares_dictionary_values_and_survives_batch_drop() {
let expected_ptr = values.value(1).as_ptr();
let mut output = convert(input(vec![Ok(ScanBatch::RecordBatch(record))]), schema());
let chunk = output.next().await.unwrap().unwrap();
assert_eq!(chunk.read_key.as_slice(), b"prefix-key");
assert_eq!(chunk.read_key.as_ptr(), expected_ptr);
assert_eq!(chunk.primary_key.as_slice(), b"prefix-key");
assert_eq!(chunk.primary_key.as_ptr(), expected_ptr);
drop(output);
assert_eq!(chunk.read_key.as_slice(), b"prefix-key");
assert_eq!(chunk.primary_key.as_slice(), b"prefix-key");
}
#[tokio::test]