mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-18 08:44:07 +00:00
fix(mito2): adapt raw readers to shared series contracts
Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
This commit is contained in:
@@ -817,8 +817,8 @@ mod test {
|
||||
let chunks = stream.try_collect::<Vec<_>>().await.unwrap();
|
||||
for chunk in chunks {
|
||||
assert!(chunk.metric.is_none());
|
||||
assert!(!chunk.read_key.is_empty());
|
||||
let key_bytes = chunk.read_key.as_slice();
|
||||
assert!(!chunk.primary_key.is_empty());
|
||||
let key_bytes = chunk.primary_key.as_slice();
|
||||
let key_codec = SparsePrimaryKeyCodec::new(&physical_metadata);
|
||||
let (table_id, _) = key_codec.decode_ids(key_bytes).unwrap();
|
||||
assert_eq!(table_id, logical_a.table_id());
|
||||
@@ -834,7 +834,7 @@ mod test {
|
||||
decoded_jobs.insert(job_value.to_string());
|
||||
keys.insert(key_bytes.to_vec());
|
||||
last_markers += usize::from(chunk.is_last);
|
||||
let times = chunk.time.as_explicit().unwrap();
|
||||
let times = chunk.time.as_buffer().as_ref();
|
||||
samples.extend(times.iter().enumerate().map(|(index, time)| {
|
||||
let value = match chunk.fields.get(0).unwrap().sample(index) {
|
||||
SampleRef::Float(value) => value,
|
||||
|
||||
@@ -4768,7 +4768,7 @@ async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() {
|
||||
use api::v1::{Row, Value};
|
||||
use common_recordbatch::filter;
|
||||
use datatypes::arrow::array::Array;
|
||||
use datatypes::series::raw::RawSeriesOrdering;
|
||||
use datatypes::series::SeriesOrdering;
|
||||
use datatypes::series::samples::SampleRef;
|
||||
use datatypes::value::Value as DatumValue;
|
||||
use mito_codec::row_converter::build_primary_key_codec;
|
||||
@@ -4907,7 +4907,7 @@ async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() {
|
||||
.scan_series_partition(&QueryScanContext::default(), &metrics, partition, &source)
|
||||
.unwrap()
|
||||
.expect("supported millisecond PerSeries scan");
|
||||
assert_eq!(RawSeriesOrdering::Contiguous, stream.ordering());
|
||||
assert_eq!(SeriesOrdering::Contiguous, stream.ordering());
|
||||
assert!(stream.schema().eval_grid.is_none());
|
||||
assert_eq!(
|
||||
vec!["field_1", "field_0"],
|
||||
@@ -4922,10 +4922,10 @@ async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() {
|
||||
while let Some(chunk) = stream.try_next().await.unwrap() {
|
||||
assert!(chunk.metric.is_none());
|
||||
assert!(
|
||||
!closed_keys.contains(chunk.read_key.as_slice()),
|
||||
!closed_keys.contains(chunk.primary_key.as_slice()),
|
||||
"no chunk may follow a key's is_last chunk"
|
||||
);
|
||||
let times = chunk.time.as_explicit().unwrap();
|
||||
let times = chunk.time.as_buffer().as_ref();
|
||||
assert_eq!(2, chunk.fields.len());
|
||||
assert_eq!(times.len(), chunk.fields.get(0).unwrap().len());
|
||||
assert_eq!(times.len(), chunk.fields.get(1).unwrap().len());
|
||||
@@ -4938,7 +4938,7 @@ async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() {
|
||||
SampleRef::Missing => None,
|
||||
_ => panic!("f0 type"),
|
||||
};
|
||||
let key = chunk.read_key.to_vec();
|
||||
let key = chunk.primary_key.to_vec();
|
||||
if let Some(previous) = previous_times.insert(key.clone(), *time) {
|
||||
assert!(
|
||||
previous < *time,
|
||||
@@ -4946,10 +4946,10 @@ async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() {
|
||||
);
|
||||
}
|
||||
if chunk.is_last {
|
||||
closed_keys.insert(chunk.read_key.to_vec());
|
||||
closed_keys.insert(chunk.primary_key.to_vec());
|
||||
}
|
||||
raw.push((
|
||||
chunk.read_key.clone(),
|
||||
chunk.primary_key.clone(),
|
||||
*time,
|
||||
f1,
|
||||
f0,
|
||||
@@ -5199,7 +5199,7 @@ async fn test_mito_raw_series_int64_predicate_uses_original_precision() {
|
||||
);
|
||||
let chunks = stream.try_collect::<Vec<_>>().await.unwrap();
|
||||
for chunk in chunks {
|
||||
let axis = chunk.time.as_explicit().unwrap();
|
||||
let axis = chunk.time.as_buffer().as_ref();
|
||||
times.extend(axis.iter().copied());
|
||||
for index in 0..axis.len() {
|
||||
output_values.push((
|
||||
@@ -5313,7 +5313,7 @@ async fn test_mito_raw_series_invalid_binding_before_legacy_read() {
|
||||
let tuples = chunks
|
||||
.iter()
|
||||
.map(|chunk| {
|
||||
let times = chunk.time.as_explicit().unwrap();
|
||||
let times = chunk.time.as_buffer().as_ref();
|
||||
let SampleRef::Float(value) = chunk.fields.get(0).unwrap().sample(0) else {
|
||||
panic!("expected raw float sample");
|
||||
};
|
||||
|
||||
@@ -36,7 +36,8 @@ use datatypes::arrow::datatypes::ArrowNativeType;
|
||||
use datatypes::arrow::datatypes::DataType;
|
||||
use datatypes::arrow::record_batch::RecordBatch;
|
||||
use datatypes::prelude::ConcreteDataType;
|
||||
use datatypes::series::raw::{RawSeriesChunk, RawSeriesOrdering, RawTimeAxis};
|
||||
use datatypes::series::SeriesOrdering;
|
||||
use datatypes::series::raw::{RawSeriesChunk, RawTimestamps};
|
||||
use datatypes::series::samples::{Samples, SeriesFields, SeriesSchema};
|
||||
use futures::stream::BoxStream;
|
||||
use futures::{Stream, StreamExt};
|
||||
@@ -82,7 +83,7 @@ fn chunk(
|
||||
if (start..end).any(|row| !timestamp.is_valid(row)) {
|
||||
return Err(invalid("null timestamp"));
|
||||
}
|
||||
let time = RawTimeAxis::try_new(timestamp.values().slice(start, end - start))
|
||||
let time = RawTimestamps::try_new(timestamp.values().slice(start, end - start))
|
||||
.context(DataTypeMismatchSnafu)
|
||||
.map_err(BoxedError::new)?;
|
||||
let mut samples = Vec::with_capacity(fields.len());
|
||||
@@ -144,8 +145,8 @@ impl RawSeriesStream for RawStream {
|
||||
fn label_source(&self) -> &RawLabelSource {
|
||||
&self.label_source
|
||||
}
|
||||
fn ordering(&self) -> RawSeriesOrdering {
|
||||
RawSeriesOrdering::Contiguous
|
||||
fn ordering(&self) -> SeriesOrdering {
|
||||
SeriesOrdering::Contiguous
|
||||
}
|
||||
}
|
||||
|
||||
@@ -238,20 +239,20 @@ pub(crate) fn make_stream(
|
||||
metrics.inc_convert_batch_cost(begun.elapsed());
|
||||
if let Some(old) = pending.as_ref()
|
||||
&& is_legacy
|
||||
&& old.read_key.as_slice() != current.read_key.as_slice()
|
||||
&& current.read_key.as_slice() <= old.read_key.as_slice()
|
||||
&& old.primary_key.as_slice() != current.primary_key.as_slice()
|
||||
&& current.primary_key.as_slice() <= old.primary_key.as_slice()
|
||||
{
|
||||
yield Err(invalid("legacy series primary keys are not strictly increasing")); return;
|
||||
}
|
||||
if let Some(old) = pending.as_ref()
|
||||
&& old.read_key.as_slice() == current.read_key.as_slice()
|
||||
&& old.primary_key.as_slice() == current.primary_key.as_slice()
|
||||
&& current.time.timestamp(0).zip(old.time.timestamp(old.time.len() - 1))
|
||||
.is_some_and(|(next, previous)| next <= previous)
|
||||
{
|
||||
yield Err(invalid("raw timestamps are not strictly increasing")); return;
|
||||
}
|
||||
if let Some(mut old) = pending.take() {
|
||||
if old.read_key.as_slice() != current.read_key.as_slice() { old.is_last = true; }
|
||||
if old.primary_key.as_slice() != current.primary_key.as_slice() { old.is_last = true; }
|
||||
yield Ok(old);
|
||||
}
|
||||
pending = Some(current);
|
||||
|
||||
@@ -338,7 +338,7 @@ async fn chunks_preserve_keys_values_times_and_final_flags_for_both_input_shapes
|
||||
assert_eq!(
|
||||
chunks
|
||||
.iter()
|
||||
.map(|c| c.read_key.as_slice())
|
||||
.map(|c| c.primary_key.as_slice())
|
||||
.collect::<Vec<_>>(),
|
||||
vec![
|
||||
b"prefix-full-key-a".as_slice(),
|
||||
@@ -398,12 +398,12 @@ async fn typed_predicate_filters_before_numeric_samples_are_cast() {
|
||||
);
|
||||
let chunk = output.next().await.unwrap().unwrap();
|
||||
assert!(chunk.is_last);
|
||||
assert_eq!(chunk.read_key.as_slice(), b"a");
|
||||
assert_eq!(chunk.primary_key.as_slice(), b"a");
|
||||
assert_eq!(chunk.time.len(), 1);
|
||||
assert_eq!(chunk.time.timestamp(0), Some(3));
|
||||
assert_eq!(values(&chunk), vec![Some((threshold + 1) as f64)]);
|
||||
let chunk = output.next().await.unwrap().unwrap();
|
||||
assert_eq!(chunk.read_key.as_slice(), b"c");
|
||||
assert_eq!(chunk.primary_key.as_slice(), b"c");
|
||||
assert!(chunk.is_last);
|
||||
assert_eq!(chunk.time.timestamp(0), Some(1));
|
||||
assert_eq!(values(&chunk), vec![Some((threshold + 1) as f64)]);
|
||||
@@ -457,13 +457,13 @@ async fn upstream_error_is_terminal_after_yielded_prefix_without_flushing_pendin
|
||||
schema(),
|
||||
);
|
||||
let first = output.next().await.unwrap().unwrap();
|
||||
assert_eq!(first.read_key.as_slice(), b"a");
|
||||
assert_eq!(first.primary_key.as_slice(), b"a");
|
||||
assert_eq!(first.time.timestamp(0), Some(1));
|
||||
assert_eq!(values(&first), vec![Some(1.0)]);
|
||||
assert!(first.metric.is_none());
|
||||
assert!(!first.is_last);
|
||||
let second = output.next().await.unwrap().unwrap();
|
||||
assert_eq!(second.read_key.as_slice(), b"a");
|
||||
assert_eq!(second.primary_key.as_slice(), b"a");
|
||||
assert_eq!(second.time.timestamp(0), Some(2));
|
||||
assert_eq!(values(&second), vec![Some(2.0)]);
|
||||
assert!(second.metric.is_none());
|
||||
@@ -502,7 +502,10 @@ async fn dropping_raw_stream_drops_unfinished_input() {
|
||||
drops: drops.clone(),
|
||||
};
|
||||
let mut raw = convert(Box::pin(spy), schema());
|
||||
assert_eq!(raw.next().await.unwrap().unwrap().read_key.as_slice(), b"a");
|
||||
assert_eq!(
|
||||
raw.next().await.unwrap().unwrap().primary_key.as_slice(),
|
||||
b"a"
|
||||
);
|
||||
drop(raw);
|
||||
assert_eq!(drops.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
@@ -532,7 +535,7 @@ async fn legacy_series_requires_increasing_keys_but_record_batches_do_not() {
|
||||
schema(),
|
||||
);
|
||||
let prefix = output.next().await.unwrap().unwrap();
|
||||
assert_eq!(prefix.read_key.as_slice(), b"a");
|
||||
assert_eq!(prefix.primary_key.as_slice(), b"a");
|
||||
assert_eq!(values(&prefix), vec![Some(1.0)]);
|
||||
assert_eq!(prefix.time.timestamp(0), Some(1));
|
||||
assert!(prefix.is_last);
|
||||
@@ -549,12 +552,12 @@ async fn legacy_series_requires_increasing_keys_but_record_batches_do_not() {
|
||||
schema(),
|
||||
);
|
||||
let first = output.next().await.unwrap().unwrap();
|
||||
assert_eq!(first.read_key.as_slice(), b"a");
|
||||
assert_eq!(first.primary_key.as_slice(), b"a");
|
||||
assert_eq!(values(&first), vec![Some(1.0)]);
|
||||
assert_eq!(first.time.timestamp(0), Some(1));
|
||||
assert!(first.is_last);
|
||||
let second = output.next().await.unwrap().unwrap();
|
||||
assert_eq!(second.read_key.as_slice(), b"b");
|
||||
assert_eq!(second.primary_key.as_slice(), b"b");
|
||||
assert_eq!(values(&second), vec![Some(2.0)]);
|
||||
assert_eq!(second.time.timestamp(0), Some(1));
|
||||
assert!(second.is_last);
|
||||
@@ -576,12 +579,12 @@ async fn legacy_series_requires_increasing_keys_but_record_batches_do_not() {
|
||||
schema(),
|
||||
);
|
||||
let first = output.next().await.unwrap().unwrap();
|
||||
assert_eq!(first.read_key.as_slice(), b"b");
|
||||
assert_eq!(first.primary_key.as_slice(), b"b");
|
||||
assert_eq!(values(&first), vec![Some(1.0)]);
|
||||
assert_eq!(first.time.timestamp(0), Some(1));
|
||||
assert!(first.is_last);
|
||||
let second = output.next().await.unwrap().unwrap();
|
||||
assert_eq!(second.read_key.as_slice(), b"a");
|
||||
assert_eq!(second.primary_key.as_slice(), b"a");
|
||||
assert_eq!(values(&second), vec![Some(2.0)]);
|
||||
assert_eq!(second.time.timestamp(0), Some(1));
|
||||
assert!(second.is_last);
|
||||
@@ -605,10 +608,10 @@ async fn raw_key_shares_dictionary_values_and_survives_batch_drop() {
|
||||
let expected_ptr = values.value(1).as_ptr();
|
||||
let mut output = convert(input(vec![Ok(ScanBatch::RecordBatch(record))]), schema());
|
||||
let chunk = output.next().await.unwrap().unwrap();
|
||||
assert_eq!(chunk.read_key.as_slice(), b"prefix-key");
|
||||
assert_eq!(chunk.read_key.as_ptr(), expected_ptr);
|
||||
assert_eq!(chunk.primary_key.as_slice(), b"prefix-key");
|
||||
assert_eq!(chunk.primary_key.as_ptr(), expected_ptr);
|
||||
drop(output);
|
||||
assert_eq!(chunk.read_key.as_slice(), b"prefix-key");
|
||||
assert_eq!(chunk.primary_key.as_slice(), b"prefix-key");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
Reference in New Issue
Block a user