From b20e22b426152afff03253e71f452a58b4e2ae85 Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:24:25 +0800 Subject: [PATCH] refactor(series): unify raw and evaluated SeriesFrame codec SeriesFrame is the internal transport of the Narthex native PromQL path, and it could only carry raw samples: an evaluated result had to fall back to a plain row table. This makes one self-describing Arrow format carry both, so a receiver decodes a frame from the frame alone, without the scanner, region, or planning context its producer had. What is unified is the physical representation and its codec, not the runtime data structures. RawSeriesChunk, Series, and EvaluatedChunk stay as they are; only the adapter boundaries branch on the sample shape. Three combinations are legal and every other one is refused in both directions: raw + key + timestamps a storage raw-series scan raw + explicit + timestamps a raw input whose labels are decoded evaluated + explicit + grid_spans a compute node's output An evaluated frame cannot claim a storage key, because the computation may rewrite its labels; raw samples have no grid to be addressed on. The schema builder and the declaration parser both enforce that, so a frame cannot be written or read in a shape no producer could mean. An evaluated frame declares its GridSpec once in schema metadata and carries absolute grid positions per row as spans, so a sparse grid costs its runs and not its holes: nothing is expanded, and no position a chunk does not hold is filled in. A chunk with no position is legal exactly when it closes its chain, and it is kept as one frame row, because dropping it would drop the finality it exists to carry. Layout is v2 and the writer and reader of the reserved greptime.series.* metadata live in one place each, so a producer and a consumer cannot spell one frame two ways. The unreleased v1 format is not read: the in-tree producers, consumers, tests, and the two frozen wire fixtures all move to v2 together. Signed-off-by: discord9 Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com> (cherry picked from commit 176f6f64798658ab0a32ae5c1bc1f63984c31963) --- src/common/query/src/series_frame.rs | 659 +++++-- src/common/query/src/series_frame/tests.rs | 363 +++- .../query/src/series_frame/tests/evaluated.rs | 1250 +++++++++++++ src/datatypes/src/series/frame.rs | 1588 ++++++++++++----- src/datatypes/src/series/frame/tests.rs | 886 ++++++--- src/datatypes/src/series/time.rs | 8 +- src/mito-codec/src/labels/tests.rs | 6 +- src/mito2/src/engine/scan_test.rs | 16 +- .../src/extension_plan/fused_series/tests.rs | 64 +- src/promql/src/source/merge/tests.rs | 16 +- src/promql/src/source/merge/tests/payload.rs | 6 +- src/promql/src/source/tests.rs | 3 +- src/query/src/dist_plan/merge_scan.rs | 74 +- src/table/src/table/scan/series_tests.rs | 26 +- .../wire/raw_series_label_decode/README.md | 26 +- .../{v1_metadata.json => v2_metadata.json} | 77 +- .../wire/raw_series_ordering/README.md | 36 +- .../wire/raw_series_ordering/v1_metadata.json | 45 - .../wire/raw_series_ordering/v2_metadata.json | 98 + 19 files changed, 4230 insertions(+), 1017 deletions(-) create mode 100644 src/common/query/src/series_frame/tests/evaluated.rs rename tests/compatibility/wire/raw_series_label_decode/{v1_metadata.json => v2_metadata.json} (62%) delete mode 100644 tests/compatibility/wire/raw_series_ordering/v1_metadata.json create mode 100644 tests/compatibility/wire/raw_series_ordering/v2_metadata.json diff --git a/src/common/query/src/series_frame.rs b/src/common/query/src/series_frame.rs index d18cdab86f1..8669a8d5d4e 100644 --- a/src/common/query/src/series_frame.rs +++ b/src/common/query/src/series_frame.rs @@ -27,8 +27,12 @@ //! preserves //! scan candidates only; it is not a residual filter, authorization check, remote transport, or //! compaction boundary. Only -//! [`SeriesOrdering::Contiguous`] and [`SeriesOrdering::KeyThenTime`] are accepted; -//! `Interleaved` is rejected before any poll. +//! [`SeriesOrdering::Contiguous`] and [`SeriesOrdering::KeyThenTime`] are accepted for raw frames and +//! only `Contiguous` for evaluated ones; `Interleaved` is rejected before any poll. +//! +//! An evaluated frame is the same physical family: [`pack_evaluated_series`] and +//! [`unpack_evaluated_series`] carry computed values under complete labels and grid positions, so a +//! compute consumer needs neither a storage key nor a scan binding to read them. use std::pin::Pin; use std::sync::Arc; @@ -40,13 +44,12 @@ use datafusion_common::DataFusionError; use datatypes::arrow::datatypes::{Schema as ArrowSchema, SchemaRef as ArrowSchemaRef}; use datatypes::error::InvalidSeriesSnafu; use datatypes::series::frame::{ - LABEL_DECODE_VERSION, LabelDecodeColumn, LabelDecodeEncoding, LabelDecodeSpec, LabelDecodeTag, - RawLabelMode, RawSeriesChunkCursor, RawSeriesFrameBuilder, RawSeriesFrameDecoder, - raw_series_frame_label_decode, raw_series_frame_schema_with_label_decode, - raw_series_frame_series_schema, + FrameContent, FrameLabelMode, LABEL_DECODE_VERSION, LabelDecodeColumn, LabelDecodeEncoding, + LabelDecodeSpec, LabelDecodeTag, SeriesFrameBuilder, SeriesFrameCursor, SeriesFrameDeclaration, + SeriesFrameDecoder, series_frame_label_decode, series_frame_schema, }; use datatypes::series::raw::RawSeriesChunk; -use datatypes::series::samples::SeriesSchema; +use datatypes::series::samples::{Series, SeriesSchema}; use datatypes::series::{SeriesLabels, SeriesOrdering}; use futures_util::Stream; use store_api::codec::PrimaryKeyEncoding; @@ -63,10 +66,19 @@ mod tests; const ORDERING_KEY: &str = "greptime.series.ordering"; +/// A stream of evaluated series chunks and their finality. +/// +/// The pair is what an evaluated producer has and an evaluated consumer needs: the values on their +/// absolute grid positions, and whether this chunk closed the chain it belongs to. It is deliberately +/// not a third runtime chunk type - `Series` is the runtime value, and finality is the one fact a +/// chunk has to carry besides it. +pub type SendableEvaluatedSeriesStream = + Pin> + Send>>; + /// Builds the canonical raw frame schema with the producer's label binding and ordering. /// /// Ordering records a producer guarantee; it is not inferred from frame contents. `Interleaved` -/// has no v1 wire representation and is rejected before the schema is assembled. The label binding +/// has no wire representation and is rejected before the schema is assembled. The label binding /// determines both the label mode and, for a primary-key binding, the self-describing label decode /// description written into the schema, so a frame can never declare a label layout it does not /// describe. @@ -75,6 +87,11 @@ pub fn raw_series_frame_schema_with_ordering( label_source: &RawLabelSource, ordering: SeriesOrdering, ) -> datafusion_common::Result { + if schema.eval_grid.is_some() { + return Err(invalid( + "a raw frame cannot carry a schema with an evaluation grid", + )); + } frame_schema_with( schema, label_mode(label_source), @@ -85,26 +102,30 @@ pub fn raw_series_frame_schema_with_ordering( /// Assembles a declared frame schema from an explicit label mode and decode description. /// -/// The mode and description pair is checked by -/// [`raw_series_frame_schema_with_label_decode`], so a key mode without a description and an -/// explicit mode with one are both rejected here instead of reaching the wire. +/// The mode and description pair is checked by [`series_frame_schema`], so a key mode without a +/// description and an explicit mode with one are both rejected here instead of reaching the wire. fn frame_schema_with( schema: &SeriesSchema, - mode: RawLabelMode, + mode: FrameLabelMode, decode: Option<&LabelDecodeSpec>, ordering: SeriesOrdering, ) -> datafusion_common::Result { + // An evaluated frame has no storage key to be ordered by, so the only claim it can honour is + // that each chain is contiguous. Writing a stronger one would produce a frame the reader refuses + // and a plan that fails where it could have been refused. + if schema.eval_grid.is_some() { + ensure_contiguous_ordering(ordering)?; + } let ordering = match ordering { SeriesOrdering::Contiguous => "contiguous", SeriesOrdering::KeyThenTime => "read_key_then_time", SeriesOrdering::Interleaved => { return Err(invalid( - "interleaved raw series ordering has no wire representation", + "interleaved series ordering has no wire representation", )); } }; - let schema = - raw_series_frame_schema_with_label_decode(schema, mode, decode).map_err(codec_error)?; + let schema = series_frame_schema(schema, mode, decode).map_err(codec_error)?; let mut metadata = schema.metadata().clone(); metadata.insert(ORDERING_KEY.to_string(), ordering.to_string()); Ok(Arc::new(ArrowSchema::new_with_metadata( @@ -163,28 +184,34 @@ pub fn label_decode_spec(source: &RawLabelSource) -> Option { Some(description) } -/// Reads the declared producer ordering, conservatively treating legacy frames as contiguous. +/// Reads the declared producer ordering, treating an absent declaration as `contiguous`. /// -/// Only the two orderings representable on the wire can be returned. -pub fn raw_series_frame_ordering( - schema: &ArrowSchema, -) -> datafusion_common::Result { +/// The absent case is deliberately the weakest guarantee rather than an error, and it is safe for +/// exactly that reason: `contiguous` says only that each series' chunks are not interleaved with +/// another's, so adopting it for a frame that never said anything can lose an optimization but +/// cannot accept a stream whose real ordering is weaker. A producer that does make a stronger claim +/// has to state it, and a producer that cannot keep one is refused: `Interleaved` is not a +/// representable value, and neither is anything this revision does not know. +/// +/// This is also what the frame's own declaration has always meant, which is what keeps a reader of +/// `v1` and `v2` text agreeing about the same bytes. +pub fn series_frame_ordering(schema: &ArrowSchema) -> datafusion_common::Result { match schema.metadata().get(ORDERING_KEY).map(String::as_str) { None | Some("contiguous") => Ok(SeriesOrdering::Contiguous), Some("read_key_then_time") => Ok(SeriesOrdering::KeyThenTime), - Some(_) => Err(invalid("unknown raw series ordering")), + Some(_) => Err(invalid("unknown series ordering")), } } /// Packs a storage raw-series stream into DataFusion record batches incrementally. /// -/// Only [`SeriesOrdering::Contiguous`] and [`SeriesOrdering::KeyThenTime`] streams are -/// accepted; an `Interleaved` declaration is rejected before the first poll, including for an -/// empty stream. Rows accumulate across chunks into batches bounded by the builder's row and -/// estimated payload limits; one legal oversized row may be emitted alone, and these limits are a -/// work bound, not a resident-memory cap. A ready batch is yielded without polling the source -/// again. Producers must keep each series' chunks contiguous. Validation checks adjacent chunk -/// conditions but does not certify nonadjacent recurrence for `Contiguous` ordering. +/// Only [`SeriesOrdering::Contiguous`] and [`SeriesOrdering::KeyThenTime`] streams are accepted; an +/// `Interleaved` declaration is rejected before the first poll, including for an empty stream. Rows +/// accumulate across chunks into batches bounded by the builder's row and estimated payload limits; +/// one legal oversized row may be emitted alone, and these limits are a work bound, not a +/// resident-memory cap. A ready batch is yielded without polling the source again. Producers must +/// keep each series' chunks contiguous. Validation checks adjacent chunk conditions but does not +/// certify nonadjacent recurrence for `Contiguous` ordering. /// /// The stream's own declaration is the only decode input: a frame unpacked without a scan-bound /// [`RawLabelSource`] round trips unchanged instead of being re-declared from a binding it does not @@ -197,27 +224,71 @@ pub fn pack_raw_series( ensure_supported_ordering(ordering)?; let schema = source.schema().clone(); let decode = source.label_decode_spec().cloned(); - let mode = declared_label_mode(&decode); + let labels = declared_label_mode(&decode); // The declared frame schema carries the producer ordering and the label decode description; - // the shared builder's schema intentionally omits both, so every ready batch is declared under - // this schema before it is yielded, including when no row is ever produced. - let frame_schema = frame_schema_with(&schema, mode, decode.as_ref(), ordering)?; - let builder = RawSeriesFrameBuilder::try_new( + // every ready batch is published under it, including when no row is ever produced. + let frame_schema = frame_schema_with(&schema, labels, decode.as_ref(), ordering)?; + let builder = SeriesFrameBuilder::try_new( &schema, - mode, - RawSeriesFrameBuilder::DEFAULT_MAX_ROWS, - RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, + labels, + decode.as_ref(), + SeriesFrameBuilder::DEFAULT_MAX_ROWS, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, ) .map_err(codec_error)?; Ok(Box::pin(PackStream { source: Some(source), schema, - mode, - frame_schema, + labels, + batches: PackBatches::new(builder, frame_schema), ordering, validator: SequenceValidator::new(ordering), - builder: Some(builder), - cursor: None, + })) +} + +/// Packs a stream of evaluated chunks into DataFusion record batches incrementally. +/// +/// An evaluated frame names its series by their complete labels and addresses time on the evaluation +/// grid `schema` declares, so there is nothing to decode and no storage key to carry: the frame +/// declares `explicit` identity and `grid_spans` time, and the grid is written once instead of +/// repeating timestamps on every row. +/// +/// Only [`SeriesOrdering::Contiguous`] is declared, because that is the one ordering this layer can +/// promise and check: each chunk chain is contiguous with itself. A caller that has reason to claim +/// [`SeriesOrdering::KeyThenTime`] about computed values would have to justify it, and a consumer +/// must never infer it from the presence of labels. +/// +/// Bounds, backpressure, and error handling are the raw path's: rows accumulate into batches bounded +/// by the builder's row and estimated payload limits, a ready batch is yielded before the source is +/// polled again, and a `Pending` source still delivers what is buffered. +pub fn pack_evaluated_series( + source: SendableEvaluatedSeriesStream, + schema: Arc, +) -> datafusion_common::Result { + if schema.eval_grid.is_none() { + return Err(invalid( + "an evaluated frame needs a schema with an evaluation grid", + )); + } + let frame_schema = frame_schema_with( + &schema, + FrameLabelMode::Explicit, + None, + SeriesOrdering::Contiguous, + )?; + let builder = SeriesFrameBuilder::try_new( + &schema, + FrameLabelMode::Explicit, + None, + SeriesFrameBuilder::DEFAULT_MAX_ROWS, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, + ) + .map_err(codec_error)?; + Ok(Box::pin(PackEvaluatedStream { + source: Some(source), + schema, + batches: PackBatches::new(builder, frame_schema), + validator: EvaluatedSequenceValidator::new(), })) } @@ -233,31 +304,40 @@ pub fn pack_raw_series( /// /// Every frame is validated here, unconditionally. The declared layout must be this revision's, the /// label decode description must parse, name a known version, and agree with the declared label -/// mode — a key frame must declare one and an explicit frame must not, since explicit labels are +/// mode - a key frame must declare one and an explicit frame must not, since explicit labels are /// already complete. The declared fields and the empty batch are then decoded once, so a stream that /// could not be decoded is rejected before it is returned. /// -/// `expected` is a check, never an input. When it is given it must match the schema the frame -/// declares — the same logical fields in the same order, and no evaluation grid — and a mismatch is -/// an error. The decoder and the returned stream's schema are built from the frame either way, so a -/// caller cannot supply a field layout the frame does not declare. +/// A frame that declares evaluated content is refused here, before anything is read: a raw consumer +/// reads timestamps rather than grid positions, so the two contents are not interchangeable. +/// +/// `expected` is a check, never an input. When it is given it must name the same logical fields in +/// the same order and the same evaluation grid as the frame declares - which for a raw frame is that +/// both sides carry none - and a mismatch is an error. The decoder and the returned stream's schema +/// are built from the frame either way, so a caller cannot supply a field layout the frame does not +/// declare. pub fn unpack_raw_series( source: DfSendableRecordBatchStream, expected: Option>, ) -> datafusion_common::Result { let frame_schema = source.schema(); - let declared = raw_series_frame_label_decode(frame_schema.as_ref()).map_err(codec_error)?; - let mode = declared_label_mode(&declared); - let schema = - Arc::new(raw_series_frame_series_schema(frame_schema.as_ref()).map_err(codec_error)?); + let declaration = SeriesFrameDeclaration::parse(frame_schema.as_ref()).map_err(codec_error)?; + if declaration.content() != FrameContent::Raw { + return Err(invalid("expected a raw frame, found an evaluated one")); + } + let labels = declaration.labels(); + let declared = declaration.label_decode().cloned(); + let schema = Arc::new(declaration.series().clone()); if let Some(expected) = &expected { ensure_declared_schema(&schema, expected)?; } - let ordering = raw_series_frame_ordering(frame_schema.as_ref())?; - let decoder = RawSeriesFrameDecoder::try_new(&schema, mode).map_err(codec_error)?; + let ordering = series_frame_ordering(frame_schema.as_ref())?; + ensure_supported_ordering(ordering)?; + let decoder = + SeriesFrameDecoder::try_new(&schema, labels, declared.as_ref()).map_err(codec_error)?; // Validate even empty streams' declared schema and layout before returning the stream. let empty = DfRecordBatch::new_empty(frame_schema.clone()); - decoder.decode(&empty).map_err(codec_error)?; + decoder.decode_raw(&empty).map_err(codec_error)?; Ok(Box::pin(UnpackStream { source: Some(source), schema, @@ -270,20 +350,53 @@ pub fn unpack_raw_series( })) } +/// Unpacks DataFusion batches declared by [`pack_evaluated_series`]. +/// +/// The frame declares its grid, its field list, and its label mode, so the returned stream carries +/// the [`SeriesSchema`] the rows are addressed on and needs no planning-side copy of it. The grid is +/// the frame's own declaration: the timestamps stay unmaterialized until a consumer asks for one. +/// +/// A frame that declares raw samples, keyed identity, or an ordering other than `contiguous` is +/// refused before the stream is returned, as is a grid the declaration check cannot make sense of. +/// `expected` is a check and never an input, exactly as on the raw path: when given it must name the +/// same fields and the same grid the frame declares. +pub fn unpack_evaluated_series( + source: DfSendableRecordBatchStream, + expected: Option>, +) -> datafusion_common::Result { + let frame_schema = source.schema(); + let declaration = SeriesFrameDeclaration::parse(frame_schema.as_ref()).map_err(codec_error)?; + if declaration.content() != FrameContent::Evaluated { + return Err(invalid("expected an evaluated frame, found a raw one")); + } + ensure_contiguous_ordering(series_frame_ordering(frame_schema.as_ref())?)?; + let schema = Arc::new(declaration.series().clone()); + if let Some(expected) = &expected { + ensure_declared_schema(&schema, expected)?; + } + let decoder = SeriesFrameDecoder::try_new(&schema, FrameLabelMode::Explicit, None) + .map_err(codec_error)?; + let empty = DfRecordBatch::new_empty(frame_schema.clone()); + decoder.decode_evaluated(&empty).map_err(codec_error)?; + Ok(Box::pin(UnpackEvaluatedStream { + source: Some(source), + frame_schema, + validator: EvaluatedSequenceValidator::new(), + pending: Vec::new().into_iter(), + decoder, + })) +} + /// Rejects an `expected` series schema the frame does not declare. /// /// Only the two things a caller can get wrong are checked, because those are the only two things -/// [`SeriesSchema`] holds: the logical fields and their order, and an evaluation grid a raw frame -/// cannot carry. +/// [`SeriesSchema`] holds: the logical fields and their order, and the evaluation grid. A raw frame +/// carries no grid and an evaluated one carries exactly its own, so the grid comparison is also what +/// keeps a raw expectation from decoding an evaluated frame. fn ensure_declared_schema( declared: &SeriesSchema, expected: &SeriesSchema, ) -> datafusion_common::Result<()> { - if expected.eval_grid.is_some() { - return Err(invalid( - "expected raw series schema declares an evaluation grid the frame cannot carry", - )); - } if expected.fields.len() != declared.fields.len() || expected .fields @@ -292,10 +405,16 @@ fn ensure_declared_schema( .any(|(expected, declared)| expected.name != declared.name) { return Err(invalid( - "raw frame fields do not match the expected series schema", + "frame fields do not match the expected series schema", )); } - Ok(()) + match (&expected.eval_grid, &declared.eval_grid) { + (None, None) => Ok(()), + (Some(expected), Some(declared)) if expected.spec() == declared.spec() => Ok(()), + _ => Err(invalid( + "frame evaluation grid does not match the expected series schema", + )), + } } /// Checks that a frame declares the label decode layout a scan-bound binding describes. @@ -313,15 +432,14 @@ fn ensure_declared_schema( /// [`unpack_raw_series`] decodes it without a binding, and a caller that received the frame from a /// peer compares the declaration here instead of decoding under whichever side it trusts. /// -/// The comparison is plain equality. It used to canonicalize logical tag order on both sides, which -/// gave local comparisons one rule and the byte-exact remote comparison another; the writer now -/// canonicalizes instead (see [`LabelDecodeSpec::canonicalize`]), so a description that reaches a -/// reader already has one spelling and equality says everything there is to say about it. +/// The comparison is plain equality. The writer canonicalizes the description (see +/// [`LabelDecodeSpec::canonicalize`]), so a description that reaches a reader already has one +/// spelling and equality says everything there is to say about it. pub fn ensure_frame_matches_label_source( frame: &ArrowSchema, label_source: &RawLabelSource, ) -> datafusion_common::Result<()> { - let declared = raw_series_frame_label_decode(frame).map_err(codec_error)?; + let declared = series_frame_label_decode(frame).map_err(codec_error)?; if declared != label_decode_spec(label_source) { return Err(invalid( "raw frame label decode description does not match the supplied binding", @@ -330,19 +448,19 @@ pub fn ensure_frame_matches_label_source( Ok(()) } -fn label_mode(source: &RawLabelSource) -> RawLabelMode { +fn label_mode(source: &RawLabelSource) -> FrameLabelMode { match source { - RawLabelSource::PrimaryKey { .. } => RawLabelMode::Key, - RawLabelSource::Explicit => RawLabelMode::Explicit, + RawLabelSource::PrimaryKey { .. } => FrameLabelMode::Key, + RawLabelSource::Explicit => FrameLabelMode::Explicit, } } /// Returns the label mode a frame declares: a key frame describes its key, an explicit frame /// carries complete labels and declares no description. -fn declared_label_mode(declared: &Option) -> RawLabelMode { +fn declared_label_mode(declared: &Option) -> FrameLabelMode { match declared { - Some(_) => RawLabelMode::Key, - None => RawLabelMode::Explicit, + Some(_) => FrameLabelMode::Key, + None => FrameLabelMode::Explicit, } } @@ -356,6 +474,21 @@ fn ensure_supported_ordering(ordering: SeriesOrdering) -> datafusion_common::Res } } +/// Rejects an evaluated frame that claims more order than its producer promised. +/// +/// Computed values have no storage key to be ordered by, so `read_key_then_time` is not a claim this +/// layer can honour; `Interleaved` is not consumable at all. Treating either as `contiguous` would +/// silently weaken a guarantee a consumer may rely on. +fn ensure_contiguous_ordering(ordering: SeriesOrdering) -> datafusion_common::Result<()> { + match ordering { + SeriesOrdering::Contiguous => Ok(()), + SeriesOrdering::KeyThenTime => Err(invalid( + "an evaluated frame cannot claim key-then-time ordering", + )), + SeriesOrdering::Interleaved => Err(invalid("interleaved series ordering is not supported")), + } +} + fn codec_error(error: datatypes::error::Error) -> DataFusionError { DataFusionError::External(Box::new(BoxedError::new(error))) } @@ -373,9 +506,10 @@ fn invalid(reason: impl Into) -> DataFusionError { ))) } -/// Cross-chunk validation state. The producer must keep each series' chunks contiguous; this -/// validator checks adjacent chunks only and never sees `Interleaved` streams. It retains only -/// the previous complete owned key (not an Arrow buffer), timestamp, metric, and finality. +/// Cross-chunk validation state for raw chains. The producer must keep each series' chunks +/// contiguous; this validator checks adjacent chunks only and never sees `Interleaved` streams. It +/// retains only the previous complete owned key (not an Arrow buffer), timestamp, metric, and +/// finality. struct SequenceValidator { ordering: SeriesOrdering, previous_key: Option>, @@ -447,30 +581,95 @@ impl SequenceValidator { } } -struct PackStream { - source: Option, - schema: Arc, - mode: RawLabelMode, - frame_schema: datatypes::arrow::datatypes::SchemaRef, - ordering: SeriesOrdering, - validator: SequenceValidator, - builder: Option, - cursor: Option, +/// Cross-chunk validation state for evaluated chains. +/// +/// `is_last` closes one chunk chain and nothing more. It is not the end of a label set: a range +/// function or a label rewrite can make two input series show the same complete labels, and each of +/// them closes itself, so a closed chain may be followed by another chain whose labels happen to be +/// equal. Only the still-open chain is remembered, which is what keeps such a contribution from being +/// refused as a re-opened series while still refusing a chain that changes identity before it closes. +struct EvaluatedSequenceValidator { + /// The labels of the open chain and the last grid position it carried. + open: Option<(SeriesLabels, usize)>, } -impl PackStream { - /// Releases the source and drops every buffered owner, leaving the stream terminal; batches - /// already yielded are unaffected. - fn finish(&mut self) { - self.source = None; - self.cursor = None; - self.builder = None; - self.validator = SequenceValidator::new(self.ordering); +impl EvaluatedSequenceValidator { + fn new() -> Self { + Self { open: None } + } + + fn finish(&self) -> datafusion_common::Result<()> { + if self.open.is_some() { + return Err(invalid( + "evaluated series stream ended before a chain closed", + )); + } + Ok(()) + } + + fn validate(&mut self, series: &Series, is_last: bool) -> datafusion_common::Result<()> { + let (Some(first), Some(last)) = (series.time.first_position(), series.time.last_position()) + else { + // No carried position: only a closing chunk may be empty, and it closes the chain it + // belongs to without advancing it. + if !is_last { + return Err(invalid( + "an evaluated chunk without positions must close its series", + )); + } + match &self.open { + Some((labels, _)) if labels == &series.metric => self.open = None, + Some(_) => { + return Err(invalid( + "evaluated series identity changed before its chain closed", + )); + } + None => {} + } + return Ok(()); + }; + if let Some((labels, previous)) = &self.open { + if labels != &series.metric { + return Err(invalid( + "evaluated series identity changed before its chain closed", + )); + } + if first <= *previous { + return Err(invalid( + "evaluated series positions are not increasing across chunks", + )); + } + } + self.open = (!is_last).then(|| (series.metric.clone(), last)); + Ok(()) + } +} + +/// The batching half of a pack stream: the builder, the cursor installed in it, and the declared +/// schema every ready batch is published under. +struct PackBatches { + frame_schema: ArrowSchemaRef, + builder: Option, + cursor: Option, +} + +impl PackBatches { + fn new(builder: SeriesFrameBuilder, frame_schema: ArrowSchemaRef) -> Self { + Self { + frame_schema, + builder: Some(builder), + cursor: None, + } + } + + /// Installs the cursor of the chunk just read, replacing whatever was left of the previous one. + fn install(&mut self, cursor: SeriesFrameCursor) { + self.cursor = Some(cursor); } /// Advances the installed cursor until it yields one ready batch or is exhausted, dropping an /// exhausted cursor. `None` means no batch is ready, so the source may be polled again. - fn advance_cursor(&mut self) -> datatypes::error::Result> { + fn advance(&mut self) -> datatypes::error::Result> { let ready = match (self.cursor.as_mut(), self.builder.as_mut()) { (Some(cursor), Some(builder)) => cursor.append_to(builder)?, _ => return Ok(None), @@ -485,38 +684,63 @@ impl PackStream { } /// Flushes the rows buffered so far, if any, so a pending source can still deliver data. - fn flush_buffered(&mut self) -> datatypes::error::Result> { + fn flush(&mut self) -> datatypes::error::Result> { match self.builder.as_mut() { Some(builder) => builder.flush(), None => Ok(None), } } - /// Publishes one ready builder batch under the declared frame schema, which is the shared - /// builder's schema plus the producer's ordering metadata. A batch whose fields cannot be - /// declared releases the stream instead of yielding rows with an undeclared layout. - fn declare_batch( + /// Releases the buffered owners, leaving the stream terminal; batches already yielded are + /// unaffected. + fn release(&mut self) { + self.cursor = None; + self.builder = None; + } + + /// Publishes one ready batch under the declared frame schema, which is the builder's own layout + /// plus the producer's ordering metadata. A batch whose fields cannot be declared releases the + /// stream instead of yielding rows with an undeclared layout. + fn declare( &mut self, batch: DfRecordBatch, ) -> Poll>> { match batch.with_schema(self.frame_schema.clone()) { Ok(batch) => Poll::Ready(Some(Ok(batch))), Err(error) => { - self.finish(); + self.release(); Poll::Ready(Some(Err(DataFusionError::from(error)))) } } } } +struct PackStream { + source: Option, + schema: Arc, + labels: FrameLabelMode, + batches: PackBatches, + ordering: SeriesOrdering, + validator: SequenceValidator, +} + +impl PackStream { + /// Releases the source and drops every buffered owner, leaving the stream terminal. + fn finish(&mut self) { + self.source = None; + self.batches.release(); + self.validator = SequenceValidator::new(self.ordering); + } +} + impl Stream for PackStream { type Item = datafusion_common::Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = &mut *self; loop { - match this.advance_cursor() { - Ok(Some(batch)) => return this.declare_batch(batch), + match this.batches.advance() { + Ok(Some(batch)) => return this.batches.declare(batch), Ok(None) => {} Err(error) => { this.finish(); @@ -529,9 +753,9 @@ impl Stream for PackStream { }; match polled { Poll::Pending => { - let flushed = this.flush_buffered(); + let flushed = this.batches.flush(); return match flushed { - Ok(Some(batch)) => this.declare_batch(batch), + Ok(Some(batch)) => this.batches.declare(batch), Ok(None) => Poll::Pending, Err(error) => { this.finish(); @@ -544,10 +768,16 @@ impl Stream for PackStream { this.finish(); return Poll::Ready(Some(Err(error))); } - let flushed = this.flush_buffered(); + let flushed = this.batches.flush(); + // The declared schema is read before the release below, so the last batch is + // published under the same schema as every batch before it. + let declared = this.batches.frame_schema.clone(); this.finish(); return match flushed { - Ok(Some(batch)) => this.declare_batch(batch), + Ok(Some(batch)) => match batch.with_schema(declared) { + Ok(batch) => Poll::Ready(Some(Ok(batch))), + Err(error) => Poll::Ready(Some(Err(DataFusionError::from(error)))), + }, Ok(None) => Poll::Ready(None), Err(error) => Poll::Ready(Some(Err(codec_error(error)))), }; @@ -561,8 +791,8 @@ impl Stream for PackStream { this.finish(); return Poll::Ready(Some(Err(error))); } - match RawSeriesChunkCursor::try_new(&this.schema, this.mode, chunk) { - Ok(cursor) => this.cursor = Some(cursor), + match SeriesFrameCursor::try_new_raw(&this.schema, this.labels, chunk) { + Ok(cursor) => this.batches.install(cursor), Err(error) => { this.finish(); return Poll::Ready(Some(Err(codec_error(error)))); @@ -575,21 +805,115 @@ impl Stream for PackStream { } impl datafusion::physical_plan::RecordBatchStream for PackStream { - fn schema(&self) -> datatypes::arrow::datatypes::SchemaRef { - self.frame_schema.clone() + fn schema(&self) -> ArrowSchemaRef { + self.batches.frame_schema.clone() + } +} + +/// The pack stream of [`pack_evaluated_series`]. +struct PackEvaluatedStream { + source: Option, + schema: Arc, + batches: PackBatches, + validator: EvaluatedSequenceValidator, +} + +impl PackEvaluatedStream { + /// Releases the source and drops every buffered owner, leaving the stream terminal. + fn finish(&mut self) { + self.source = None; + self.batches.release(); + self.validator = EvaluatedSequenceValidator::new(); + } +} + +impl Stream for PackEvaluatedStream { + type Item = datafusion_common::Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = &mut *self; + loop { + match this.batches.advance() { + Ok(Some(batch)) => return this.batches.declare(batch), + Ok(None) => {} + Err(error) => { + this.finish(); + return Poll::Ready(Some(Err(codec_error(error)))); + } + } + let polled = match this.source.as_mut() { + Some(source) => source.as_mut().poll_next(cx), + None => return Poll::Ready(None), + }; + match polled { + Poll::Pending => { + let flushed = this.batches.flush(); + return match flushed { + Ok(Some(batch)) => this.batches.declare(batch), + Ok(None) => Poll::Pending, + Err(error) => { + this.finish(); + Poll::Ready(Some(Err(codec_error(error)))) + } + }; + } + Poll::Ready(None) => { + if let Err(error) = this.validator.finish() { + this.finish(); + return Poll::Ready(Some(Err(error))); + } + let flushed = this.batches.flush(); + // The declared schema is read before the release below, so the last batch is + // published under the same schema as every batch before it. + let declared = this.batches.frame_schema.clone(); + this.finish(); + return match flushed { + Ok(Some(batch)) => match batch.with_schema(declared) { + Ok(batch) => Poll::Ready(Some(Ok(batch))), + Err(error) => Poll::Ready(Some(Err(DataFusionError::from(error)))), + }, + Ok(None) => Poll::Ready(None), + Err(error) => Poll::Ready(Some(Err(codec_error(error)))), + }; + } + Poll::Ready(Some(Err(error))) => { + this.finish(); + return Poll::Ready(Some(Err(upstream_error(error)))); + } + Poll::Ready(Some(Ok((series, is_last)))) => { + if let Err(error) = this.validator.validate(&series, is_last) { + this.finish(); + return Poll::Ready(Some(Err(error))); + } + match SeriesFrameCursor::try_new_evaluated(&this.schema, series, is_last) { + Ok(cursor) => this.batches.install(cursor), + Err(error) => { + this.finish(); + return Poll::Ready(Some(Err(codec_error(error)))); + } + } + } + } + } + } +} + +impl datafusion::physical_plan::RecordBatchStream for PackEvaluatedStream { + fn schema(&self) -> ArrowSchemaRef { + self.batches.frame_schema.clone() } } struct UnpackStream { source: Option, schema: Arc, - frame_schema: datatypes::arrow::datatypes::SchemaRef, + frame_schema: ArrowSchemaRef, /// The description parsed from the frame schema; the only input decoding needs. label_decode_spec: Option, ordering: SeriesOrdering, validator: SequenceValidator, pending: std::vec::IntoIter, - decoder: RawSeriesFrameDecoder, + decoder: SeriesFrameDecoder, } impl RawSeriesStream for UnpackStream { @@ -628,7 +952,6 @@ impl Stream for UnpackStream { type Item = Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - const MAX_EMPTY_BATCHES_PER_POLL: usize = 32; let this = &mut *self; let mut empty_batches = 0; loop { @@ -644,12 +967,7 @@ impl Stream for UnpackStream { Poll::Ready(None) => { let result = this.validator.finish(); this.finish(); - return Poll::Ready(result.err().map(|error| { - Err(BoxedError::new(crate::error::Error::GeneralDataFusion { - error, - location: snafu::Location::default(), - })) - })); + return Poll::Ready(result.err().map(|error| Err(unpack_error(error)))); } Poll::Ready(Some(Err(error))) => { this.finish(); @@ -666,7 +984,7 @@ impl Stream for UnpackStream { return Err(invalid("raw frame schema mismatch")); } this.decoder - .decode(&batch) + .decode_raw(&batch) .map_err(codec_error) .and_then(|chunks| { for chunk in &chunks { @@ -689,12 +1007,7 @@ impl Stream for UnpackStream { } Err(error) => { this.finish(); - return Poll::Ready(Some(Err(BoxedError::new( - crate::error::Error::GeneralDataFusion { - error, - location: snafu::Location::default(), - }, - )))); + return Poll::Ready(Some(Err(unpack_error(error)))); } } } @@ -702,3 +1015,99 @@ impl Stream for UnpackStream { } } } + +/// The unpack stream of [`unpack_evaluated_series`]. +struct UnpackEvaluatedStream { + source: Option, + frame_schema: ArrowSchemaRef, + validator: EvaluatedSequenceValidator, + pending: std::vec::IntoIter<(Series, bool)>, + decoder: SeriesFrameDecoder, +} + +impl UnpackEvaluatedStream { + fn finish(&mut self) { + self.source = None; + self.pending = Vec::new().into_iter(); + self.validator = EvaluatedSequenceValidator::new(); + } +} + +impl Stream for UnpackEvaluatedStream { + type Item = Result<(Series, bool), BoxedError>; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = &mut *self; + let mut empty_batches = 0; + loop { + if let Some(chunk) = this.pending.next() { + return Poll::Ready(Some(Ok(chunk))); + } + let polled = match this.source.as_mut() { + Some(source) => source.as_mut().poll_next(cx), + None => return Poll::Ready(None), + }; + match polled { + Poll::Pending => return Poll::Pending, + Poll::Ready(None) => { + let result = this.validator.finish(); + this.finish(); + return Poll::Ready(result.err().map(|error| Err(unpack_error(error)))); + } + Poll::Ready(Some(Err(error))) => { + this.finish(); + return Poll::Ready(Some(Err(BoxedError::new( + crate::error::Error::GeneralDataFusion { + error, + location: snafu::Location::default(), + }, + )))); + } + Poll::Ready(Some(Ok(batch))) => { + let result = (|| { + if batch.schema() != this.frame_schema { + return Err(invalid("evaluated frame schema mismatch")); + } + this.decoder + .decode_evaluated(&batch) + .map_err(codec_error) + .and_then(|chunks| { + for (series, is_last) in &chunks { + this.validator.validate(series, *is_last)?; + } + Ok(chunks) + }) + })(); + match result { + Ok(chunks) => { + if chunks.is_empty() { + empty_batches += 1; + if empty_batches == MAX_EMPTY_BATCHES_PER_POLL { + cx.waker().wake_by_ref(); + return Poll::Pending; + } + } else { + this.pending = chunks.into_iter(); + } + } + Err(error) => { + this.finish(); + return Poll::Ready(Some(Err(unpack_error(error)))); + } + } + } + } + } + } +} + +/// How many empty batches one poll reads before yielding, so a stream of them cannot hold the task. +const MAX_EMPTY_BATCHES_PER_POLL: usize = 32; + +/// Wraps a decode or validation error as the boxed error an unpack stream reports. +fn unpack_error(error: DataFusionError) -> BoxedError { + BoxedError::new(crate::error::Error::GeneralDataFusion { + error, + location: snafu::Location::default(), + }) +} diff --git a/src/common/query/src/series_frame/tests.rs b/src/common/query/src/series_frame/tests.rs index 61509c4f12c..327e9295d53 100644 --- a/src/common/query/src/series_frame/tests.rs +++ b/src/common/query/src/series_frame/tests.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod evaluated; mod lifecycle; use std::collections::{HashMap, VecDeque}; @@ -34,7 +35,9 @@ use datatypes::arrow::record_batch::RecordBatch; use datatypes::series::Label; use datatypes::series::SeriesLabels; use datatypes::series::SeriesOrdering; -use datatypes::series::frame::{LabelDecodeSpec, RawLabelMode, raw_series_frame_schema}; +use datatypes::series::frame::{ + FrameLabelMode, LabelDecodeSpec, raw_series_frame_schema, series_frame_schema, +}; use datatypes::series::histogram::{BucketSideView, CounterResetHint, HistogramBatchBuilder}; use datatypes::series::raw::{RawSeriesChunk, RawTimestamps}; use datatypes::series::samples::{ @@ -46,8 +49,8 @@ use store_api::region_engine::{ }; use super::{ - ensure_frame_matches_label_source, label_decode_spec, pack_raw_series, - raw_series_frame_ordering, unpack_raw_series, + ensure_frame_matches_label_source, label_decode_spec, pack_raw_series, series_frame_ordering, + unpack_raw_series, }; fn schema() -> Arc { @@ -59,6 +62,28 @@ fn schema() -> Arc { eval_grid: None, }) } + +/// The one grid every evaluated fixture is addressed on: 512 positions, 1000 ms apart. +fn evaluated_grid() -> Arc { + Arc::new( + datatypes::series::time::EvalGrid::try_new( + datatypes::series::time::GridSpec::try_new(0, 1000, 512).unwrap(), + ) + .unwrap(), + ) +} + +/// A schema with one `value` field addressed on [`evaluated_grid`], which is what makes it an +/// evaluated declaration. +fn evaluated_schema() -> Arc { + Arc::new(SeriesSchema { + fields: vec![FieldSpec { + name: Arc::from("value"), + }] + .into(), + eval_grid: Some(evaluated_grid()), + }) +} fn metric() -> SeriesLabels { SeriesLabels::try_new(Some(Arc::from("cpu")), vec![]).unwrap() } @@ -629,7 +654,8 @@ async fn invalid_sequence_reports_valid_prefix_then_one_terminal_error() { assert_eq!(batch.num_rows(), case.prefix.len(), "{}", case.name); let chunks = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &batch, ) .unwrap(); @@ -656,7 +682,8 @@ async fn invalid_sequence_reports_valid_prefix_then_one_terminal_error() { frames.extend( datatypes::series::frame::pack_raw_series_chunk( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, item, ) .unwrap(), @@ -692,7 +719,8 @@ async fn invalid_sequence_reports_valid_prefix_then_one_terminal_error() { // missing final key is reported once, at EOF. let frame = datatypes::series::frame::pack_raw_series_chunk( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &chunk(b"unfinished", &[7], false), ) .unwrap(); @@ -923,7 +951,8 @@ async fn explicit_metric_must_remain_stable_within_key() { ); let chunks = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &batch, ) .unwrap(); @@ -975,7 +1004,7 @@ async fn empty_streams_keep_self_describing_schema() { assert_eq!( frame_schema.field(0).name(), "read_key", - "the v1 wire column name is unchanged" + "the wire column name is unchanged" ); assert!(packed.next().await.is_none()); // A declared-but-empty stream is still validated against the expected schema before any @@ -1018,7 +1047,7 @@ async fn upstream_error_status_is_preserved_once_and_releases_source() { assert_eq!(drops.load(Ordering::SeqCst), 1); let batch_schema = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let plain = PlainError::new("upstream df".into(), status); @@ -1062,7 +1091,8 @@ async fn dropping_adapter_releases_upstream_and_pending_item_is_resumed() { ); let chunks = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &first, ) .unwrap(); @@ -1076,7 +1106,8 @@ async fn dropping_adapter_releases_upstream_and_pending_item_is_resumed() { ); let chunks = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &second, ) .unwrap(); @@ -1094,18 +1125,25 @@ async fn unpack_rejects_mismatched_batch_schema_before_polling() { assert_eq!(drops.load(Ordering::SeqCst), 1); } +/// A schema with an evaluation grid is a legal *evaluated* frame declaration, so it is no longer a +/// reason to refuse a stream at construction; it is still no schema to carry *raw* chunks under, so +/// the first raw chunk is rejected instead of the stream. #[tokio::test] -async fn invalid_grid_schema_rejected_for_empty_raw_stream() { - let mut invalid = (*schema()).clone(); - invalid.eval_grid = Some(Arc::new( +async fn grid_schema_is_legal_but_cannot_carry_a_raw_chunk() { + let mut grid_schema = (*schema()).clone(); + grid_schema.eval_grid = Some(Arc::new( datatypes::series::time::EvalGrid::try_new( datatypes::series::time::GridSpec::try_new(0, 10, 1).unwrap(), ) .unwrap(), )); + let grid_schema = Arc::new(grid_schema); + + // An empty stream under a grid schema constructs: the declaration is legal and no row disagrees + // with it. let polls = Arc::new(AtomicUsize::new(0)); let source = Box::pin(RawMock { - schema: Arc::new(invalid), + schema: grid_schema.clone(), items: VecDeque::new(), binding: RawLabelSource::Explicit, label_decode_spec: None, @@ -1117,8 +1155,33 @@ async fn invalid_grid_schema_rejected_for_empty_raw_stream() { panic_on_poll: false, emitted: 0, }); - assert!(pack_raw_series(source).is_err()); - assert_eq!(polls.load(Ordering::SeqCst), 0); + let mut packed = pack_raw_series(source).unwrap(); + assert!(packed.next().await.is_none()); + assert_eq!(polls.load(Ordering::SeqCst), 1); + + // A raw chunk cannot be addressed on a grid, so the chunk that arrives under this declaration is + // what fails. + let polls = Arc::new(AtomicUsize::new(0)); + let source = Box::pin(RawMock { + schema: grid_schema, + items: VecDeque::from([Ok(chunk(b"a", &[1], true))]), + binding: RawLabelSource::Explicit, + label_decode_spec: None, + ordering: SeriesOrdering::Contiguous, + polls: polls.clone(), + drops: Arc::new(AtomicUsize::new(0)), + pending_after: None, + pending_reported: false, + panic_on_poll: false, + emitted: 0, + }); + let mut packed = pack_raw_series(source).unwrap(); + let error = packed.next().await.unwrap().unwrap_err(); + assert_eq!( + crate::error::datafusion_status_code::(&error, None), + StatusCode::InvalidArguments + ); + assert!(packed.next().await.is_none()); } #[tokio::test] @@ -1150,19 +1213,21 @@ async fn unpack_batch_sequence_error_is_atomic_for_all_rows() { async fn unpack_rejects_late_invalid_frame_row_before_prefix() { let frame_schema = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let first = datatypes::series::frame::pack_raw_series_chunk( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &chunk(b"a", &[1], true), ) .unwrap() .remove(0); let mut second = datatypes::series::frame::pack_raw_series_chunk( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &chunk(b"b", &[2], true), ) .unwrap() @@ -1211,7 +1276,7 @@ async fn unpack_rejects_late_invalid_frame_row_before_prefix() { async fn frame_application_metadata_is_stable_and_schema_change_rejected() { let base_schema = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let mut application_metadata = base_schema.metadata().clone(); @@ -1223,7 +1288,8 @@ async fn frame_application_metadata_is_stable_and_schema_change_rejected() { ); let first = datatypes::series::frame::pack_raw_series_chunk( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &chunk(b"a", &[1], true), ) .unwrap() @@ -1232,7 +1298,8 @@ async fn frame_application_metadata_is_stable_and_schema_change_rejected() { .unwrap(); let stable_second = datatypes::series::frame::pack_raw_series_chunk( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &chunk(b"b", &[2], true), ) .unwrap() @@ -1267,7 +1334,8 @@ async fn frame_application_metadata_is_stable_and_schema_change_rejected() { let changed_schema = Arc::new((*base_schema).clone().with_metadata(changed)); let bad_batch = datatypes::series::frame::pack_raw_series_chunk( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &chunk(b"b", &[2], true), ) .unwrap() @@ -1380,7 +1448,7 @@ fn unpack_yields_at_empty_batch_budget_before_observing_eof() { const BUDGET: usize = 32; let frame_schema = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let batches = vec![RecordBatch::new_empty(frame_schema.clone()); BUDGET]; @@ -1406,7 +1474,7 @@ fn unpack_yields_at_empty_batch_budget_before_observing_eof() { fn unpack_propagates_upstream_pending_without_extra_wake() { let frame_schema = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let (source, polls, _) = df_pending( @@ -1431,7 +1499,7 @@ fn unpack_reports_bad_batch_at_budget_boundary_before_yielding() { const BUDGET: usize = 32; let frame_schema = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let mut batches = vec![RecordBatch::new_empty(frame_schema.clone()); BUDGET - 1]; @@ -1460,7 +1528,7 @@ fn unpack_surfaces_upstream_error_after_budget_yield_once() { const BUDGET: usize = 32; let frame_schema = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let error = datafusion_common::DataFusionError::External(Box::new(BoxedError::new( @@ -1537,6 +1605,10 @@ async fn unpack_drains_decoded_rows_before_polling_another_batch() { assert!(unpacked.next().await.is_none()); } +/// An `expected` schema the raw frame does not declare is rejected before any poll, whether it names +/// no field at all or carries an evaluation grid. A grid schema is a legal *evaluated* declaration +/// now, so it is refused here because a raw frame carries no grid, not because the schema itself is +/// impossible. #[tokio::test] async fn unpack_preflights_empty_and_grid_schemas_without_polling() { let invalid_schemas = [ @@ -1557,7 +1629,7 @@ async fn unpack_preflights_empty_and_grid_schemas_without_polling() { for expected in invalid_schemas { let frame_schema = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let (source, polls, drops) = df_source(vec![], frame_schema); @@ -1749,7 +1821,8 @@ async fn pack_bounds_ready_rows_at_default_row_limit_before_eof() { for batch in [&first, &second] { let decoded = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, batch, ) .unwrap(); @@ -1824,7 +1897,8 @@ async fn pack_splits_one_chunk_into_bounded_batches_without_repolling_the_source for batch in [&first, &second] { let decoded = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, batch, ) .unwrap(); @@ -1877,7 +1951,8 @@ async fn dropping_pack_mid_chunk_keeps_last_batch_and_releases_source() { drop(owners); let decoded = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &batch, ) .unwrap(); @@ -1926,7 +2001,8 @@ async fn pack_flushes_ready_rows_at_pending_then_remainder_at_eof() { assert_eq!(wake.0.load(Ordering::SeqCst), 1, "upstream wakes the task"); let decoded = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &first, ) .unwrap(); @@ -1943,7 +2019,8 @@ async fn pack_flushes_ready_rows_at_pending_then_remainder_at_eof() { assert_eq!(drops.load(Ordering::SeqCst), 1); let decoded = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &second, ) .unwrap(); @@ -1997,7 +2074,8 @@ async fn pack_emits_oversized_legal_rows_whole() { ] { let chunks = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, batch, ) .unwrap(); @@ -2095,7 +2173,7 @@ async fn multirow_ipc_roundtrip_preserves_cross_chunk_variable_segments() { assert_eq!( reader.schema(), batches[0].schema(), - "IPC preserves the v1 layout, metadata, and field order" + "IPC preserves the layout, metadata, and field order" ); let frame_schema = reader.schema(); let ipc_batches = reader.map(Result::unwrap).collect::>(); @@ -2285,8 +2363,8 @@ async fn public_chunk_with_empty_times_is_rejected_without_panic() { } /// `Interleaved` is rejected by the constructor before any source poll, for a pack source and for -/// an empty unpack stream; the wire-schema helper rejects it before assembling a schema, since no -/// v1 metadata value represents it. No adjacent-only fallback is installed. +/// an empty unpack stream; the wire-schema helper rejects it before assembling a schema, since the +/// wire carries only the two representable spellings. No adjacent-only fallback is installed. #[tokio::test] async fn interleaved_ordering_is_rejected_before_any_poll() { assert!( @@ -2319,7 +2397,7 @@ async fn interleaved_ordering_is_rejected_before_any_poll() { // a frame that declares one; that spelling has no wire representation and is rejected. let frame = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let mut metadata = frame.metadata().clone(); @@ -2343,17 +2421,20 @@ async fn interleaved_ordering_is_rejected_before_any_poll() { /// The unpack stream rejects a batch whose schema differs from the stream's declared schema, /// including drift in metadata the codec itself does not interpret and an empty batch that -/// carries the drift; the reserved layout metadata is rejected as well. +/// carries the drift; the reserved layout metadata is rejected as well. Since the current layout is +/// `v2`, the drift case rewrites the layout to `v1`: any value other than this revision's is refused, +/// so a stale or a future spelling cannot be decoded under the current columns. #[tokio::test] async fn unpack_rejects_unrelated_metadata_drift_and_empty_schema_changes() { let base = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let first = datatypes::series::frame::pack_raw_series_chunk( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &chunk(b"a", &[1], true), ) .unwrap() @@ -2361,8 +2442,9 @@ async fn unpack_rejects_unrelated_metadata_drift_and_empty_schema_changes() { let mut unrelated = base.metadata().clone(); unrelated.insert("app.unrelated".into(), "x".into()); + // The current layout is `v2`, so a frame that still declares `v1` is the drift here. let mut reserved = base.metadata().clone(); - reserved.insert("greptime.series.layout".into(), "v2".into()); + reserved.insert("greptime.series.layout".into(), "v1".into()); let mut renamed = base.metadata().clone(); renamed.insert("app.renamed".into(), "y".into()); @@ -2408,6 +2490,9 @@ async fn unpack_rejects_unrelated_metadata_drift_and_empty_schema_changes() { } } +/// The ordering and layout fixture: literal metadata maps are read the way a receiver reads them, +/// so the same text pins both what `series_frame_ordering` answers and whether the declaration as a +/// whole is accepted. #[test] fn raw_series_ordering_metadata_fixture() { #[derive(serde::Deserialize)] @@ -2415,10 +2500,12 @@ fn raw_series_ordering_metadata_fixture() { name: String, metadata: HashMap, ordering: Option, + /// `accepted` or `rejected`: whether the whole declaration is one this revision reads. + declaration: String, } let cases: Vec = serde_json::from_str(include_str!( - "../../../../../tests/compatibility/wire/raw_series_ordering/v1_metadata.json" + "../../../../../tests/compatibility/wire/raw_series_ordering/v2_metadata.json" )) .unwrap(); for case in cases { @@ -2428,22 +2515,52 @@ fn raw_series_ordering_metadata_fixture() { Some(value) => panic!("{} has invalid fixture ordering {value}", case.name), None => None, }; - let frame_schema = raw_series_frame_schema(&schema(), RawLabelMode::Explicit).unwrap(); + // The columns have to match the content the case declares, so an evaluated case is built on + // an evaluated layout rather than read through a raw one. + let evaluated = case + .metadata + .get("greptime.series.content") + .is_some_and(|content| content == "evaluated"); + let frame_schema = if evaluated { + series_frame_schema(&evaluated_schema(), FrameLabelMode::Explicit, None).unwrap() + } else { + raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap() + }; let frame_schema = frame_schema.as_ref().clone().with_metadata(case.metadata); match literal_ordering { Some(expected) => assert_eq!( - raw_series_frame_ordering(&frame_schema) + series_frame_ordering(&frame_schema) .unwrap_or_else(|error| panic!("{}: {error}", case.name)), expected, "{}", case.name ), None => assert!( - raw_series_frame_ordering(&frame_schema).is_err(), + series_frame_ordering(&frame_schema).is_err(), "{}", case.name ), } + let declared = datatypes::series::frame::SeriesFrameDeclaration::parse(&frame_schema); + match case.declaration.as_str() { + "accepted" => { + let declared = declared.unwrap_or_else(|error| panic!("{}: {error}", case.name)); + if evaluated { + assert_eq!( + declared.content(), + datatypes::series::frame::FrameContent::Evaluated, + "{}", + case.name + ); + } + } + "rejected" => assert!( + declared.is_err(), + "{}: the declaration must be refused", + case.name + ), + other => panic!("{} has invalid fixture declaration {other}", case.name), + } } } @@ -2463,14 +2580,14 @@ async fn pack_declares_ordering_on_every_batch_and_ipc_roundtrips_it() { // replaced single-row packing emitted one batch per chunk. let expected_batch_count = if is_empty { 0 } else { 1 }; let canonical_schema = - raw_series_frame_schema(&schema(), RawLabelMode::Explicit).unwrap(); + raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap(); let mut expected_metadata = canonical_schema.metadata().clone(); expected_metadata.insert( "greptime.series.ordering".to_string(), match ordering { SeriesOrdering::Contiguous => "contiguous", SeriesOrdering::KeyThenTime => "read_key_then_time", - // The v1 wire carries exactly these two spellings, so the loop above + // The wire carries exactly these two spellings, so the loop above // admits only the two representable modes; `Interleaved` is rejected // before any poll (see `interleaved_ordering_is_rejected_before_any_poll`). SeriesOrdering::Interleaved => { @@ -2484,7 +2601,7 @@ async fn pack_declares_ordering_on_every_batch_and_ipc_roundtrips_it() { let declared = packed.schema(); assert_eq!(declared.fields(), canonical_schema.fields()); assert_eq!(declared.metadata(), &expected_metadata); - assert_eq!(raw_series_frame_ordering(&declared).unwrap(), ordering); + assert_eq!(series_frame_ordering(&declared).unwrap(), ordering); let mut batches = Vec::new(); while let Some(batch) = packed.next().await { let batch = batch.unwrap(); @@ -2502,10 +2619,7 @@ async fn pack_declares_ordering_on_every_batch_and_ipc_roundtrips_it() { } let reader = StreamReader::try_new(Cursor::new(ipc), None).unwrap(); assert_eq!(reader.schema(), declared); - assert_eq!( - raw_series_frame_ordering(&reader.schema()).unwrap(), - ordering - ); + assert_eq!(series_frame_ordering(&reader.schema()).unwrap(), ordering); let ipc_batches = reader.map(Result::unwrap).collect::>(); assert_eq!(ipc_batches.len(), batches.len()); assert_eq!(ipc_batches.len(), expected_batch_count); @@ -2621,17 +2735,33 @@ fn two_tag_binding(metric: &str) -> RawLabelSource { } } -/// A key frame whose metadata carries exactly the supplied description and nothing else. -fn key_frame_with(description: &datatypes::series::frame::LabelDecodeSpec) -> SchemaRef { - let layout = raw_series_frame_schema(&schema(), RawLabelMode::Key).unwrap(); +/// A raw key frame declaring the supplied description, as a producer with a primary-key binding +/// writes it. +fn key_layout() -> SchemaRef { + let description = + label_decode_spec(&key_binding("cpu")).expect("a primary-key binding describes its key"); + series_frame_schema(&schema(), FrameLabelMode::Key, Some(&description)).unwrap() +} + +/// A key frame whose metadata carries exactly the supplied description text and nothing else. +/// +/// The text is written verbatim, not canonicalized, so a case can pin what a writer must have +/// ordered and what a reader refuses to reorder. +fn key_frame_with_encoded(encoded: &str) -> SchemaRef { + let layout = key_layout(); let mut metadata = layout.metadata().clone(); metadata.insert( "greptime.series.label_decode".to_string(), - serde_json::to_string(description).unwrap(), + encoded.to_string(), ); Arc::new(Schema::new_with_metadata(layout.fields().clone(), metadata)) } +/// A key frame whose `greptime.series.label_decode` entry is the supplied description. +fn key_frame_with(description: &datatypes::series::frame::LabelDecodeSpec) -> SchemaRef { + key_frame_with_encoded(&serde_json::to_string(description).unwrap()) +} + /// Logical tag order is fixed by the writer, so the wire form has one spelling and the comparison /// is plain equality. The ordered key columns are the encoding, so the same columns in another /// order are a different layout and are rejected. @@ -2720,7 +2850,7 @@ async fn frame_check_rejects_a_binding_the_frame_does_not_describe() { "an explicit identity expects a frame that carries complete labels" ); - let explicit = raw_series_frame_schema(&schema(), RawLabelMode::Explicit).unwrap(); + let explicit = raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap(); assert!( ensure_frame_matches_label_source(explicit.as_ref(), &key_binding("cpu")).is_err(), "a frame declaring no layout cannot satisfy a primary-key identity" @@ -2746,7 +2876,7 @@ async fn unpack_retains_the_declared_description_without_a_binding() { "no physical region key layout is invented for a frame that carries none" ); - let explicit = raw_series_frame_schema(&schema(), RawLabelMode::Explicit).unwrap(); + let explicit = raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap(); let (source, _, _) = df_source(vec![], explicit); let unpacked = unpack_raw_series(source, None).unwrap(); assert_eq!( @@ -2756,17 +2886,6 @@ async fn unpack_retains_the_declared_description_without_a_binding() { ); } -/// A key frame whose `greptime.series.label_decode` entry is exactly the supplied text. -fn key_frame_with_encoded(encoded: &str) -> SchemaRef { - let layout = raw_series_frame_schema(&schema(), RawLabelMode::Key).unwrap(); - let mut metadata = layout.metadata().clone(); - metadata.insert( - "greptime.series.label_decode".to_string(), - encoded.to_string(), - ); - Arc::new(Schema::new_with_metadata(layout.fields().clone(), metadata)) -} - /// The description is validated whether or not a binding is supplied, so trusting the frame never /// means skipping validation: an unknown version, an unknown field, a malformed entry, a logical /// tag outside the key, a key frame without a description, and an explicit frame carrying one are @@ -2810,14 +2929,21 @@ async fn unpack_rejects_invalid_descriptions_without_a_binding() { ); } - let key_without_description = raw_series_frame_schema(&schema(), RawLabelMode::Key).unwrap(); + // A key frame nobody can decode: the description is removed from the frame the producer wrote. + // Construction cannot express this, which is the point - the reader is what refuses it. + let mut without_description = key_layout().metadata().clone(); + without_description.remove("greptime.series.label_decode"); + let key_without_description = Arc::new(Schema::new_with_metadata( + key_layout().fields().clone(), + without_description, + )); let (source, _, _) = df_source(vec![], key_without_description); assert!( unpack_raw_series(source, None).is_err(), "a key frame nobody can decode is rejected instead of being trusted" ); - let explicit = raw_series_frame_schema(&schema(), RawLabelMode::Explicit).unwrap(); + let explicit = raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap(); let mut metadata = explicit.metadata().clone(); metadata.insert( "greptime.series.label_decode".to_string(), @@ -2850,14 +2976,16 @@ async fn pack_declares_the_label_decode_description_of_its_binding() { ); let mut packed = pack_raw_series(input).unwrap(); let declared = packed.schema(); - let canonical = raw_series_frame_schema(&schema(), RawLabelMode::Key).unwrap(); + // A key frame's canonical layout already carries its description, so the producer adds only the + // ordering on top of it. + let canonical = key_layout(); assert_eq!( declared.metadata().len(), - canonical.metadata().len() + 2, - "a key frame adds the producer ordering and the label decode description" + canonical.metadata().len() + 1, + "a key frame adds the producer ordering to the description it must declare" ); assert_eq!( - datatypes::series::frame::raw_series_frame_label_decode(&declared).unwrap(), + datatypes::series::frame::series_frame_label_decode(&declared).unwrap(), label_decode_spec(&binding), "the declared description is derived from the binding, not from the first row" ); @@ -2872,11 +3000,16 @@ async fn pack_declares_the_label_decode_description_of_its_binding() { let packed = pack_raw_series(input).unwrap(); let declared = packed.schema(); assert_eq!( - datatypes::series::frame::raw_series_frame_label_decode(&declared).unwrap(), + datatypes::series::frame::series_frame_label_decode(&declared).unwrap(), None, "explicit labels are already complete" ); - assert_eq!(declared.metadata().len(), canonical.metadata().len() + 1); + let canonical_explicit = raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap(); + assert_eq!( + declared.metadata().len(), + canonical_explicit.metadata().len() + 1, + "an explicit frame adds only the producer ordering" + ); } /// A key frame that omits its description is not decodable, and unpacking says so instead of falling @@ -2899,13 +3032,11 @@ async fn unpack_round_trips_a_key_frame_and_rejects_one_without_a_description() let chunk = decoded.next().await.unwrap().unwrap(); assert_eq!(chunk.primary_key.as_slice(), b"encoded-complete-key"); - // A frame that omits the description is not decodable from the schema alone. - let canonical = raw_series_frame_schema(&schema(), RawLabelMode::Key).unwrap(); - let mut metadata = canonical.metadata().clone(); - metadata.insert( - "greptime.series.ordering".to_string(), - "contiguous".to_string(), - ); + // A frame that omits the description is not decodable from the schema alone. The producer + // cannot write one, so the entry is dropped from the frame it did write: the reader is what has + // to refuse it. + let mut metadata = declared.metadata().clone(); + metadata.remove("greptime.series.label_decode"); let stripped = Arc::new(datatypes::arrow::datatypes::Schema::new_with_metadata( declared.fields().clone(), metadata, @@ -2933,7 +3064,7 @@ async fn unpack_reports_the_ordering_the_frame_declares() { ); let packed = pack_raw_series(input).unwrap(); assert_eq!( - raw_series_frame_ordering(packed.schema().as_ref()).unwrap(), + series_frame_ordering(packed.schema().as_ref()).unwrap(), SeriesOrdering::KeyThenTime, "the producer declaration reaches the wire" ); @@ -2946,7 +3077,7 @@ async fn unpack_reports_the_ordering_the_frame_declares() { ); // A frame that declares no ordering is a legacy contiguous frame. - let frame = raw_series_frame_schema(&schema(), RawLabelMode::Explicit).unwrap(); + let frame = raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap(); let (source, _, _) = df_source(vec![], frame); assert_eq!( unpack_raw_series(source, None).unwrap().ordering(), @@ -2955,17 +3086,19 @@ async fn unpack_reports_the_ordering_the_frame_declares() { } /// Reserved layout metadata this revision does not write is rejected instead of being read as the -/// current layout, so a frame cannot be decoded under a contract it did not declare. +/// current layout, so a frame cannot be decoded under a contract it did not declare. The current +/// layout is `v2`; a `v1` frame and every unknown value are refused alike. #[tokio::test] async fn unpack_rejects_unknown_layout_metadata() { let cases = [ - ("greptime.series.layout", "v2"), + ("greptime.series.layout", "v1"), + ("greptime.series.layout", "v3"), ("greptime.series.content", "series"), ("greptime.series.time", "timestamps_v2"), ("greptime.series.ordering", "hash_then_time"), ]; for (key, value) in cases { - let frame = raw_series_frame_schema(&schema(), RawLabelMode::Explicit).unwrap(); + let frame = raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap(); let mut metadata = frame.metadata().clone(); metadata.insert(key.to_string(), value.to_string()); let frame = Arc::new(Schema::new_with_metadata(frame.fields().clone(), metadata)); @@ -2982,7 +3115,7 @@ async fn unpack_rejects_unknown_layout_metadata() { /// same frame unpacks without it because the frame is the only source of decode information. #[tokio::test] async fn unpack_rejects_an_expected_schema_the_frame_does_not_declare() { - let frame = raw_series_frame_schema(&schema(), RawLabelMode::Explicit).unwrap(); + let frame = raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap(); let other_field = Arc::new(SeriesSchema { fields: vec![FieldSpec { @@ -3049,7 +3182,7 @@ async fn unpack_pack_unpack_preserves_the_description_and_the_ordering() { let (input, _, _) = raw_custom(vec![key_chunk()], SeriesOrdering::KeyThenTime, binding); let packed = pack_raw_series(input).unwrap(); assert_eq!( - datatypes::series::frame::raw_series_frame_label_decode(packed.schema().as_ref()).unwrap(), + datatypes::series::frame::series_frame_label_decode(packed.schema().as_ref()).unwrap(), Some(description.clone()) ); @@ -3059,13 +3192,12 @@ async fn unpack_pack_unpack_preserves_the_description_and_the_ordering() { let repacked = pack_raw_series(unpacked).unwrap(); assert_eq!( - datatypes::series::frame::raw_series_frame_label_decode(repacked.schema().as_ref()) - .unwrap(), + datatypes::series::frame::series_frame_label_decode(repacked.schema().as_ref()).unwrap(), Some(description.clone()), "the repacked frame declares what the unpacked stream declared" ); assert_eq!( - raw_series_frame_ordering(repacked.schema().as_ref()).unwrap(), + series_frame_ordering(repacked.schema().as_ref()).unwrap(), SeriesOrdering::KeyThenTime, "the repacked frame keeps the ordering it was unpacked under" ); @@ -3078,3 +3210,42 @@ async fn unpack_pack_unpack_preserves_the_description_and_the_ordering() { assert!(chunk.metric.is_none(), "key chunks stay undecoded"); assert!(again.next().await.is_none()); } + +/// Each typed schema entry point refuses the content it cannot describe. +/// +/// The raw entry point is handed an evaluated series schema and the shared builder is handed an +/// evaluated schema with a key-then-time ordering. Both would otherwise produce a frame whose +/// declaration contradicts its own columns, and the reader would be the first to notice — after the +/// producer had already published it. +#[test] +fn typed_schema_entry_points_refuse_the_content_they_cannot_describe() { + assert!( + super::raw_series_frame_schema_with_ordering( + &evaluated_schema(), + &RawLabelSource::Explicit, + SeriesOrdering::Contiguous, + ) + .is_err(), + "a raw frame cannot carry a schema with an evaluation grid" + ); + assert!( + super::frame_schema_with( + &evaluated_schema(), + FrameLabelMode::Explicit, + None, + SeriesOrdering::KeyThenTime, + ) + .is_err(), + "an evaluated frame has no storage key to be ordered by" + ); + assert!( + super::frame_schema_with( + &evaluated_schema(), + FrameLabelMode::Explicit, + None, + SeriesOrdering::Contiguous, + ) + .is_ok(), + "contiguous is the one ordering an evaluated frame can honour" + ); +} diff --git a/src/common/query/src/series_frame/tests/evaluated.rs b/src/common/query/src/series_frame/tests/evaluated.rs new file mode 100644 index 00000000000..5a2336a4707 --- /dev/null +++ b/src/common/query/src/series_frame/tests/evaluated.rs @@ -0,0 +1,1250 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Streaming round trips through an evaluated series frame. +//! +//! An evaluated frame names its series by their complete labels and addresses time on the grid its +//! schema declares, so every test here drives the real path - pack, a record batch stream, unpack - +//! and compares what a consumer receives against what a producer sent. A schema snapshot would +//! prove the declaration; these prove the values. +//! +//! A frame row is one run of positions on the shared grid, so unpacking yields one [`Series`] per +//! row rather than one per source chunk. A consumer rebuilds a chunk chain by concatenating rows up +//! to the one that carries `is_last`, which is what these tests do. + +use std::io::Cursor; +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use common_error::ext::{BoxedError, ErrorExt}; +use common_error::status_code::StatusCode; +use common_recordbatch::DfSendableRecordBatchStream; +use datatypes::arrow::array::{ + Array, BooleanArray, Float64Array, ListArray, StructArray, UInt32Array, UInt64Array, +}; +use datatypes::arrow::buffer::OffsetBuffer; +use datatypes::arrow::datatypes::{DataType, SchemaRef}; +use datatypes::arrow::ipc::{reader::StreamReader, writer::StreamWriter}; +use datatypes::arrow::record_batch::{RecordBatch, RecordBatchOptions}; +use datatypes::series::frame::{FrameLabelMode, SeriesFrameBuilder, SeriesFrameCursor}; +use datatypes::series::histogram::{ + BucketSideView, CounterResetHint, HistogramBatch, HistogramBatchBuilder, HistogramView, +}; +use datatypes::series::samples::{ + DenseSamplesRef, FieldSpec, SampleRef, Samples, Series, SeriesFields, SeriesSchema, +}; +use datatypes::series::time::{GridSpan, TimeAxis}; +use datatypes::series::{Label, SeriesLabels, SeriesOrdering}; +use futures_util::StreamExt; + +use super::super::{ + SendableEvaluatedSeriesStream, pack_evaluated_series, series_frame_ordering, + unpack_evaluated_series, unpack_raw_series, +}; +use super::{df_source, evaluated_grid, evaluated_schema, schema}; + +/// Complete labels of the `cpu` metric on one host. +fn labels(host: &str) -> SeriesLabels { + SeriesLabels::try_new( + Some(Arc::from("cpu")), + vec![Label { + name: Arc::from("host"), + value: Arc::from(host), + }], + ) + .unwrap() +} + +/// One validated evaluated chunk: `values` on the runs `spans` of the shared test grid. +fn series(spans: &[(usize, usize)], values: Vec>, metric: SeriesLabels) -> Series { + let time = TimeAxis::from_grid_spans( + evaluated_grid(), + spans + .iter() + .map(|(start, len)| GridSpan::new(*start, *len)) + .collect(), + ) + .unwrap(); + let fields = SeriesFields::one(Samples::from_float(Float64Array::from(values))); + Series::try_new(&evaluated_schema(), metric, time, fields).unwrap() +} + +/// An evaluated producer stream over the supplied chunks and their finality. +fn source(items: Vec<(Series, bool)>) -> SendableEvaluatedSeriesStream { + Box::pin(futures_util::stream::iter( + items.into_iter().map(Ok::<_, BoxedError>), + )) +} + +/// Packs the supplied chunks, keeping the stream so a test can observe its errors. +fn pack(items: Vec<(Series, bool)>) -> DfSendableRecordBatchStream { + pack_evaluated_series(source(items), evaluated_schema()).unwrap() +} + +/// Packs the supplied chunks and drains every ready batch. +async fn packed(items: Vec<(Series, bool)>) -> Vec { + let mut stream = pack(items); + let mut out = Vec::new(); + while let Some(batch) = stream.next().await { + out.push(batch.unwrap()); + } + out +} + +/// Unpacks the batches a producer published, under the schema they declare. +fn unpack(batches: Vec, declared: SchemaRef) -> SendableEvaluatedSeriesStream { + let (source, _, _) = df_source(batches, declared); + unpack_evaluated_series(source, Some(evaluated_schema())).unwrap() +} + +/// Drains one unpacked stream, so a test can assert on the whole sequence at once. +async fn drain(mut stream: SendableEvaluatedSeriesStream) -> Vec<(Series, bool)> { + let mut out = Vec::new(); + while let Some(chunk) = stream.next().await { + out.push(chunk.unwrap()); + } + out +} + +/// The absolute grid positions a decoded chunk carries, in order. +fn positions(series: &Series) -> Vec { + series + .time + .positions() + .map(|(position, _)| position) + .collect() +} + +/// The float payload of the one `value` field, in compact-row order. +fn values(series: &Series) -> Vec> { + series + .fields + .as_slice() + .first() + .expect("the fixture has one value field") + .iter() + .map(|sample| match sample { + SampleRef::Missing => None, + SampleRef::Float(value) => Some(value), + _ => panic!("the fixture carries float samples"), + }) + .collect() +} + +/// The positions and values of every decoded row, concatenated in stream order. +fn flat(rows: &[(Series, bool)]) -> (Vec, Vec>) { + let mut all_positions = Vec::new(); + let mut all_values = Vec::new(); + for (row, _) in rows { + all_positions.extend(positions(row)); + all_values.extend(values(row)); + } + (all_positions, all_values) +} + +/// Rewrites the `is_last` column of a frame, so a test can produce a row a producer refuses to write +/// and check that the reader refuses it too. +fn with_is_last(batch: &RecordBatch, is_last: bool) -> RecordBatch { + let mut columns = batch.columns().to_vec(); + columns[1] = Arc::new(BooleanArray::from(vec![is_last; batch.num_rows()])); + RecordBatch::try_new(batch.schema(), columns).unwrap() +} + +/// Rewrites the `read_key` column of a frame, so a test can attach the storage key an evaluated row +/// must not carry. +fn with_read_key(batch: &RecordBatch, key: &[u8]) -> RecordBatch { + let keys = vec![Some(key); batch.num_rows()]; + let mut columns = batch.columns().to_vec(); + columns[0] = Arc::new(datatypes::arrow::array::BinaryArray::from(keys)); + RecordBatch::try_new(batch.schema(), columns).unwrap() +} + +/// The frame schema a producer's batches carry, with one metadata entry changed the way a divergent +/// producer would write it. +fn relabelled(declared: &SchemaRef, key: &str, value: Option<&str>) -> SchemaRef { + let mut metadata = declared.metadata().clone(); + match value { + Some(value) => metadata.insert(key.to_string(), value.to_string()), + None => metadata.remove(key), + }; + Arc::new(datatypes::arrow::datatypes::Schema::new_with_metadata( + declared.fields().clone(), + metadata, + )) +} + +/// One dense chunk keeps every position, label, and value, and its finality. +#[tokio::test] +async fn dense_roundtrip_preserves_positions_labels_and_values() { + let metric = labels("a"); + let expected: Vec> = (0..8) + .map(|row| (row % 3 != 0).then_some(row as f64)) + .collect(); + let batches = packed(vec![( + series(&[(0, 8)], expected.clone(), metric.clone()), + true, + )]) + .await; + assert_eq!(batches.len(), 1, "eight positions are one row"); + assert_eq!(batches[0].num_rows(), 1); + + let declared = batches[0].schema(); + assert_eq!( + series_frame_ordering(declared.as_ref()).unwrap(), + SeriesOrdering::Contiguous, + "an evaluated frame declares contiguous ordering" + ); + let declaration = datatypes::series::frame::SeriesFrameDeclaration::parse(declared.as_ref()) + .expect("an evaluated frame declares itself"); + assert_eq!( + declaration.content(), + datatypes::series::frame::FrameContent::Evaluated + ); + assert_eq!( + declaration.labels(), + datatypes::series::frame::FrameLabelMode::Explicit + ); + assert_eq!( + declared + .metadata() + .get("greptime.series.time") + .map(String::as_str), + Some("grid_spans"), + "an evaluated frame addresses time on its grid" + ); + assert!(declaration.label_decode().is_none(), "no key is decoded"); + + let rows = drain(unpack(batches, declared)).await; + assert_eq!(rows.len(), 1); + let (chunk, is_last) = &rows[0]; + assert!(is_last, "the source closed the chain"); + assert_eq!(chunk.metric, metric, "labels round trip"); + assert_eq!(positions(chunk), (0..8).collect::>()); + assert_eq!(values(chunk), expected, "every value, including nulls"); + assert!( + chunk + .time + .grid() + .is_some_and(|grid| grid.spec() == evaluated_grid().spec()), + "the decoded chunk is addressed on the frame's own grid" + ); +} + +/// A sparse chunk keeps its holes: the decoded positions are exactly the ones the producer sent, and +/// every field stays aligned with them. +#[tokio::test] +async fn sparse_roundtrip_keeps_holes_and_field_alignment() { + let metric = labels("b"); + let expected = vec![Some(1.0), None, Some(3.0), Some(4.0), None]; + let batches = packed(vec![( + series(&[(0, 2), (5, 3)], expected.clone(), metric.clone()), + true, + )]) + .await; + assert_eq!(batches.len(), 1); + let declared = batches[0].schema(); + let rows = drain(unpack(batches, declared)).await; + + let (decoded_positions, decoded_values) = flat(&rows); + assert_eq!( + decoded_positions, + vec![0, 1, 5, 6, 7], + "the hole between the runs is not filled" + ); + assert_eq!( + decoded_values, expected, + "values stay aligned to positions, hole included" + ); + for (row, _) in &rows { + assert_eq!(row.metric, metric); + for hole in [2, 3, 4] { + assert!( + row.time.row_of(hole).is_none(), + "no row exists at a position the chunk does not hold" + ); + } + } + assert!( + rows.last().expect("at least one row").1, + "only the last row of the chunk closes the chain" + ); + assert_eq!( + rows.iter().filter(|(_, is_last)| *is_last).count(), + 1, + "one chain, one close" + ); +} + +/// A chunk with no positions that closes its chain is one frame row, not a dropped chunk: a consumer +/// has to see the finality it carries. +#[tokio::test] +async fn zero_sample_closing_chunk_is_kept_as_one_row() { + let metric = labels("c"); + let batches = packed(vec![ + ( + series(&[(3, 2)], vec![Some(1.0), Some(2.0)], metric.clone()), + false, + ), + (series(&[], vec![], metric.clone()), true), + ]) + .await; + assert_eq!(batches.len(), 1); + assert_eq!( + batches[0].num_rows(), + 2, + "the closing chunk keeps its own row" + ); + + let declared = batches[0].schema(); + let rows = drain(unpack(batches, declared)).await; + assert_eq!(rows.len(), 2); + let (carried, carried_is_last) = &rows[0]; + assert!(!carried_is_last, "the first chunk stays open"); + assert_eq!(positions(carried), vec![3, 4]); + assert_eq!(values(carried), vec![Some(1.0), Some(2.0)]); + let (closing, closing_is_last) = &rows[1]; + assert!(closing_is_last, "the closing chunk keeps its finality"); + assert_eq!( + positions(closing), + Vec::::new(), + "and carries no position" + ); + assert_eq!(values(closing), Vec::>::new()); + assert_eq!(closing.metric, metric, "labels survive on a closing chunk"); +} + +/// One grid position per run, so a chunk with `count` positions starting at `start` becomes `count` +/// frame rows: one row per run, which is what a batch bound can split. +fn single_position_spans(start: usize, count: usize) -> Vec<(usize, usize)> { + (0..count).map(|offset| (start + offset, 1)).collect() +} + +/// Rows split across batches keep their order and finality: every position appears once and in +/// order, and only the last row of the source's final chunk closes the chain. +#[tokio::test] +async fn multi_chunk_multi_batch_stream_preserves_order_and_finality() { + let metric = labels("d"); + let first: Vec> = (0..300).map(|row| Some(row as f64)).collect(); + let second: Vec> = (0..10).map(|row| Some(row as f64)).collect(); + let batches = packed(vec![ + ( + series(&single_position_spans(0, 300), first, metric.clone()), + false, + ), + ( + series(&single_position_spans(300, 10), second, metric.clone()), + true, + ), + ]) + .await; + assert!( + batches.len() > 1, + "the default row bound splits 310 rows into several batches" + ); + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + 310 + ); + + let declared = batches[0].schema(); + assert!(batches.iter().all(|batch| batch.schema() == declared)); + let rows = drain(unpack(batches, declared)).await; + let (decoded_positions, _) = flat(&rows); + assert_eq!( + decoded_positions, + (0..310).collect::>(), + "no position is repeated, reordered, or dropped across batches" + ); + assert!(rows.iter().all(|(row, _)| row.metric == metric)); + let closes = rows + .iter() + .enumerate() + .filter(|(_, (_, is_last))| *is_last) + .map(|(index, _)| index) + .collect::>(); + assert_eq!( + closes, + vec![rows.len() - 1], + "exactly the last row closes the chain, inherited from the source's final chunk" + ); +} + +/// Two chains whose labels happen to be equal are two chains: each closes itself, and the second is +/// not mistaken for a reopened series. +#[tokio::test] +async fn consecutive_chains_with_equal_labels_each_close_independently() { + let metric = labels("e"); + let batches = packed(vec![ + ( + series(&[(0, 2)], vec![Some(1.0), Some(2.0)], metric.clone()), + true, + ), + ( + series(&[(2, 2)], vec![Some(3.0), Some(4.0)], metric.clone()), + true, + ), + ]) + .await; + + let declared = batches[0].schema(); + let rows = drain(unpack(batches, declared)).await; + assert_eq!(rows.len(), 2, "one row per source chunk"); + assert!( + rows.iter().all(|(_, is_last)| *is_last), + "each chain closes itself, so neither is refused as a reopened series" + ); + assert!(rows.iter().all(|(row, _)| row.metric == metric)); + assert_eq!(positions(&rows[0].0), vec![0, 1]); + assert_eq!(values(&rows[0].0), vec![Some(1.0), Some(2.0)]); + assert_eq!(positions(&rows[1].0), vec![2, 3]); + assert_eq!(values(&rows[1].0), vec![Some(3.0), Some(4.0)]); +} + +/// A chain still open at EOF is an error on both sides: a producer that never closed a series cannot +/// finish, and a receiver cannot pretend the stream ended cleanly. +#[tokio::test] +async fn eof_with_an_open_chain_is_an_error() { + let metric = labels("f"); + let mut stream = pack(vec![( + series(&[(0, 2)], vec![Some(1.0), Some(2.0)], metric.clone()), + false, + )]); + // The rows are buffered until EOF, where the missing close is reported instead of a batch. + let error = loop { + match stream.next().await { + Some(Ok(_)) => continue, + Some(Err(error)) => break error, + None => panic!("an open chain at EOF must be reported"), + } + }; + assert_eq!( + crate::error::datafusion_status_code::(&error, None), + StatusCode::InvalidArguments + ); + assert!(stream.next().await.is_none(), "the error is reported once"); + + // The receiving side reaches the same conclusion from the frame alone. + let closing = packed(vec![( + series(&[(0, 2)], vec![Some(1.0), Some(2.0)], metric), + true, + )]) + .await; + let declared = closing[0].schema(); + let opened = with_is_last(&closing[0], false); + let (source, _, _) = df_source(vec![opened], declared); + let mut unpacked = unpack_evaluated_series(source, Some(evaluated_schema())).unwrap(); + assert!(!unpacked.next().await.unwrap().unwrap().1); + let error = unpacked.next().await.unwrap().unwrap_err(); + assert_eq!(error.status_code(), StatusCode::InvalidArguments); + assert!(unpacked.next().await.is_none()); +} + +/// A chunk with no position that does not close its chain can never make progress, so both sides +/// refuse it. +#[tokio::test] +async fn zero_sample_chunk_that_does_not_close_is_rejected() { + let metric = labels("g"); + let mut stream = pack(vec![(series(&[], vec![], metric.clone()), false)]); + let error = stream.next().await.unwrap().unwrap_err(); + assert_eq!( + crate::error::datafusion_status_code::(&error, None), + StatusCode::InvalidArguments + ); + assert!(stream.next().await.is_none()); + + let closing = packed(vec![(series(&[], vec![], metric), true)]).await; + assert_eq!(closing[0].num_rows(), 1); + let declared = closing[0].schema(); + let not_closing = with_is_last(&closing[0], false); + let (source, _, _) = df_source(vec![not_closing], declared); + let mut unpacked = unpack_evaluated_series(source, Some(evaluated_schema())).unwrap(); + let error = unpacked.next().await.unwrap().unwrap_err(); + assert_eq!(error.status_code(), StatusCode::InvalidArguments); + assert!(unpacked.next().await.is_none()); +} + +/// Computed values have no storage key to be ordered by, so an evaluated frame that declares any +/// other ordering, or a spelling the wire does not define, is refused before the stream is returned. +#[tokio::test] +async fn evaluated_frame_rejects_non_contiguous_ordering() { + let batches = packed(vec![( + series(&[(0, 1)], vec![Some(1.0)], labels("h")), + true, + )]) + .await; + let declared = batches[0].schema(); + for ordering in ["read_key_then_time", "interleaved"] { + let schema = relabelled(&declared, "greptime.series.ordering", Some(ordering)); + let (source, polls, _) = df_source(vec![], schema); + assert!( + unpack_evaluated_series(source, Some(evaluated_schema())).is_err(), + "{ordering} is not an ordering computed values can claim" + ); + assert_eq!(polls.load(Ordering::SeqCst), 0); + } +} + +/// Every contradictory or unreadable declaration is refused instead of decoded: a missing or +/// malformed grid, a time mode that contradicts the content, and keyed identity on computed values. +#[tokio::test] +async fn evaluated_frame_rejects_contradictory_declarations() { + let batches = packed(vec![( + series(&[(0, 1)], vec![Some(1.0)], labels("i")), + true, + )]) + .await; + let declared = batches[0].schema(); + let cases: [(&str, SchemaRef); 6] = [ + ( + "no grid", + relabelled(&declared, "greptime.series.grid", None), + ), + ( + "malformed grid", + relabelled(&declared, "greptime.series.grid", Some("grid")), + ), + ( + "incomplete grid", + relabelled(&declared, "greptime.series.grid", Some(r#"{"start_ms":0}"#)), + ), + ( + "evaluated with timestamps", + relabelled(&declared, "greptime.series.time", Some("timestamps")), + ), + ( + "evaluated with key labels", + relabelled(&declared, "greptime.series.labels", Some("key")), + ), + ( + "raw content", + relabelled(&declared, "greptime.series.content", Some("raw")), + ), + ]; + for (label, schema) in cases { + let (source, polls, _) = df_source(vec![], schema); + assert!( + unpack_evaluated_series(source, None).is_err(), + "{label}: must be rejected" + ); + assert_eq!( + polls.load(Ordering::SeqCst), + 0, + "{label}: a rejected frame is never polled" + ); + } + + // `expected` is a check, never an input: a schema without the frame's grid is a disagreement. + let (source, _, _) = df_source(vec![], declared); + assert!( + unpack_evaluated_series(source, Some(schema())).is_err(), + "a raw expectation cannot decode an evaluated frame" + ); +} + +/// The two contents are not interchangeable: each unpack entry point refuses the other's frame. +#[tokio::test] +async fn each_unpack_entry_point_refuses_the_other_content() { + let evaluated = packed(vec![( + series(&[(0, 1)], vec![Some(1.0)], labels("j")), + true, + )]) + .await; + let declared = evaluated[0].schema(); + let (source, _, _) = df_source(evaluated, declared); + assert!( + unpack_raw_series(source, None).is_err(), + "a raw reader cannot decode grid-addressed rows" + ); + + let raw_frame = datatypes::series::frame::raw_series_frame_schema( + &schema(), + datatypes::series::frame::FrameLabelMode::Explicit, + ) + .unwrap(); + let (source, _, _) = df_source(vec![], raw_frame); + assert!( + unpack_evaluated_series(source, None).is_err(), + "an evaluated reader cannot decode timestamp-addressed rows" + ); +} + +/// An evaluated row names its series by label, so a row that also carries a storage key is a +/// contradiction: the reader refuses it instead of decoding an identity it cannot check. +#[tokio::test] +async fn evaluated_frame_rejects_a_storage_key_on_a_row() { + let batches = packed(vec![( + series(&[(0, 2)], vec![Some(1.0), Some(2.0)], labels("l")), + true, + )]) + .await; + let declared = batches[0].schema(); + let keyed = with_read_key(&batches[0], b"encoded-key"); + let (source, _, _) = df_source(vec![keyed], declared); + let mut unpacked = unpack_evaluated_series(source, Some(evaluated_schema())).unwrap(); + let error = unpacked.next().await.unwrap().unwrap_err(); + assert_eq!(error.status_code(), StatusCode::InvalidArguments); + assert!(unpacked.next().await.is_none()); +} + +/// An evaluated frame is self-describing across Arrow IPC: the schema, its metadata, and its rows +/// survive serialization, and a consumer decodes them with no planning-side copy of the grid. +#[tokio::test] +async fn evaluated_frame_roundtrips_through_arrow_ipc() { + let metric = labels("k"); + let expected: Vec> = vec![Some(1.5), None, Some(3.5)]; + let batches = packed(vec![( + series(&[(0, 2), (4, 1)], expected.clone(), metric.clone()), + true, + )]) + .await; + let declared = batches[0].schema(); + + let mut ipc = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut ipc, &declared).unwrap(); + for batch in &batches { + writer.write(batch).unwrap(); + } + writer.finish().unwrap(); + } + let reader = StreamReader::try_new(Cursor::new(ipc), None).unwrap(); + let read_schema = reader.schema(); + assert_eq!(read_schema, declared); + assert_eq!( + datatypes::series::frame::SeriesFrameDeclaration::parse(read_schema.as_ref()) + .unwrap() + .series() + .eval_grid + .as_ref() + .expect("the frame still declares its grid") + .spec(), + evaluated_grid().spec() + ); + + let read: Vec = reader.map(Result::unwrap).collect(); + assert_eq!(read.len(), batches.len()); + let (source, _, _) = df_source(read, read_schema); + // No `expected` schema is supplied: the frame alone says what its rows mean. + let rows = drain(unpack_evaluated_series(source, None).unwrap()).await; + let (decoded_positions, decoded_values) = flat(&rows); + assert_eq!(decoded_positions, vec![0, 1, 4]); + assert_eq!(decoded_values, expected); + assert!(rows.iter().all(|(row, _)| row.metric == metric)); + assert!(rows.last().expect("at least one row").1); +} + +/// A schema with two value fields addressed on [`evaluated_grid`], so a fixture can carry more than +/// the single float field [`series`] declares. +fn pair_schema() -> Arc { + Arc::new(SeriesSchema { + fields: vec![ + FieldSpec { + name: Arc::from("value"), + }, + FieldSpec { + name: Arc::from("score"), + }, + ] + .into(), + eval_grid: Some(evaluated_grid()), + }) +} + +/// One validated evaluated chunk: `fields` on the runs `spans` of the shared test grid, declared +/// under `schema`. +fn chunk_on( + schema: &SeriesSchema, + spans: &[(usize, usize)], + fields: SeriesFields, + metric: SeriesLabels, +) -> Series { + let time = TimeAxis::from_grid_spans( + evaluated_grid(), + spans + .iter() + .map(|(start, len)| GridSpan::new(*start, *len)) + .collect(), + ) + .unwrap(); + Series::try_new(schema, metric, time, fields).unwrap() +} + +/// Packs the supplied chunks under `schema` and drains every ready batch, for a fixture whose field +/// list is not the single `value` field [`evaluated_schema`] declares. +async fn packed_under(schema: Arc, items: Vec<(Series, bool)>) -> Vec { + let mut stream = pack_evaluated_series(source(items), schema).unwrap(); + let mut out = Vec::new(); + while let Some(batch) = stream.next().await { + out.push(batch.unwrap()); + } + out +} + +/// Unpacks a producer's batches as a consumer that planned the same series schema would, which is +/// how a fixture with more than the one `value` field is read back. +fn unpack_as( + batches: Vec, + declared: SchemaRef, + expected: Arc, +) -> SendableEvaluatedSeriesStream { + let (source, _, _) = df_source(batches, declared); + unpack_evaluated_series(source, Some(expected)).unwrap() +} + +/// The float payload of field `index`, in compact-row order. +fn field_values(series: &Series, index: usize) -> Vec> { + series + .fields + .get(index) + .expect("the fixture reads a field it declared") + .iter() + .map(|sample| match sample { + SampleRef::Missing => None, + SampleRef::Float(value) => Some(value), + _ => panic!("the fixture carries float samples"), + }) + .collect() +} + +/// Compares two per-position float payloads, with `NaN` comparing equal to itself. +/// +/// `Option` cannot be compared with `==` here: the fixtures carry `NaN` as a *value*, and +/// `NaN != NaN`, so equality would report a faithful round trip as a mismatch. Comparing bit +/// patterns keeps a value distinct from a missing position, and `None` still has to meet `None`. +fn assert_float_payload_eq(actual: &[Option], expected: &[Option], field: &str) { + assert_eq!(actual.len(), expected.len(), "{field}: position count"); + for (position, (lhs, rhs)) in actual.iter().zip(expected).enumerate() { + match (lhs, rhs) { + (None, None) => {} + (Some(lhs), Some(rhs)) => assert_eq!( + lhs.to_bits(), + rhs.to_bits(), + "{field}: position {position} must keep its value" + ), + _ => panic!("{field}: position {position} is {lhs:?}, expected {rhs:?}"), + } + } +} + +/// How many positions carry a `NaN` value and how many carry nothing, counted separately because +/// the two must not be confused on the way through the codec. +fn nan_and_null_counts(values: &[Option]) -> (usize, usize) { + let nan = values + .iter() + .filter(|value| value.is_some_and(|value| value.is_nan())) + .count(); + let null = values.iter().filter(|value| value.is_none()).count(); + (nan, null) +} + +/// The native histogram table the histogram fixture references: two rows with distinct sums, the +/// first with a start timestamp and the second without. +/// +/// The shape is the one `datatypes::series::frame::tests::histogram_batch_with_sum` builds, so this +/// test compares against the same payload the frame layer's own tests use instead of inventing a +/// second fixture. +fn histogram_batch() -> Arc { + let custom = [1.5, 4.5]; + let positive_offsets = [1]; + let positive_lengths = [2]; + let positive_buckets = [3.25, 7.5]; + let negative_offsets = [-2]; + let negative_lengths = [1]; + let negative_buckets = [0.75]; + let view = |sum, start_timestamp| HistogramView { + schema: -7, + zero_threshold: 0.125, + sum, + count: 17.25, + zero_count: 2.5, + reset_hint: CounterResetHint::Unrecognized(37), + start_timestamp, + custom_values: &custom, + positive: BucketSideView { + span_offsets: &positive_offsets, + span_lengths: &positive_lengths, + buckets: &positive_buckets, + }, + negative: BucketSideView { + span_offsets: &negative_offsets, + span_lengths: &negative_lengths, + buckets: &negative_buckets, + }, + }; + let mut builder = HistogramBatchBuilder::new(); + builder.push(view(-9.5, Some(-123))).unwrap(); + builder.push(view(90.5, None)).unwrap(); + builder.finish().unwrap() +} + +/// The payload struct of one field of a frame batch: the float list and the histogram payload. +/// +/// A field is one column of the frame's `fields` struct, so this is how a test can tell which +/// representation the codec put on the wire instead of only what came back out of it. +fn frame_field_payload(batch: &RecordBatch, field: usize) -> &StructArray { + batch + .column(3) + .as_any() + .downcast_ref::() + .expect("a frame carries its fields as a struct") + .column(field) + .as_any() + .downcast_ref::() + .expect("a field payload is a struct") +} + +/// A multi-field chunk keeps each field's missing positions independent: one position carries a +/// value in one field and nothing in the other, and a `NaN` value never collapses into a missing one. +/// +/// This is the spec's `evaluated/multi-field` case. A schema snapshot cannot stand in for it, because +/// `NaN` and a missing value are both "no ordinary number" once the payload is only a list: which of +/// the two a position holds is visible only after a round trip. +#[tokio::test] +async fn multi_field_partial_nulls_are_not_nan() { + // Position 0 carries only `value` and position 2 only `score`. `NaN` is a value in both fields, + // and the position next to it in the other field is missing. + let value: Vec> = vec![Some(1.0), Some(f64::NAN), None, Some(f64::NAN), Some(4.0)]; + let score: Vec> = vec![None, None, Some(3.0), Some(5.0), Some(f64::NAN)]; + let metric = labels("n"); + let schema = pair_schema(); + let fields = SeriesFields::try_from_vec(vec![ + Samples::from_float(Float64Array::from(value.clone())), + Samples::from_float(Float64Array::from(score.clone())), + ]) + .unwrap(); + let batches = packed_under( + schema.clone(), + vec![( + chunk_on(&schema, &[(0, 2), (5, 3)], fields, metric.clone()), + true, + )], + ) + .await; + // One row per carried run, so the hole between the runs is not what the frame trades away. + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 2); + // Both fields are columns of every row, and both carry float values of their own. + for row in 0..batches[0].num_rows() { + for field in [0, 1] { + let payload = frame_field_payload(&batches[0], field); + assert!( + !payload.column(0).is_null(row), + "field {field} carries a float payload on row {row}" + ); + } + } + + let declared = batches[0].schema(); + let rows = drain(unpack_as(batches, declared, schema)).await; + assert_eq!(rows.len(), 2, "one decoded chunk per frame row"); + assert!( + rows.iter().all(|(row, _)| row.metric == metric), + "both rows name the same series" + ); + assert!(!rows[0].1, "only the last row closes the chain"); + assert!(rows[1].1); + // Concatenated in stream order, exactly as the producer's single chunk was laid out. + let decoded_positions: Vec = rows.iter().flat_map(|(row, _)| positions(row)).collect(); + assert_eq!( + decoded_positions, + vec![0, 1, 5, 6, 7], + "the hole between the two runs is not filled" + ); + let decoded_value: Vec> = rows + .iter() + .flat_map(|(row, _)| field_values(row, 0)) + .collect(); + let decoded_score: Vec> = rows + .iter() + .flat_map(|(row, _)| field_values(row, 1)) + .collect(); + // Per position, because each field decides for itself whether it holds a value. + assert_float_payload_eq(&decoded_value, &value, "value"); + assert_float_payload_eq(&decoded_score, &score, "score"); + // `NaN` and null are counted apart: a codec that turned one into the other would keep the + // position count and the alignment while losing the distinction this test is about. + assert_eq!( + nan_and_null_counts(&decoded_value), + nan_and_null_counts(&value), + "value: the NaN and null counts survive" + ); + assert_eq!( + nan_and_null_counts(&decoded_score), + nan_and_null_counts(&score), + "score: the NaN and null counts survive" + ); + assert_eq!( + nan_and_null_counts(&decoded_value), + (2, 1), + "two NaN values and one null, not three nulls" + ); + assert_eq!( + nan_and_null_counts(&decoded_score), + (1, 2), + "one NaN value and two nulls, not three values" + ); + // The same position carries a value in one field and nothing in the other, in both directions. + assert_eq!(decoded_value[0], Some(1.0)); + assert_eq!(decoded_score[0], None); + assert_eq!(decoded_value[2], None); + assert_eq!(decoded_score[2], Some(3.0)); + // And a NaN is a value: it keeps its own position, where the other field is missing. + assert!(decoded_value[1].is_some_and(f64::is_nan)); + assert_eq!(decoded_score[1], None); +} + +/// A histogram field keeps its native payload across the wire: the frame carries the table and the +/// per-position references to it, so what a consumer decodes is still a histogram. +/// +/// This is the spec's `native histogram/mixed` case. A length check cannot stand in for it, because a +/// silent float rendition preserves the position count; what rules the conversion out is the payload +/// kind on both sides of the wire and the values behind the references. +#[tokio::test] +async fn histogram_payload_roundtrips_without_float_conversion() { + let table = histogram_batch(); + let indices = UInt32Array::from(vec![Some(0), None, Some(1)]); + let samples = Samples::try_from_histogram(indices.clone(), table).unwrap(); + let metric = labels("m"); + let batches = packed(vec![( + chunk_on( + &evaluated_schema(), + &[(3, 3)], + SeriesFields::one(samples), + metric.clone(), + ), + true, + )]) + .await; + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 1, "one run is one frame row"); + + // On the wire the field is a payload struct that carries the histogram and no float list, so the + // codec never filled a float rendition in on the way out. + let payload = frame_field_payload(&batches[0], 0); + assert!( + payload.column(0).is_null(0), + "a histogram field has no float payload" + ); + assert!( + !payload.column(1).is_null(0), + "the histogram payload is what the frame carries" + ); + + let declared = batches[0].schema(); + let rows = drain(unpack(batches, declared)).await; + assert_eq!(rows.len(), 1); + let (decoded, is_last) = &rows[0]; + assert!(is_last); + assert_eq!(decoded.metric, metric); + assert_eq!(positions(decoded), vec![3, 4, 5]); + let field = decoded.fields.get(0).expect("the fixture has one field"); + let DenseSamplesRef::Histogram(histograms) = field.dense().expect("the field is dense") else { + panic!("a histogram field must decode as a histogram, not as a float or mixed segment"); + }; + assert_eq!(histograms.len(), 3, "every position is still there"); + assert_eq!( + histograms.batch().len(), + 2, + "both input histograms are still referenced" + ); + let decoded_indices = histograms.indices(); + assert_eq!(decoded_indices.len(), indices.len()); + for position in 0..indices.len() { + assert_eq!( + decoded_indices.is_valid(position), + indices.is_valid(position), + "position {position}: a null reference stays null" + ); + if indices.is_valid(position) { + assert_eq!(decoded_indices.value(position), indices.value(position)); + } + } + // The values behind the references are the input payload, not a lossy rendition of it. + let first = histograms + .view_at(0) + .expect("position 0 references a histogram"); + assert_eq!(first.count.to_bits(), 17.25_f64.to_bits()); + assert_eq!(first.sum.to_bits(), (-9.5_f64).to_bits()); + assert_eq!(first.start_timestamp, Some(-123)); + assert_eq!( + first + .positive + .buckets + .iter() + .map(|bucket| bucket.to_bits()) + .collect::>(), + [3.25_f64.to_bits(), 7.5_f64.to_bits()] + ); + assert!( + histograms.view_at(1).is_none(), + "a null reference stays missing instead of resolving to a neighbour's payload" + ); + let second = histograms + .view_at(2) + .expect("position 2 references a histogram"); + assert_eq!(second.count.to_bits(), 17.25_f64.to_bits()); + assert_eq!(second.sum.to_bits(), 90.5_f64.to_bits()); + assert_eq!(second.start_timestamp, None); + // The per-position kind survives as well: a referenced position is a histogram sample, never a + // float one. + assert!(matches!(field.sample(0), SampleRef::Histogram(_))); + assert!(matches!(field.sample(1), SampleRef::Missing)); + assert!(matches!(field.sample(2), SampleRef::Histogram(_))); +} + +/// One field can hold floats and histograms at different positions, and the codec keeps them apart: +/// each position decodes as the kind it was written as. +/// +/// This is the `native histogram/mixed` case in its harder form. A codec could keep the position +/// count and still promote one kind into the other in a single position, which only a per-position +/// kind comparison catches. +#[tokio::test] +async fn mixed_payload_roundtrips_without_dropping_either_kind() { + let table = histogram_batch(); + // Position 0 is a float, 1 a histogram, 2 nothing, and 3 a NaN float - which must not be read as + // a missing position, nor as the neighbouring histogram. + let floats = Float64Array::from(vec![Some(1.5), None, None, Some(f64::NAN)]); + let indices = UInt32Array::from(vec![None, Some(0), None, None]); + let histograms = datatypes::series::samples::HistogramSamples::try_new(indices, table).unwrap(); + let samples = Samples::try_from_mixed(floats, histograms).unwrap(); + let metric = labels("o"); + let batches = packed(vec![( + chunk_on( + &evaluated_schema(), + &[(6, 4)], + SeriesFields::one(samples), + metric.clone(), + ), + true, + )]) + .await; + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 1); + + // A mixed field is the one payload shape that carries both halves, so the frame neither dropped + // the float positions nor flattened the histogram ones. + let payload = frame_field_payload(&batches[0], 0); + assert!( + !payload.column(0).is_null(0), + "a mixed field carries its floats" + ); + assert!( + !payload.column(1).is_null(0), + "a mixed field carries its histograms" + ); + + let declared = batches[0].schema(); + let rows = drain(unpack(batches, declared)).await; + assert_eq!(rows.len(), 1); + let (decoded, is_last) = &rows[0]; + assert!(is_last); + assert_eq!(decoded.metric, metric); + assert_eq!(positions(decoded), vec![6, 7, 8, 9]); + let field = decoded.fields.get(0).expect("the fixture has one field"); + let DenseSamplesRef::Mixed(mixed) = field.dense().expect("the field is dense") else { + panic!("a mixed field must decode as mixed, not as one of its two halves"); + }; + assert_eq!(mixed.floats().len(), 4); + assert_eq!(mixed.histograms().len(), 4); + assert_eq!( + mixed.histograms().batch().len(), + 2, + "the referenced table survives whole" + ); + match field.sample(0) { + SampleRef::Float(value) => assert_eq!(value.to_bits(), 1.5_f64.to_bits()), + _ => panic!("position 0 was a float"), + } + match field.sample(1) { + SampleRef::Histogram(histogram) => { + assert_eq!(histogram.count.to_bits(), 17.25_f64.to_bits()); + assert_eq!(histogram.sum.to_bits(), (-9.5_f64).to_bits()); + } + _ => panic!("position 1 was a histogram"), + } + assert!(matches!(field.sample(2), SampleRef::Missing)); + match field.sample(3) { + // A NaN float stays a float value, and its histogram slot stays empty. + SampleRef::Float(value) => assert!(value.is_nan()), + _ => panic!("position 3 was a NaN float"), + } +} + +/// A frame whose spans add up beyond `usize` is refused, not answered with a wrapped count. +/// +/// The count of a row's positions is the first arithmetic a decoded row does, and it happens before +/// the row is checked against the grid it claims: two spans of `usize::MAX / 2 + 1` each are neither +/// ordered nor inside any grid, but summing them is what a decoder would do first. The sum is +/// therefore checked, so the frame is refused with an error instead of panicking in a debug build or +/// wrapping in a release one. +#[tokio::test] +async fn spans_that_overflow_the_position_count_are_refused() { + let batches = packed(vec![( + series(&[(0, 2)], vec![Some(1.0), Some(2.0)], labels("v")), + true, + )]) + .await; + let declared = batches[0].schema(); + let huge = u64::try_from(usize::MAX / 2 + 1).expect("a stride that leaves usize behind"); + // Two spans, because one is already representable: only their sum leaves `usize`, and that sum + // is what a decoder must check before it can report which span was wrong. + let hostile = with_spans(&batches[0], &[(0, huge), (huge, huge)], true); + let mut stream = unpack(vec![hostile], declared); + let error = stream + .next() + .await + .expect("a refused frame still yields its error") + .expect_err("the position count overflows"); + assert!( + error.to_string().contains("grid span length overflow"), + "refused for another reason: {error}" + ); +} + +/// An empty row that closes nothing is refused by the decoder itself, not only by the stream. +/// +/// A row covering no position is the closing chunk's shape: it exists to carry `is_last`. One that +/// carries `is_last = false` opens a chain it can never advance, so a consumer would wait for a +/// close that no later row owes. The stream validator refuses this, and so does the public decoder, +/// because a caller driving the decoder directly must not be handed a chunk no producer could have +/// written. +#[tokio::test] +async fn an_empty_row_that_closes_nothing_is_refused_by_the_decoder() { + // A legal empty closing chunk is packed first: the illegal shape cannot be produced, which is + // the point of the check below. Making it illegal means clearing its finality and nothing else, + // so the emptiness and the payload lengths stay consistent and only finality can refuse it. + let batches = packed(vec![(series(&[], vec![], labels("w")), true)]).await; + let declared = batches[0].schema(); + assert_eq!(1, batches[0].num_rows(), "the closing chunk is one row"); + let emptied = with_spans(&batches[0], &[], false); + let mut stream = unpack(vec![emptied], declared); + let error = stream + .next() + .await + .expect("a refused frame still yields its error") + .expect_err("an empty row that closes nothing is refused"); + assert!( + error + .to_string() + .contains("an evaluated chunk without positions must close its series"), + "refused for another reason: {error}" + ); +} + +/// Rewrites the span list of every row in `batch` to `spans`, and its finality to `is_last`. +/// +/// Both are rewritten together because a test that breaks a row's shape usually breaks its finality +/// as well: a producer cannot write an empty row that closes nothing, so such a row only exists once +/// a test builds it. +fn with_spans(batch: &RecordBatch, spans: &[(u64, u64)], is_last: bool) -> RecordBatch { + let declared = batch.schema(); + let DataType::Struct(time_members) = declared.field(2).data_type() else { + panic!("time is a struct"); + }; + let DataType::List(span_item) = time_members[0].data_type() else { + panic!("spans are a list"); + }; + let DataType::Struct(span_fields) = span_item.data_type() else { + panic!("a span is a struct"); + }; + let starts: Vec = spans.iter().map(|(start, _)| *start).collect(); + let lengths: Vec = spans.iter().map(|(_, len)| *len).collect(); + let values = StructArray::try_new( + span_fields.clone(), + vec![ + Arc::new(UInt64Array::from(starts)), + Arc::new(UInt64Array::from(lengths)), + ], + None, + ) + .unwrap(); + let lengths_per_row = vec![spans.len(); batch.num_rows()]; + let list = ListArray::try_new( + span_item.clone(), + OffsetBuffer::new(offset_buffer(&lengths_per_row).into()), + Arc::new(values), + None, + ) + .unwrap(); + let time = StructArray::try_new(time_members.clone(), vec![Arc::new(list)], None).unwrap(); + let mut columns = batch.columns().to_vec(); + columns[1] = Arc::new(BooleanArray::from(vec![is_last; batch.num_rows()])); + columns[2] = Arc::new(time); + RecordBatch::try_new_with_options( + declared, + columns, + &RecordBatchOptions::new().with_row_count(Some(batch.num_rows())), + ) + .unwrap() +} + +/// The cumulative list offsets of the given row lengths. +fn offset_buffer(lengths: &[usize]) -> Vec { + let mut offsets = Vec::with_capacity(lengths.len() + 1); + let mut total = 0i32; + offsets.push(0); + for length in lengths { + total += i32::try_from(*length).expect("a test row fits an Arrow offset"); + offsets.push(total); + } + offsets +} + +/// A cursor holding positions on one grid cannot fill a builder that declares another. +/// +/// Both grids describe the same number of points, so every position the cursor writes is in range +/// for the builder and the frame would decode cleanly. What it would decode to is a different set of +/// instants, which no later check could detect: the numbers are the only thing on the wire. The +/// builder therefore compares the grids themselves, not whether a grid is present. +#[test] +fn a_cursor_on_one_grid_cannot_fill_a_builder_declaring_another() { + // Both grids have four points, so the cursor's positions fit the builder exactly. + let other_grid = || { + Arc::new( + datatypes::series::time::EvalGrid::try_new( + datatypes::series::time::GridSpec::try_new(60_000, 10_000, 4) + .expect("a valid grid"), + ) + .expect("a valid grid"), + ) + }; + let other_schema = SeriesSchema { + fields: evaluated_schema().fields.clone(), + eval_grid: Some(other_grid()), + }; + // A chunk that is legal on its own grid: the series and the schema agree, so nothing but the + // builder's own declaration can refuse it. + let fields = SeriesFields::one(Samples::from_float(Float64Array::from(vec![Some(1.0)]))); + let time = TimeAxis::from_grid_spans(other_grid(), vec![GridSpan::new(0, 1)]) + .expect("one span on its own grid"); + let chunk = Series::try_new(&other_schema, labels("g"), time, fields).expect("a legal chunk"); + let mut cursor = + SeriesFrameCursor::try_new_evaluated(&other_schema, chunk, true).expect("a legal cursor"); + + let mut builder = SeriesFrameBuilder::try_new( + &evaluated_schema(), + FrameLabelMode::Explicit, + None, + SeriesFrameBuilder::DEFAULT_MAX_ROWS, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, + ) + .expect("the schema is a frame"); + let error = cursor + .append_to(&mut builder) + .expect_err("the grids differ"); + assert!( + error.to_string().contains("schema mismatch"), + "refused for another reason: {error}" + ); +} diff --git a/src/datatypes/src/series/frame.rs b/src/datatypes/src/series/frame.rs index cf1b5a41492..a5f51e655dc 100644 --- a/src/datatypes/src/series/frame.rs +++ b/src/datatypes/src/series/frame.rs @@ -12,14 +12,31 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Finite Arrow batches for raw series chunks. Field names preserve the logical series schema in order. +//! One Arrow frame family for raw and evaluated series chunks. //! -//! Packing accumulates bounded multi-row batches: [`RawSeriesChunkCursor`] walks one validated -//! chunk and [`RawSeriesFrameBuilder`] emits rows directly as Arrow arrays once a row or estimated -//! byte bound is reached. Decoding reuses one prepared schema per stream through -//! [`RawSeriesFrameDecoder`]. Column layout, metadata, and wire strings stay on the frozen `v1` -//! contract; a key-encoded frame additionally declares a versioned [`LabelDecodeSpec`] so its -//! labels can be decoded from the received schema alone. +//! A frame carries the same thing whichever sample shape it holds: one row per chunk of one series, +//! with that series' identity, its finality, its time axis, and one column of samples per logical +//! field. Field names preserve the logical series schema in order. What varies is declared rather +//! than inferred, because the receiver of a frame has no scanner, region, or planning context to +//! infer it from: +//! +//! | Content | Identity | Time | Producer | +//! | --- | --- | --- | --- | +//! | `raw` | `key` | `timestamps` | a storage raw-series scan | +//! | `raw` | `explicit` | `timestamps` | a raw input whose labels are already decoded | +//! | `evaluated` | `explicit` | `grid_spans` | a compute node's evaluated output | +//! +//! Any other combination is rejected here, because no producer can mean it: an evaluated value's +//! labels may be rewritten by the computation that produced it, so an evaluated frame cannot claim +//! a storage key as its identity, and raw samples have no evaluation grid to be addressed on. +//! +//! Packing accumulates bounded multi-row batches: [`SeriesFrameCursor`] walks one validated chunk +//! and [`SeriesFrameBuilder`] emits rows directly as Arrow arrays once a row or estimated byte bound +//! is reached. Decoding reuses one prepared +//! schema per stream through [`SeriesFrameDecoder`]. Column layout, metadata, and wire strings stay +//! on the frozen `v2` contract; a key-encoded frame additionally declares a versioned +//! [`LabelDecodeSpec`] so its labels can be decoded from the received schema alone, and an evaluated +//! frame declares its [`GridSpec`] once instead of repeating timestamps. //! //! # Label decode description versioning //! @@ -61,7 +78,7 @@ use std::sync::Arc; use arrow::array::{ Array, ArrayRef, BinaryArray, BooleanArray, Float64Array, Int64Array, ListArray, StringArray, - StructArray, UInt32Array, + StructArray, UInt32Array, UInt64Array, }; use arrow::buffer::{Buffer, NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow::compute::concat; @@ -74,8 +91,9 @@ use crate::error::{InvalidSeriesSnafu, Result}; use crate::series::histogram::{HistogramBatch, fields as histogram_fields}; use crate::series::raw::{RawSeriesChunk, RawTimestamps}; use crate::series::samples::{ - DenseSamplesRef, FieldSpec, HistogramSamples, Samples, SeriesFields, SeriesSchema, + DenseSamplesRef, FieldSpec, HistogramSamples, Samples, Series, SeriesFields, SeriesSchema, }; +use crate::series::time::{EvalGrid, GridRun, GridSpan, GridSpec, TimeAxis}; use crate::series::{Label, SeriesLabels, invalid_series}; const LAYOUT: &str = "greptime.series.layout"; @@ -85,19 +103,107 @@ const TIME: &str = "greptime.series.time"; const GRID: &str = "greptime.series.grid"; /// Metadata key of the versioned label decode description carried by key-encoded frames. const LABEL_DECODE: &str = "greptime.series.label_decode"; +/// Metadata key of the producer ordering, written by the streaming layer rather than here. +const ORDERING: &str = "greptime.series.ordering"; + +/// Every `greptime.series.*` key a frame this revision writes may carry. +/// +/// [`ORDERING`] is a stream-level declaration this module neither writes nor reads; it is listed so +/// that a reserved key nobody checks is refused here instead of travelling as dead metadata. +const RESERVED_KEYS: [&str; 7] = [LAYOUT, CONTENT, LABELS, TIME, GRID, LABEL_DECODE, ORDERING]; + +/// The only layout this revision writes and reads. +/// +/// A frame declaring any other value is refused instead of decoded: the sample representation of an +/// unreleased format is not worth a second decoder, and silently guessing would turn a format change +/// into wrong values. +pub const FRAME_LAYOUT_VERSION: &str = "v2"; /// Version of the label decode description this revision writes and reads. +/// +/// This is independent of [`FRAME_LAYOUT_VERSION`]: the description versions its own text, which a +/// reader refuses to guess at, while the layout versions the column and metadata set around it. pub const LABEL_DECODE_VERSION: u32 = 1; -/// Whether a raw frame carries only the complete key or explicit metric labels as well. +/// What one frame's rows hold: raw samples with their own timestamps, or evaluated samples +/// addressed on the grid the frame declares. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum RawLabelMode { - /// Labels remain encoded solely in `read_key`. +pub enum FrameContent { + /// Samples read from storage, each row carrying the timestamps it was read at. + Raw, + /// Values computed on an evaluation grid, each row carrying the grid positions it holds. + Evaluated, +} + +impl FrameContent { + /// The `greptime.series.content` text of this content. + pub const fn as_str(self) -> &'static str { + match self { + Self::Raw => "raw", + Self::Evaluated => "evaluated", + } + } + + /// Parses a `greptime.series.content` value this revision implements. + const fn parse(text: &str) -> Option { + match text.as_bytes() { + b"raw" => Some(Self::Raw), + b"evaluated" => Some(Self::Evaluated), + _ => None, + } + } + + /// The `greptime.series.time` text this content's rows are addressed by. + /// + /// Time mode is a function of content and never an independent choice: raw samples carry their + /// own timestamps and evaluated ones sit on a grid, so a frame has no way to hold one without + /// the other. Writing it as a derived text rather than a second enum keeps the two from ever + /// disagreeing. + const fn time_text(self) -> &'static str { + match self { + Self::Raw => "timestamps", + Self::Evaluated => "grid_spans", + } + } + + /// The struct member a row's time column carries for this content. + const fn time_member(self) -> &'static str { + match self { + Self::Raw => "timestamps", + Self::Evaluated => "spans", + } + } +} + +/// How a frame names the series of its rows. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FrameLabelMode { + /// Identity is only the encoded storage key in `read_key`, decoded through the frame's own + /// [`LabelDecodeSpec`]. Key, - /// Complete metric name and ordinary labels are present in the frame. + /// The complete metric name and ordinary labels are present in the frame as well. Explicit, } +impl FrameLabelMode { + /// The `greptime.series.labels` text of this mode. + pub const fn as_str(self) -> &'static str { + match self { + Self::Key => "key", + Self::Explicit => "explicit", + } + } + + /// Parses a `greptime.series.labels` value this revision implements. + const fn parse(text: &str) -> Option { + match text.as_bytes() { + b"key" => Some(Self::Key), + b"explicit" => Some(Self::Explicit), + _ => None, + } + } +} + /// Primary-key encoding of the labels a key-encoded frame carries. /// /// This mirrors the storage contract's `PrimaryKeyEncoding` without making this crate depend on @@ -148,7 +254,7 @@ pub struct LabelDecodeTag { /// a silent fallback. /// /// It is JSON-encoded in one schema metadata entry, always through -/// [`raw_series_frame_schema_with_label_decode`], so both the producing and the expecting side of a +/// [`series_frame_schema`], so both the producing and the expecting side of a /// `greptime.series.*` exact metadata comparison see byte-identical text. The text is /// self-describing but not a general interchange format: readers reject unknown fields here only /// at this structure's own levels, while [`LabelDecodeColumn::physical_type`] is decoded by @@ -188,7 +294,7 @@ pub struct LabelDecodeSpec { impl LabelDecodeSpec { /// Puts the order-insensitive part of the description in its canonical order. /// - /// [`raw_series_frame_schema_with_label_decode`] applies this to a copy before serializing, and + /// [`series_frame_schema`] applies this to a copy before serializing, and /// so does the description a binding derives (in `common_query::series_frame`), so both sides of /// a text comparison produce the same bytes and a reader can compare descriptions structurally. /// `primary_key` is deliberately untouched: its order is the encoding order. @@ -253,27 +359,76 @@ impl LabelDecodeSpec { } } -/// Builds the typed schema for a raw frame, preserving logical field names and order. +/// Builds the one frame schema family: raw or evaluated, keyed or explicit. /// -/// This is the column layout shared with the frame builder and decoder; it deliberately carries no -/// label decode description. A key-encoded stream must be declared through -/// [`raw_series_frame_schema_with_label_decode`], which refuses such a layout without one. -pub fn raw_series_frame_schema(schema: &SeriesSchema, mode: RawLabelMode) -> Result { - if schema.eval_grid.is_some() { - return invalid_series("raw series schema cannot have an evaluation grid"); - } +/// Everything the wire states is derived from the two things a producer actually holds. The +/// evaluation grid comes from `schema.eval_grid`, where a schema without one describes raw samples +/// and a schema with one describes values addressed on that grid, so content and time mode can never +/// contradict the columns they describe. `labels` decides identity, and with it whether the frame +/// carries the `name`/`labels` columns: a key frame keeps its labels encoded in `read_key` and must +/// declare the [`LabelDecodeSpec`] that decodes them, while an explicit frame carries complete +/// labels and must not declare one. +/// +/// This is the only place the reserved `greptime.series.*` text is written, so a producer and a +/// reader cannot spell one frame two ways. +pub fn series_frame_schema( + schema: &SeriesSchema, + labels: FrameLabelMode, + decode: Option<&LabelDecodeSpec>, +) -> Result { if schema.fields.is_empty() { - return invalid_series("raw series schema must have fields"); + return invalid_series("series frame schema must have fields"); + } + let content = match &schema.eval_grid { + Some(_) => FrameContent::Evaluated, + None => FrameContent::Raw, + }; + match (labels, decode) { + (FrameLabelMode::Key, None) => { + return Err(err( + "key-encoded frames must carry a label decode description", + )); + } + (FrameLabelMode::Explicit, Some(_)) => { + return Err(err( + "explicit-label frames carry no label decode description", + )); + } + (FrameLabelMode::Key, Some(_)) | (FrameLabelMode::Explicit, None) => {} + } + if content == FrameContent::Evaluated && labels == FrameLabelMode::Key { + return Err(err( + "an evaluated frame names its series by label, not by a storage key", + )); } let list = |ty, nullable| DataType::List(Arc::new(Field::new("item", ty, nullable))); - let time = DataType::Struct( - vec![Field::new( - "timestamps", - list(DataType::Int64, false), - false, - )] - .into(), - ); + let time = match content { + FrameContent::Raw => DataType::Struct( + vec![Field::new( + "timestamps", + list(DataType::Int64, false), + false, + )] + .into(), + ), + FrameContent::Evaluated => DataType::Struct( + vec![Field::new( + "spans", + list( + DataType::Struct( + vec![ + Field::new("start", DataType::UInt64, false), + Field::new("len", DataType::UInt64, false), + ] + .into(), + ), + false, + ), + false, + )] + .into(), + ), + }; let hist = DataType::Struct( vec![ Field::new("indices", list(DataType::UInt32, true), false), @@ -298,12 +453,19 @@ pub fn raw_series_frame_schema(schema: &SeriesSchema, mode: RawLabelMode) -> Res .map(|field| Field::new(field.name.as_ref(), payload.clone(), false)) .collect(); let mut columns = vec![ - Field::new("read_key", DataType::Binary, false), + // A key frame *is* its key, so the column is required. An explicit frame names its series + // by label and keeps the column for the runtime chunk's own contract: a raw chunk still + // carries the key it was read under, an evaluated one carries none. + Field::new( + "read_key", + DataType::Binary, + labels == FrameLabelMode::Explicit, + ), Field::new("is_last", DataType::Boolean, false), Field::new("time", time, false), Field::new("fields", DataType::Struct(fields), false), ]; - if mode == RawLabelMode::Explicit { + if labels == FrameLabelMode::Explicit { columns.push(Field::new("name", DataType::Utf8, true)); columns.push(Field::new( "labels", @@ -320,189 +482,294 @@ pub fn raw_series_frame_schema(schema: &SeriesSchema, mode: RawLabelMode) -> Res false, )); } - let metadata = [ - (LAYOUT.to_string(), "v1".to_string()), - (CONTENT.to_string(), "raw".to_string()), - ( - LABELS.to_string(), - match mode { - RawLabelMode::Key => "key", - RawLabelMode::Explicit => "explicit", - } - .to_string(), - ), - (TIME.to_string(), "timestamps".to_string()), - ] - .into_iter() - .collect(); - Ok(Arc::new(Schema::new_with_metadata(columns, metadata))) -} - -/// Builds the typed schema for a raw frame together with its label decode description. -/// -/// The description is written only for [`RawLabelMode::Key`], where labels stay encoded in -/// `read_key`; explicit-label frames carry complete labels and must not declare one. Both -/// directions are enforced here so a frame cannot claim a label layout it does not describe. -pub fn raw_series_frame_schema_with_label_decode( - schema: &SeriesSchema, - mode: RawLabelMode, - decode: Option<&LabelDecodeSpec>, -) -> Result { - match (mode, decode) { - (RawLabelMode::Key, None) => { - return Err(err( - "key-encoded raw frames must carry a label decode description", - )); - } - (RawLabelMode::Explicit, Some(_)) => { - return Err(err("explicit raw frames carry no label decode description")); - } - (RawLabelMode::Key, Some(_)) | (RawLabelMode::Explicit, None) => {} + let mut metadata = vec![ + (LAYOUT.to_string(), FRAME_LAYOUT_VERSION.to_string()), + (CONTENT.to_string(), content.as_str().to_string()), + (LABELS.to_string(), labels.as_str().to_string()), + (TIME.to_string(), content.time_text().to_string()), + ]; + if let Some(grid) = &schema.eval_grid { + metadata.push((GRID.to_string(), grid_text(grid.spec())?)); + } + if let Some(decode) = decode { + // Writing is faithful: the description records what the producer declares, and nothing is + // rewritten or dropped. The one exception is the order of `logical_tags`, which carries no + // meaning to a decoder and which this text nevertheless has to fix so two producers of the + // same labels write the same bytes; a copy is canonicalized and the caller's value is + // untouched. + // + // What is not accepted is a description this revision would refuse to read. A frame is worth + // writing only if it can be decoded, so the same check runs here: a duplicate key column, a + // logical tag naming no listed column, or a description of an unknown version is refused at + // the producer rather than when the frame reaches its consumer, where the producer is no + // longer in a position to fix it. + let mut canonical = decode.clone(); + canonical.canonicalize(); + canonical.validate()?; + metadata.push(( + LABEL_DECODE.to_string(), + serde_json::to_string(&canonical) + .map_err(|e| err(format!("cannot encode label decode description: {e}")))?, + )); } - let layout = raw_series_frame_schema(schema, mode)?; - let Some(decode) = decode else { - return Ok(layout); - }; - // Writing is faithful: the description records what the producer declares, and nothing is - // rewritten or dropped. The one exception is the order of `logical_tags`, which carries no - // meaning to a decoder and which this text nevertheless has to fix so two producers of the same - // labels write the same bytes; a copy is canonicalized and the caller's value is untouched. A - // description this revision cannot honour is rejected when it is read, which is the only - // direction a consumer can rely on. - let mut canonical = decode.clone(); - canonical.canonicalize(); - let encoded = serde_json::to_string(&canonical) - .map_err(|e| err(format!("cannot encode label decode description: {e}")))?; - let mut metadata = layout.metadata().clone(); - metadata.insert(LABEL_DECODE.to_string(), encoded); Ok(Arc::new(Schema::new_with_metadata( - layout.fields().clone(), - metadata, + columns, + metadata.into_iter().collect(), ))) } -/// Recovers the label decode description a raw frame schema declares. +/// Builds the typed schema for a raw frame, refusing a series schema with an evaluation grid. +/// +/// A key frame is unreadable without its description, so this constructor serves explicit-label +/// frames only; a caller that holds a binding declares through [`series_frame_schema`]. +pub fn raw_series_frame_schema(schema: &SeriesSchema, labels: FrameLabelMode) -> Result { + if schema.eval_grid.is_some() { + return invalid_series("raw series schema cannot have an evaluation grid"); + } + match labels { + FrameLabelMode::Key => Err(err( + "raw key frames must declare a label decode description", + )), + FrameLabelMode::Explicit => series_frame_schema(schema, labels, None), + } +} + +/// The canonical `greptime.series.grid` text of one grid. +/// +/// Field order is the struct's declaration order and the text is compact, so two producers of the +/// same grid write the same bytes; nothing else in the tree spells this text. +fn grid_text(spec: GridSpec) -> Result { + serde_json::to_string(&spec).map_err(|e| err(format!("cannot encode evaluation grid: {e}"))) +} + +/// Parses one `greptime.series.grid` text back into a validated, deferrable grid. +/// +/// Validation is [`GridSpec::try_new`]'s plus the unknown-field refusal serde performs, so a frame +/// cannot declare a grid this revision would compute differently. The timestamps themselves stay +/// unmaterialized until a consumer asks for them. +fn parse_grid(text: &str) -> Result> { + let spec: GridSpec = serde_json::from_str(text) + .map_err(|e| err(format!("cannot decode evaluation grid: {e}")))?; + Ok(Arc::new(EvalGrid::try_deferred(spec)?)) +} + +/// A frame schema's reserved declarations, parsed and cross-checked. +/// +/// The three facts a receiver needs are not independent - the content decides the time mode, the +/// label mode decides whether a description must be present, and only an evaluated frame may name a +/// grid - so they are parsed together and every contradictory combination is refused instead of one +/// side of it being picked. +#[derive(Clone, Debug)] +pub struct SeriesFrameDeclaration { + series: SeriesSchema, + labels: FrameLabelMode, + label_decode: Option, +} + +impl SeriesFrameDeclaration { + /// Parses and cross-checks one frame schema. + pub fn parse(schema: &Schema) -> Result { + for key in schema.metadata().keys() { + if key.starts_with("greptime.series.") && !RESERVED_KEYS.contains(&key.as_str()) { + return Err(err(format!( + "frame declares unknown reserved metadata {key}" + ))); + } + } + if schema.metadata().get(LAYOUT).map(String::as_str) != Some(FRAME_LAYOUT_VERSION) { + return Err(err(format!("frame declares no known {LAYOUT} value"))); + } + let content = schema + .metadata() + .get(CONTENT) + .and_then(|text| FrameContent::parse(text)) + .ok_or_else(|| err(format!("frame declares no known {CONTENT} value")))?; + let labels = schema + .metadata() + .get(LABELS) + .and_then(|text| FrameLabelMode::parse(text)) + .ok_or_else(|| err(format!("frame declares no known {LABELS} value")))?; + // Time mode is read as the content's own text: a frame whose time column is addressed the + // other way contradicts itself, and there is no third time mode to fall back to. + if schema.metadata().get(TIME).map(String::as_str) != Some(content.time_text()) { + return Err(err(format!( + "frame declares no {TIME} value this content can carry" + ))); + } + if content == FrameContent::Evaluated && labels == FrameLabelMode::Key { + return Err(err( + "an evaluated frame names its series by label, not by a storage key", + )); + } + let grid = match (content, schema.metadata().get(GRID)) { + (FrameContent::Raw, None) => None, + (FrameContent::Raw, Some(_)) => { + return Err(err("a raw frame cannot declare an evaluation grid")); + } + (FrameContent::Evaluated, None) => { + return Err(err("an evaluated frame must declare its evaluation grid")); + } + (FrameContent::Evaluated, Some(text)) => Some(parse_grid(text)?), + }; + let label_decode = match schema.metadata().get(LABEL_DECODE) { + None => None, + Some(text) => { + let decode: LabelDecodeSpec = serde_json::from_str(text) + .map_err(|e| err(format!("cannot decode label decode description: {e}")))?; + decode.validate()?; + Some(decode) + } + }; + match (labels, &label_decode) { + (FrameLabelMode::Key, None) => { + return Err(err( + "key-encoded frames must carry a label decode description", + )); + } + (FrameLabelMode::Explicit, Some(_)) => { + return Err(err( + "explicit-label frames carry no label decode description", + )); + } + (FrameLabelMode::Key, Some(_)) | (FrameLabelMode::Explicit, None) => {} + } + let fields = schema + .fields() + .iter() + .find(|field| field.name() == "fields") + .ok_or_else(|| err("frame declares no fields column"))?; + let DataType::Struct(members) = fields.data_type() else { + return Err(err("frame fields column is not a struct")); + }; + if members.is_empty() { + return Err(err("series frame schema must have fields")); + } + let declared_time = schema + .fields() + .iter() + .find(|field| field.name() == "time") + .ok_or_else(|| err("frame declares no time column"))?; + let DataType::Struct(time_members) = declared_time.data_type() else { + return Err(err("frame time column is not a struct")); + }; + if time_members.len() != 1 || time_members[0].name() != content.time_member() { + return Err(err( + "frame time column does not match its declared time mode", + )); + } + Ok(Self { + series: SeriesSchema { + fields: members + .iter() + .map(|member| FieldSpec { + name: Arc::from(member.name().as_str()), + }) + .collect(), + eval_grid: grid, + }, + labels, + label_decode, + }) + } + + /// The logical fields and evaluation grid the frame declares. + pub fn series(&self) -> &SeriesSchema { + &self.series + } + + /// What the frame's rows hold. + pub fn content(&self) -> FrameContent { + match self.series.eval_grid { + Some(_) => FrameContent::Evaluated, + None => FrameContent::Raw, + } + } + + /// How the frame names its series. + pub fn labels(&self) -> FrameLabelMode { + self.labels + } + + /// The label decode description the frame declares, present exactly for key identity. + pub fn label_decode(&self) -> Option<&LabelDecodeSpec> { + self.label_decode.as_ref() + } +} + +/// Recovers the label decode description a frame schema declares. /// /// Returns `None` for an explicit-label frame, which must not declare one. A key-encoded frame must /// declare one, so a missing description is an error rather than a reason to reuse a caller-side -/// binding. A present description is parsed strictly: an unknown version, a missing or unknown -/// field at this structure's own levels, a type mismatch, or a description whose logical tags do -/// not name listed key columns is an error, never a silent default. Only the description's own -/// levels reject unknown fields; [`LabelDecodeColumn::physical_type`] is a -/// [`ConcreteDataType`] and accepts whatever that type's own serde accepts. -pub fn raw_series_frame_label_decode(schema: &Schema) -> Result> { - let mode = match schema.metadata().get(LABELS).map(String::as_str) { - Some("key") => RawLabelMode::Key, - Some("explicit") => RawLabelMode::Explicit, - _ => return Err(err("raw frame schema declares no known label mode")), - }; - let decode = match schema.metadata().get(LABEL_DECODE) { - None => None, - Some(encoded) => { - let decode: LabelDecodeSpec = serde_json::from_str(encoded) - .map_err(|e| err(format!("malformed label decode description: {e}")))?; - decode.validate()?; - Some(decode) - } - }; - match (mode, &decode) { - (RawLabelMode::Key, None) => Err(err( - "key-encoded raw frame declares no label decode description", - )), - (RawLabelMode::Explicit, Some(_)) => Err(err( - "explicit raw frame declares a label decode description", - )), - (RawLabelMode::Key, Some(_)) | (RawLabelMode::Explicit, None) => Ok(decode), +/// binding; [`series_frame_schema`] and [`SeriesFrameDeclaration::parse`] enforce that both ways. +pub fn series_frame_label_decode(schema: &Schema) -> Result> { + Ok(SeriesFrameDeclaration::parse(schema)?.label_decode) +} + +/// The two declarations the encoder and decoder dispatch on. +/// +/// A frame's time mode follows from its content and its identity is the label mode, so those two +/// are all that has to travel next to the column layout. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct FrameShape { + content: FrameContent, + labels: FrameLabelMode, +} + +impl FrameShape { + fn of(schema: &SeriesSchema, labels: FrameLabelMode) -> Self { + let content = match schema.eval_grid { + Some(_) => FrameContent::Evaluated, + None => FrameContent::Raw, + }; + Self { content, labels } } } -/// Reconstructs the series schema a raw frame declares. +/// Bounded multi-row builder for finite series frame batches. /// -/// A raw frame names its payload fields by name only: every member of the `fields` struct carries -/// the same payload type ([`raw_series_frame_schema`]), and [`SeriesSchema`] holds nothing else per -/// field. The reconstruction is therefore exact rather than a projection — the declared names in -/// declared order, and no evaluation grid, which a raw frame cannot carry. -/// -/// The reserved layout, content, and time metadata are checked here together with the absence of an -/// evaluation grid, so a frame this revision cannot read is rejected instead of being silently -/// interpreted as the current layout. -pub fn raw_series_frame_series_schema(schema: &Schema) -> Result { - for (key, expected) in [(LAYOUT, "v1"), (CONTENT, "raw"), (TIME, "timestamps")] { - if schema.metadata().get(key).map(String::as_str) != Some(expected) { - return Err(err(format!("raw frame declares no known {key} value"))); - } - } - if schema - .metadata() - .keys() - .any(|key| key == GRID || key.starts_with("greptime.series.grid.")) - { - return Err(err("raw series schema cannot have an evaluation grid")); - } - let fields = schema - .fields() - .iter() - .find(|field| field.name() == "fields") - .ok_or_else(|| err("raw frame declares no fields column"))?; - let DataType::Struct(members) = fields.data_type() else { - return Err(err("raw frame fields column is not a struct")); - }; - if members.is_empty() { - return Err(err("raw series schema must have fields")); - } - Ok(SeriesSchema { - fields: members - .iter() - .map(|member| FieldSpec { - name: Arc::from(member.name().as_str()), - }) - .collect(), - eval_grid: None, - }) -} - -/// Bounded multi-row builder for finite raw frame batches. -/// -/// Rows accumulate until the row limit or the estimated byte limit is reached; the next -/// [`RawSeriesChunkCursor::append_to`] call then emits them as one batch. `max_bytes` is stored as -/// the effective group cap `min(requested, i32::MAX)`, and each row carries a measure that -/// dominates every cumulative Arrow offset domain, so any admitted group can be encoded. The -/// measure is an internal work estimate, not an exact IPC size and not a resident-memory cap: one -/// individually representable row that exceeds the requested cap is still accepted and emitted -/// alone, because that slice cannot be split into smaller legal rows. -pub struct RawSeriesFrameBuilder { +/// Rows accumulate until the row limit or the estimated byte limit is reached; the next cursor call +/// then emits them as one batch. `max_bytes` is stored as the effective group cap +/// `min(requested, i32::MAX)`, and each row carries a measure that dominates every cumulative Arrow +/// offset domain, so any admitted group can be encoded. The measure is an internal work estimate, not +/// an exact IPC size and not a resident-memory cap: one individually representable row that exceeds +/// the requested cap is still accepted and emitted alone, because that slice cannot be split into +/// smaller legal rows. +pub struct SeriesFrameBuilder { schema: SeriesSchema, frame_schema: SchemaRef, - mode: RawLabelMode, + shape: FrameShape, max_rows: usize, max_bytes: usize, - rows: Vec, + rows: Vec, bytes: usize, } -impl RawSeriesFrameBuilder { +impl SeriesFrameBuilder { /// Default maximum rows in one packed batch when no explicit bound is needed. pub const DEFAULT_MAX_ROWS: usize = 256; /// Default maximum estimated encoded bytes in one packed batch when no explicit bound is /// needed. pub const DEFAULT_MAX_BYTES: usize = 1024 * 1024; - /// Creates an empty builder for the given schema, label mode, and nonzero bounds. + /// Creates an empty builder for the given schema, identity declaration, and nonzero bounds. /// - /// The frame schema is built once here and reused for every emitted batch. + /// The frame schema is built once here and reused for every emitted batch. It declares + /// everything the rows carry except the stream's producer ordering, which only the streaming + /// layer knows and adds when it declares the batch. pub fn try_new( schema: &SeriesSchema, - mode: RawLabelMode, + labels: FrameLabelMode, + decode: Option<&LabelDecodeSpec>, max_rows: usize, max_bytes: usize, ) -> Result { if max_rows == 0 || max_bytes == 0 { - return invalid_series("raw series frame limits must be nonzero"); + return invalid_series("series frame limits must be nonzero"); } Ok(Self { schema: schema.clone(), - frame_schema: raw_series_frame_schema(schema, mode)?, - mode, + frame_schema: series_frame_schema(schema, labels, decode)?, + shape: FrameShape::of(schema, labels), max_rows, max_bytes: max_bytes.min(i32::MAX as usize), rows: Vec::new(), @@ -520,7 +787,7 @@ impl RawSeriesFrameBuilder { if self.rows.is_empty() { return Ok(None); } - let batch = encode_rows(&self.frame_schema, self.mode, &self.rows)?; + let batch = encode_rows(&self.frame_schema, self.shape, &self.rows)?; self.rows.clear(); self.bytes = 0; Ok(Some(batch)) @@ -536,7 +803,7 @@ impl RawSeriesFrameBuilder { /// An empty builder accepts any individually measured row; a group total that would overflow /// or exceed the cap flushes the buffered rows before this row is consumed, never dropping or /// splitting the row. - fn accepts(&self, row: &RawFrameRow) -> bool { + fn accepts(&self, row: &FrameRow) -> bool { self.rows.is_empty() || (self.bytes < self.max_bytes && self.rows.len() < self.max_rows @@ -547,29 +814,98 @@ impl RawSeriesFrameBuilder { } /// Buffers one individually measured row. - fn push(&mut self, row: RawFrameRow) -> Result<()> { + fn push(&mut self, row: FrameRow) -> Result<()> { self.bytes = add_checked(self.bytes, row.bytes)?; self.rows.push(row); Ok(()) } + + /// Checks that a cursor walks the same fields, identity, and time of what this builder encodes. + /// + /// The grid is compared by its specification rather than by whether one is present: a cursor + /// holding positions on one grid and a builder encoding them under another would write the same + /// numbers meaning different instants, and nothing downstream could tell. Two grids with equal + /// bounds are the same grid, so they compare equal however they were allocated. + fn matches(&self, schema: &SeriesSchema, shape: FrameShape) -> Result<()> { + let same_fields = schema.fields.len() == self.schema.fields.len() + && schema + .fields + .iter() + .zip(self.schema.fields.iter()) + .all(|(left, right)| left.name == right.name); + let same_grid = match (&schema.eval_grid, &self.schema.eval_grid) { + (None, None) => true, + (Some(left), Some(right)) => left.spec() == right.spec(), + _ => false, + }; + if shape != self.shape || !same_grid || !same_fields { + return invalid_series("series frame cursor and builder schema mismatch"); + } + Ok(()) + } } -/// One validated chunk walked segment by segment into a bounded builder. -pub struct RawSeriesChunkCursor { +/// One validated chunk walked into a bounded builder, one row at a time. +/// +/// The two sample shapes differ only in how a row's positions are addressed and where its identity +/// comes from, so one cursor serves both: a raw chunk is walked segment by segment over the +/// timestamps it carries, an evaluated chunk run by run over the grid positions it holds. Each row is +/// measured once and retried after a flush, so a flush error never consumes input. +pub struct SeriesFrameCursor { schema: SeriesSchema, - mode: RawLabelMode, + shape: FrameShape, + walk: ChunkWalk, + pending: Option, +} + +/// Where a cursor is in the chunk it walks. +enum ChunkWalk { + /// A raw chunk: one timestamp per row of samples. + Raw(RawWalk), + /// An evaluated chunk: absolute grid positions, carried as runs. + Evaluated(EvaluatedWalk), +} + +/// The walk state of one raw chunk. +/// +/// The chunk's fields are walked segment by segment, so a row never spans two segments of one field +/// and no per-row temporary vectors are allocated. +struct RawWalk { chunk: RawSeriesChunk, + /// The dense segment each field is being read from. + segments: Vec, + /// How much of each field's current segment has been consumed. + offsets: Vec, + /// How many timestamps have been consumed. + pos: usize, +} + +/// The walk state of one evaluated chunk. +/// +/// The runs are collected once, so a row is one intersection of a run with a sample segment and the +/// walk never re-derives the runs per row. A chunk that carries no position keeps a single `closing` +/// flag: it still owes one row, and that row is what tells its consumer the chain ended. +struct EvaluatedWalk { + series: Series, + is_last: bool, + /// The runs of carried rows, in absolute grid order and never empty. + runs: Vec, + /// The run being read. + run: usize, + /// How much of the current run has been consumed. + offset: usize, + /// The dense segment each field is being read from. segments: Vec, offsets: Vec, - pos: usize, - pending: Option, + /// Whether the one row a chunk without positions still owes is yet to be emitted. + closing: bool, } -impl RawSeriesChunkCursor { - /// Validates one public chunk's shape and label mode once, taking ownership of it. - pub fn try_new( +impl SeriesFrameCursor { + /// Validates one public raw chunk's shape and label mode once, taking ownership of it. + pub fn try_new_raw( schema: &SeriesSchema, - mode: RawLabelMode, + mode: FrameLabelMode, chunk: RawSeriesChunk, ) -> Result { let RawSeriesChunk { @@ -581,18 +917,64 @@ impl RawSeriesChunkCursor { } = chunk; let chunk = RawSeriesChunk::try_new(schema, primary_key, metric, time, fields, is_last)?; match (mode, &chunk.metric) { - (RawLabelMode::Key, None) | (RawLabelMode::Explicit, Some(_)) => {} + (FrameLabelMode::Key, None) | (FrameLabelMode::Explicit, Some(_)) => {} _ => return invalid_series("raw chunk label mode does not match frame mode"), } let segments = vec![0usize; chunk.fields.len()]; let offsets = vec![0usize; chunk.fields.len()]; Ok(Self { schema: schema.clone(), - mode, - chunk, - segments, - offsets, - pos: 0, + shape: FrameShape::of(schema, mode), + walk: ChunkWalk::Raw(RawWalk { + chunk, + segments, + offsets, + pos: 0, + }), + pending: None, + }) + } + + /// Validates one evaluated chunk against the frame's schema, grid, and finality rules. + /// + /// A chunk that carries no position is legal only when it closes its series: it is how a + /// consumer tells "nothing more here" from "no more input", and a chain of empty chunks that + /// close nothing would never make progress. It still occupies one frame row, because dropping it + /// would drop the finality it exists to carry. + pub fn try_new_evaluated(schema: &SeriesSchema, series: Series, is_last: bool) -> Result { + if schema.eval_grid.is_none() { + return invalid_series("an evaluated chunk needs a schema with an evaluation grid"); + } + let series = Series::try_new( + schema, + series.metric.clone(), + series.time.clone(), + series.fields.clone(), + )?; + if series.time.grid().is_none() { + return invalid_series("an evaluated chunk must be addressed on a grid"); + } + if series.time.is_empty() && !is_last { + return invalid_series("an evaluated chunk without positions must close its series"); + } + // The runs are the only runs this chunk carries, so collecting them once is what keeps the + // walk from re-deriving them per row; a maximal run is merged by the axis itself. + let runs: Vec = series.time.runs().collect(); + let segments = vec![0usize; series.fields.len()]; + let offsets = vec![0usize; series.fields.len()]; + Ok(Self { + schema: schema.clone(), + shape: FrameShape::of(schema, FrameLabelMode::Explicit), + walk: ChunkWalk::Evaluated(EvaluatedWalk { + closing: runs.is_empty(), + series, + is_last, + runs, + run: 0, + offset: 0, + segments, + offsets, + }), pending: None, }) } @@ -602,11 +984,8 @@ impl RawSeriesChunkCursor { /// `None` means the cursor is exhausted; calling again returns `None` without touching the /// builder. The single pending row is measured once by [`Self::next_row`] and retried after the /// buffered rows are emitted. - pub fn append_to( - &mut self, - builder: &mut RawSeriesFrameBuilder, - ) -> Result> { - self.compatible(builder)?; + pub fn append_to(&mut self, builder: &mut SeriesFrameBuilder) -> Result> { + builder.matches(&self.schema, self.shape)?; loop { let row = match self.pending.take() { Some(row) => row, @@ -623,7 +1002,7 @@ impl RawSeriesChunkCursor { self.pending = Some(row); let batch = builder .flush()? - .ok_or_else(|| err("raw frame builder rejected a row without buffered rows"))?; + .ok_or_else(|| err("frame builder rejected a row without buffered rows"))?; return Ok(Some(batch)); } builder.push(row)?; @@ -635,68 +1014,157 @@ impl RawSeriesChunkCursor { } } - /// The cursor and builder must describe the same fields and label mode. - fn compatible(&self, builder: &RawSeriesFrameBuilder) -> Result<()> { - let same_fields = self.schema.fields.len() == builder.schema.fields.len() - && self - .schema - .fields - .iter() - .zip(builder.schema.fields.iter()) - .all(|(left, right)| left.name == right.name); - if self.mode != builder.mode - || self.schema.eval_grid.is_some() != builder.schema.eval_grid.is_some() - || !same_fields - { - return invalid_series("raw frame cursor and builder schema mismatch"); + /// Builds and measures the next row, or `None` when the chunk is exhausted. + fn next_row(&mut self) -> Result> { + match &mut self.walk { + ChunkWalk::Raw(walk) => walk.next_row(), + ChunkWalk::Evaluated(walk) => walk.next_row(), } - Ok(()) } +} - /// Builds and measures the next union-of-segment row, or `None` when the chunk is exhausted. +impl RawWalk { + /// Builds the next row, or `None` when the chunk is exhausted. /// - /// The chunk fields are walked three times so no per-row temporary vectors are allocated; the - /// offsets advance only after the row measure succeeded. - fn next_row(&mut self) -> Result> { - if self.pos >= self.chunk.time.len() { + /// The chunk fields are walked twice: once to find how many positions every field can cover, and + /// once to slice them. A row does collect one slice per field, so the saving here is not + /// allocation-free rows but no row spanning two segments of one field, which is what keeps the + /// slice arithmetic per row a single offset. The offsets advance only after the row measure + /// succeeded. + fn next_row(&mut self) -> Result> { + let Self { + chunk, + segments, + offsets, + pos, + } = self; + if *pos >= chunk.time.len() { return Ok(None); } - let time_axis = self.chunk.time.as_buffer(); - let mut take = time_axis.len() - self.pos; - for (index, field) in self.chunk.fields.iter().enumerate() { - let part = segment(field, self.segments[index])?; - take = take.min(part.len().saturating_sub(self.offsets[index])); + let time_axis = chunk.time.as_buffer(); + let mut take = time_axis.len() - *pos; + for (index, field) in chunk.fields.iter().enumerate() { + let part = segment(field, segments[index])?; + take = take.min(part.len().saturating_sub(offsets[index])); } if take == 0 { return invalid_series("empty sample segment"); } - let mut samples = Vec::with_capacity(self.chunk.fields.len()); - for (index, field) in self.chunk.fields.iter().enumerate() { - let part = segment(field, self.segments[index])?; - samples.push(part.slice(self.offsets[index], take)?); + let mut samples = Vec::with_capacity(chunk.fields.len()); + for (index, field) in chunk.fields.iter().enumerate() { + let part = segment(field, segments[index])?; + samples.push(part.slice(offsets[index], take)?); } - let mut row = RawFrameRow { - primary_key: self.chunk.primary_key.clone(), - metric: self.chunk.metric.clone(), - is_last: self.chunk.is_last && self.pos + take == self.chunk.time.len(), - times: time_axis.slice(self.pos, take), + let mut row = FrameRow { + key: Some(chunk.primary_key.clone()), + metric: chunk.metric.clone(), + is_last: chunk.is_last && *pos + take == chunk.time.len(), + time: RowTime::Timestamps(time_axis.slice(*pos, take)), samples, bytes: 0, }; row.bytes = measure_row(&row)?; - self.pos += take; - for (index, field) in self.chunk.fields.iter().enumerate() { - let length = segment(field, self.segments[index])?.len(); - self.offsets[index] += take; - if self.offsets[index] == length { - self.segments[index] += 1; - self.offsets[index] = 0; + *pos += take; + for (index, field) in chunk.fields.iter().enumerate() { + let length = segment(field, segments[index])?.len(); + offsets[index] += take; + if offsets[index] == length { + segments[index] += 1; + offsets[index] = 0; } } Ok(Some(row)) } } +impl EvaluatedWalk { + /// Builds the next row, or `None` when the chunk is exhausted. + /// + /// A row covers one `(run of carried positions, sample segment)` intersection, which is why its + /// positions are always one span: the holes between runs are not rows, so nothing is ever filled + /// in for a grid position the chunk does not hold. The `is_last` a producer declared is carried + /// by the chunk's final row only, so a consumer that sees it knows the chain ended exactly there. + fn next_row(&mut self) -> Result> { + let Self { + series, + is_last, + runs, + run, + offset, + segments, + offsets, + closing, + } = self; + if *closing { + *closing = false; + // A chunk that carries no position still owes its consumer one row: the row that says the + // chain ends here. It holds no grid position and no sample, and each field is kept as the + // producer built it - an empty float stays float, an empty histogram stays histogram - so a + // decoder sees an empty closing chunk of the same payload kind rather than a missing one or + // one silently converted. + let mut samples = Vec::with_capacity(series.fields.len()); + for field in series.fields.iter() { + samples.push(field.clone()); + } + let mut row = FrameRow { + key: None, + metric: Some(series.metric.clone()), + is_last: *is_last, + time: RowTime::Spans(Vec::new()), + samples, + bytes: 0, + }; + row.bytes = measure_row(&row)?; + return Ok(Some(row)); + } + let Some(current) = runs.get(*run) else { + return Ok(None); + }; + let remaining = current.len - *offset; + let mut take = remaining; + for (index, field) in series.fields.iter().enumerate() { + let part = segment(field, segments[index])?; + take = take.min(part.len().saturating_sub(offsets[index])); + } + if take == 0 { + return invalid_series("empty sample segment"); + } + let row_offset = current.row + *offset; + let mut samples = Vec::with_capacity(series.fields.len()); + for (index, field) in series.fields.iter().enumerate() { + let part = segment(field, segments[index])?; + samples.push(part.slice(offsets[index], take)?); + } + let mut row = FrameRow { + key: None, + metric: Some(series.metric.clone()), + is_last: *is_last && *run + 1 == runs.len() && take == remaining, + time: RowTime::Spans(vec![GridSpan::new(current.position + *offset, take)]), + samples, + bytes: 0, + }; + row.bytes = measure_row(&row)?; + *offset += take; + if *offset == current.len { + *run += 1; + *offset = 0; + } + for (index, field) in series.fields.iter().enumerate() { + let length = segment(field, segments[index])?.len(); + offsets[index] += take; + if offsets[index] == length { + segments[index] += 1; + offsets[index] = 0; + } + } + debug_assert!( + row_offset + take <= series.time.len(), + "an evaluated row exceeds the axis it walks" + ); + Ok(Some(row)) + } +} + /// The next dense segment of one field, or an error when the walk left the segment list. fn segment(field: &Samples, index: usize) -> Result<&Samples> { field @@ -708,19 +1176,21 @@ fn segment(field: &Samples, index: usize) -> Result<&Samples> { /// Packs one raw chunk, retaining the `Vec` compatibility signature. /// /// The chunk packs into bounded multi-row batches; callers that need streaming behavior should -/// drive [`RawSeriesChunkCursor`] and [`RawSeriesFrameBuilder`] directly instead of collecting. +/// drive [`SeriesFrameCursor`] and [`SeriesFrameBuilder`] directly instead of collecting. pub fn pack_raw_series_chunk( schema: &SeriesSchema, - mode: RawLabelMode, + mode: FrameLabelMode, + decode: Option<&LabelDecodeSpec>, chunk: &RawSeriesChunk, ) -> Result> { - let mut builder = RawSeriesFrameBuilder::try_new( + let mut builder = SeriesFrameBuilder::try_new( schema, mode, - RawSeriesFrameBuilder::DEFAULT_MAX_ROWS, - RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, + decode, + SeriesFrameBuilder::DEFAULT_MAX_ROWS, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, )?; - let mut cursor = RawSeriesChunkCursor::try_new(schema, mode, chunk.clone())?; + let mut cursor = SeriesFrameCursor::try_new_raw(schema, mode, chunk.clone())?; let mut batches = Vec::new(); while let Some(batch) = cursor.append_to(&mut builder)? { batches.push(batch); @@ -731,17 +1201,53 @@ pub fn pack_raw_series_chunk( Ok(batches) } -/// One pending output row: a complete key with one dense sample slice per field and its checked -/// encoded-size measure. -struct RawFrameRow { - primary_key: Buffer, +/// One pending output row: one chunk's identity, finality, time slice, and one dense sample slice per +/// field, with its checked encoded-size measure. +/// +/// This is the codec's own row, not a third runtime chunk type: `RawSeriesChunk` and `Series` are +/// what a producer and a consumer see, and this is what the encoder writes between them. +struct FrameRow { + /// The encoded storage key. A raw row always carries the key it was read under, a key-encoded + /// frame names its series only by it, and an evaluated row has none. + key: Option, + /// Complete labels, present exactly for explicit identity. metric: Option, + /// Whether this row closes the chunk chain it belongs to. is_last: bool, - times: ScalarBuffer, + /// The positions this row's samples are addressed at. + time: RowTime, samples: Vec, bytes: usize, } +/// How one row addresses the time of its samples. +enum RowTime { + /// The timestamps of this row's samples, strictly increasing within the row. + Timestamps(ScalarBuffer), + /// Runs of absolute grid positions, ordered, non-overlapping, and never zero length. + Spans(Vec), +} + +impl RowTime { + /// How many grid positions or timestamps this row's samples cover. + /// + /// The sum is checked because this is the first place a decoded row's positions are counted, and + /// a frame can name spans whose lengths overflow `usize` long before any span is validated + /// against the grid it claims to sit on. Adding them up unchecked would panic on a hostile frame + /// in a debug build and wrap in a release build, both before the grid check that should have + /// refused it. + fn len(&self) -> Result { + match self { + Self::Timestamps(timestamps) => Ok(timestamps.len()), + Self::Spans(spans) => spans.iter().try_fold(0usize, |total, span| { + total + .checked_add(span.len()) + .ok_or_else(|| err("grid span length overflow")) + }), + } + } +} + /// Per-row encoded-size measure, computed once per row when the cursor produces it. /// /// The measure dominates every cumulative offset domain the encoder writes: binary/UTF-8 child @@ -749,11 +1255,19 @@ struct RawFrameRow { /// A group total of `min(max_bytes, i32::MAX)` therefore guarantees that all Arrow offsets and /// lengths stay representable, without tracking a separate offset counter. This is a bound used /// for batching work, not the exact encoded size. -fn measure_row(row: &RawFrameRow) -> Result { - checked_offset(row.primary_key.len())?; - checked_offset(row.times.len())?; - let mut bytes = add_checked(ROW_CHARGE, row.primary_key.len())?; - bytes = add_checked(bytes, mul_checked(row.times.len(), 8)?)?; +fn measure_row(row: &FrameRow) -> Result { + let key_len = row.key.as_ref().map(Buffer::len).unwrap_or(0); + checked_offset(key_len)?; + checked_offset(row.time.len()?)?; + let mut bytes = add_checked(ROW_CHARGE, key_len)?; + bytes = match &row.time { + RowTime::Timestamps(timestamps) => add_checked(bytes, mul_checked(timestamps.len(), 8)?)?, + // Two `u64` per span, plus the list's own offsets and validity. + RowTime::Spans(spans) => add_checked( + add_checked(bytes, mul_checked(spans.len(), SPAN_CHARGE)?)?, + mul_checked(spans.len(), 2)?, + )?, + }; for samples in &row.samples { bytes = add_checked(bytes, FIELD_CHARGE)?; match samples @@ -846,11 +1360,7 @@ fn histogram_charge(values: &HistogramSamples) -> Result { /// complete table, including unreferenced entries; many segments referencing one large table can /// therefore repeat that table in the encoded batch. A single constituent child array is reused /// by clone (sharing its buffers); multiple constituents are concatenated, which copies. -fn encode_rows( - schema: &SchemaRef, - mode: RawLabelMode, - rows: &[RawFrameRow], -) -> Result { +fn encode_rows(schema: &SchemaRef, shape: FrameShape, rows: &[FrameRow]) -> Result { let count = rows.len(); let DataType::Struct(time_fields) = schema.field(2).data_type() else { return invalid_series("time schema mismatch"); @@ -861,17 +1371,72 @@ fn encode_rows( let DataType::Struct(field_types) = schema.field(3).data_type() else { return invalid_series("fields schema mismatch"); }; - let mut time_lengths = Vec::with_capacity(count); - let mut time_parts: Vec = Vec::with_capacity(count); - for row in rows { - time_lengths.push(row.times.len()); - time_parts.push(Arc::new(Int64Array::new(row.times.clone(), None))); - } - let time_child = materialize_parts(&time_parts, || { - Ok(Arc::new(Int64Array::new_null(0)) as ArrayRef) - })?; - let times = list_array(&time_lengths, time_child, time_item.as_ref().clone(), None)?; - let time = struct_array(time_fields.clone(), vec![Arc::new(times)])?; + let time = match shape.content { + FrameContent::Raw => { + let mut time_lengths = Vec::with_capacity(count); + let mut time_parts: Vec = Vec::with_capacity(count); + for row in rows { + let RowTime::Timestamps(timestamps) = &row.time else { + return invalid_series("a raw frame row carries grid positions"); + }; + time_lengths.push(timestamps.len()); + time_parts.push(Arc::new(Int64Array::new(timestamps.clone(), None))); + } + let time_child = materialize_parts(&time_parts, || { + Ok(Arc::new(Int64Array::new_null(0)) as ArrayRef) + })?; + let times = list_array(&time_lengths, time_child, time_item.as_ref().clone(), None)?; + struct_array(time_fields.clone(), vec![Arc::new(times)])? + } + FrameContent::Evaluated => { + let DataType::Struct(span_fields) = time_item.data_type() else { + return invalid_series("grid span schema mismatch"); + }; + let mut span_lengths = Vec::with_capacity(count); + let mut starts = Vec::new(); + let mut lengths = Vec::new(); + for row in rows { + let RowTime::Spans(spans) = &row.time else { + return invalid_series("an evaluated frame row carries timestamps"); + }; + span_lengths.push(spans.len()); + for span in spans { + starts.push(u64::try_from(span.start()).map_err(|_| { + err("a grid position does not fit the frame's unsigned span") + })?); + lengths.push(u64::try_from(span.len()).map_err(|_| { + err("a grid span length does not fit the frame's unsigned span") + })?); + } + } + // A batch with no span at all still needs a typed child, so the empty case is spelled + // out instead of being left to whatever `concat` does with nothing. + let span_struct = if starts.is_empty() { + struct_array( + span_fields.clone(), + vec![ + Arc::new(UInt64Array::new_null(0)), + Arc::new(UInt64Array::new_null(0)), + ], + )? + } else { + struct_array( + span_fields.clone(), + vec![ + Arc::new(UInt64Array::from(starts)), + Arc::new(UInt64Array::from(lengths)), + ], + )? + }; + let spans = list_array( + &span_lengths, + Arc::new(span_struct), + time_item.as_ref().clone(), + None, + )?; + struct_array(time_fields.clone(), vec![Arc::new(spans)])? + } + }; let mut empty_table: Option = None; let mut field_arrays = Vec::with_capacity(field_types.len()); @@ -990,14 +1555,14 @@ fn encode_rows( let fields = struct_array(field_types.clone(), field_arrays)?; let mut columns: Vec = vec![ - Arc::new(binary_array(rows)?), + Arc::new(binary_array(rows, shape)?), Arc::new(BooleanArray::from( rows.iter().map(|row| row.is_last).collect::>(), )), Arc::new(time), Arc::new(fields), ]; - if mode == RawLabelMode::Explicit { + if shape.labels == FrameLabelMode::Explicit { columns.push(Arc::new( rows.iter() .map(|row| row.metric.as_ref().and_then(SeriesLabels::name)) @@ -1045,28 +1610,55 @@ fn encode_rows( )) } -/// Row-level binary keys; keys are copied exactly like the previous single-row encoder. -fn binary_array(rows: &[RawFrameRow]) -> Result { +/// Row-level binary keys. +/// +/// A raw row always carries the key its chunk was read under, so the column is complete and +/// non-null for a key frame and for a raw explicit frame alike; only an evaluated row has none, and +/// it is a null entry rather than an empty key. A null and `b""` stay distinct, because the second is +/// a key a producer could hold and the first is the absence of one. +fn binary_array(rows: &[FrameRow], shape: FrameShape) -> Result { let mut offsets = Vec::with_capacity(rows.len() + 1); let mut total = 0usize; + let mut valid = Vec::with_capacity(rows.len()); offsets.push(0); for row in rows { - total = add_checked(total, row.primary_key.len())?; + match (&row.key, shape) { + (Some(key), _) => { + total = add_checked(total, key.len())?; + valid.push(true); + } + (None, shape) if shape.content == FrameContent::Raw => { + // `RawSeriesChunk` still carries the key it was read under, and an explicit frame + // only adds the decoded labels beside it. Dropping the key would lose the identity + // a consumer of the raw chunk needs. + return invalid_series("a raw frame row carries no key"); + } + (None, _) => valid.push(false), + } offsets.push(checked_offset(total)?); } let mut values = Vec::with_capacity(total); for row in rows { - values.extend_from_slice(row.primary_key.as_slice()); + if let Some(key) = &row.key { + values.extend_from_slice(key.as_slice()); + } } + let nulls = if valid.iter().all(|valid| *valid) { + None + } else { + validity(&valid) + }; arrow(BinaryArray::try_new( OffsetBuffer::new(ScalarBuffer::from(offsets)), Buffer::from_vec(values), - None, + nulls, )) } /// Per-row charge for key/time/nullable overhead shared by every row. const ROW_CHARGE: usize = 64; +/// Per-span charge for the two 64-bit coordinates one grid span carries. +const SPAN_CHARGE: usize = 16; /// Per-field charge for the payload struct, nested list and validity overhead. const FIELD_CHARGE: usize = 64; /// Conservative per-table-row charge: the fixed fourteen-column histogram schema has seven @@ -1149,172 +1741,298 @@ fn mul_checked(len: usize, width: usize) -> Result { } /// Reuses one prepared frame schema while decoding all batches of a stream. -pub struct RawSeriesFrameDecoder { +/// +/// The decoder is built from the schema a consumer expects and then checks every batch against it: +/// the reserved declarations are compared as text, the column layout as Arrow fields, and every row +/// on its own. A batch that disagrees anywhere is an error rather than a partial decode, because a +/// receiver has no way to tell which half of a contradictory frame is the true one. +pub struct SeriesFrameDecoder { schema: SeriesSchema, - mode: RawLabelMode, + shape: FrameShape, expected: SchemaRef, } -impl RawSeriesFrameDecoder { - /// Builds the expected frame schema once from the series schema and label mode. - pub fn try_new(schema: &SeriesSchema, mode: RawLabelMode) -> Result { +impl SeriesFrameDecoder { + /// Builds the expected frame schema once from the series schema and identity declaration. + pub fn try_new( + schema: &SeriesSchema, + labels: FrameLabelMode, + decode: Option<&LabelDecodeSpec>, + ) -> Result { Ok(Self { schema: schema.clone(), - mode, - expected: raw_series_frame_schema(schema, mode)?, + shape: FrameShape::of(schema, labels), + expected: series_frame_schema(schema, labels, decode)?, }) } + /// Decodes all rows of one batch as raw chunks. + pub fn decode_raw(&self, batch: &RecordBatch) -> Result> { + if self.shape.content != FrameContent::Raw { + return invalid_series("a raw frame decoder cannot read an evaluated frame"); + } + let rows = self.decode_rows(batch)?; + let mut out = Vec::with_capacity(rows.len()); + for row in rows { + let RowTime::Timestamps(timestamps) = row.time else { + return invalid_series("a raw frame row carries grid positions"); + }; + let key = row + .key + .ok_or_else(|| err("a raw frame row carries no key"))?; + let time = RawTimestamps::try_new(timestamps)?; + let fields = SeriesFields::try_from_vec(row.samples)?; + out.push(RawSeriesChunk::try_new( + &self.schema, + key, + row.metric, + time, + fields, + row.is_last, + )?); + } + Ok(out) + } + + /// Decodes all rows of one batch as evaluated series and their finality. + /// + /// The pair is what a compute consumer needs and nothing more: the values on their absolute grid + /// positions, and whether this chunk closed the chain. A row that carries no position is kept, so + /// a closing chunk with no value still reaches the consumer that has to see it. + pub fn decode_evaluated(&self, batch: &RecordBatch) -> Result> { + if self.shape.content != FrameContent::Evaluated { + return invalid_series("an evaluated frame decoder cannot read a raw frame"); + } + let grid = self + .schema + .eval_grid + .clone() + .ok_or_else(|| err("an evaluated frame needs an evaluation grid"))?; + let rows = self.decode_rows(batch)?; + let mut out = Vec::with_capacity(rows.len()); + for row in rows { + let RowTime::Spans(spans) = row.time else { + return invalid_series("an evaluated frame row carries timestamps"); + }; + let metric = row + .metric + .ok_or_else(|| err("an evaluated frame row carries no labels"))?; + let time = TimeAxis::from_grid_spans(grid.clone(), spans)?; + // A row that covers no position is the closing chunk's shape and nothing else. The + // streaming validator refuses one that closes nothing, but this decoder is public and a + // caller driving it directly must not be handed a chunk it could never have produced + // either: an empty chunk that opens a chain carries no value and no finality, so no + // consumer could act on it. + if time.is_empty() && !row.is_last { + return invalid_series( + "an evaluated chunk without positions must close its series", + ); + } + let fields = SeriesFields::try_from_vec(row.samples)?; + out.push(( + Series::try_new(&self.schema, metric, time, fields)?, + row.is_last, + )); + } + Ok(out) + } + /// Decodes all rows of one batch, checking its reserved metadata, declared fields, and rows. - pub fn decode(&self, batch: &RecordBatch) -> Result> { - decode_raw_series_frame(&self.schema, self.mode, &self.expected, batch) + fn decode_rows(&self, batch: &RecordBatch) -> Result> { + let actual = batch.schema(); + for key in [LAYOUT, CONTENT, LABELS, TIME, GRID, LABEL_DECODE] { + if actual.metadata().get(key) != self.expected.metadata().get(key) { + return invalid_series("frame metadata mismatch"); + } + } + // The same reservation rule the declaration parser applies, because a frame's protocol + // legality must not depend on which entry point reads it: a reserved key nobody checks would + // travel as dead metadata here while being refused a layer up. + for key in actual.metadata().keys() { + if key.starts_with("greptime.series.") && !RESERVED_KEYS.contains(&key.as_str()) { + return invalid_series("frame declares unknown reserved metadata"); + } + } + if actual.fields() != self.expected.fields() { + return invalid_series("frame schema mismatch"); + } + let cols = batch.columns(); + let keys = down::(&cols[0], "read_key")?; + let lasts = down::(&cols[1], "is_last")?; + let time_struct = down::(&cols[2], "time")?; + let fs = down::(&cols[3], "fields")?; + let (names, labels) = if self.shape.labels == FrameLabelMode::Explicit { + ( + Some(down::(&cols[4], "name")?), + Some(down::(&cols[5], "labels")?), + ) + } else { + (None, None) + }; + let mut out = Vec::with_capacity(batch.num_rows()); + for row in 0..batch.num_rows() { + if lasts.is_null(row) || time_struct.is_null(row) || fs.is_null(row) { + return invalid_series("null required frame parent"); + } + match self.shape.labels { + FrameLabelMode::Key => { + if keys.is_null(row) { + return invalid_series("null required frame parent"); + } + } + // An explicit row names its series by label; the key column is the raw chunk's own + // contract, and an evaluated row has none at all. + FrameLabelMode::Explicit => { + if self.shape.content == FrameContent::Raw && keys.is_null(row) { + return invalid_series("a raw explicit frame row carries no key"); + } + if self.shape.content == FrameContent::Evaluated && !keys.is_null(row) { + return invalid_series("an evaluated frame row carries a storage key"); + } + } + } + let time = match self.shape.content { + FrameContent::Raw => { + let times = down::(time_struct.column(0), "timestamps")?; + if times.is_null(row) { + return invalid_series("null required raw frame parent"); + } + let values = times.value(row); + let tv = down::(&values, "timestamp values")?; + if tv.null_count() != 0 { + return invalid_series("null timestamp"); + } + RowTime::Timestamps(tv.values().clone()) + } + FrameContent::Evaluated => { + let typed = down::(time_struct.column(0), "spans")?; + if typed.is_null(row) { + return invalid_series("null required evaluated frame parent"); + } + let values = typed.value(row); + let span_struct = down::(&values, "spans")?; + if span_struct.null_count() != 0 { + return invalid_series("null grid span"); + } + let starts = down::(span_struct.column(0), "span starts")?; + let lengths = down::(span_struct.column(1), "span lengths")?; + if starts.null_count() != 0 || lengths.null_count() != 0 { + return invalid_series("null grid span bound"); + } + let mut spans = Vec::with_capacity(starts.len()); + for index in 0..starts.len() { + let start = usize::try_from(starts.value(index)) + .map_err(|_| err("grid span start exceeds the supported range"))?; + let len = usize::try_from(lengths.value(index)) + .map_err(|_| err("grid span length exceeds the supported range"))?; + if len == 0 { + return invalid_series("grid spans must not be empty"); + } + spans.push(GridSpan::new(start, len)); + } + RowTime::Spans(spans) + } + }; + let positions = time.len()?; + let metric = match (names, labels) { + (None, None) => None, + (Some(ns), Some(ls)) => { + if ls.is_null(row) { + return invalid_series("null labels list"); + } + let label_values = ls.value(row); + let label_struct = down::(&label_values, "labels")?; + if label_struct.null_count() != 0 { + return invalid_series("null label struct element"); + } + let ln = down::(label_struct.column(0), "label names")?; + let lv = down::(label_struct.column(1), "label values")?; + if ln.null_count() != 0 || lv.null_count() != 0 { + return invalid_series("null label element"); + } + let tags = (0..ln.len()) + .map(|i| Label { + name: Arc::from(ln.value(i)), + value: Arc::from(lv.value(i)), + }) + .collect(); + Some(SeriesLabels::try_new( + (!ns.is_null(row)).then(|| Arc::from(ns.value(row))), + tags, + )?) + } + _ => return invalid_series("incomplete explicit label columns"), + }; + let mut decoded = Vec::with_capacity(fs.num_columns()); + for col in fs.columns() { + let payload = down::(col, "field payload")?; + if payload.is_null(row) { + return invalid_series("null field payload"); + } + let floats_list = down::(payload.column(0), "float values")?; + let hist = down::(payload.column(1), "histogram payload")?; + let has_float = !floats_list.is_null(row); + let has_hist = !hist.is_null(row); + if !has_float && !has_hist { + return invalid_series("field has neither float nor histogram payload"); + } + let floats = if has_float { + down::(&floats_list.value(row), "float values")?.clone() + } else { + Float64Array::new_null(0) + }; + if has_float && floats.len() != positions { + return invalid_series("float payload length mismatch"); + } + let samples = if !has_hist { + Samples::from_float(floats) + } else { + let inds = down::(hist.column(0), "histogram indices")?; + let tables = down::(hist.column(1), "histogram table")?; + if inds.is_null(row) || tables.is_null(row) { + return invalid_series("null histogram indices or table"); + } + let indices = + down::(&inds.value(row), "histogram indices")?.clone(); + if indices.len() != positions { + return invalid_series("histogram index length mismatch"); + } + let table = down::(&tables.value(row), "histogram table")?.clone(); + let batch = Arc::new(HistogramBatch::try_from_arrow(table)?); + if has_float { + Samples::try_from_mixed(floats, HistogramSamples::try_new(indices, batch)?)? + } else { + Samples::try_from_histogram(indices, batch)? + } + }; + decoded.push(samples); + } + let key = if keys.is_null(row) { + None + } else { + Some(binary_slice(keys, row)?) + }; + out.push(FrameRow { + key, + metric, + is_last: lasts.value(row), + time, + samples: decoded, + bytes: 0, + }); + } + Ok(out) } } -/// Decodes all rows of a raw frame; any malformed row makes the entire call fail. +/// Decodes all rows of one raw frame; any malformed row makes the entire call fail. pub fn unpack_raw_series_frame( schema: &SeriesSchema, - mode: RawLabelMode, + mode: FrameLabelMode, + decode: Option<&LabelDecodeSpec>, batch: &RecordBatch, ) -> Result> { - RawSeriesFrameDecoder::try_new(schema, mode)?.decode(batch) -} - -fn decode_raw_series_frame( - schema: &SeriesSchema, - mode: RawLabelMode, - expected: &SchemaRef, - batch: &RecordBatch, -) -> Result> { - let actual = batch.schema(); - for key in [LAYOUT, CONTENT, LABELS, TIME] { - if actual.metadata().get(key) != expected.metadata().get(key) { - return invalid_series("raw frame metadata mismatch"); - } - } - if actual - .metadata() - .keys() - .any(|key| key == GRID || key.starts_with("greptime.series.grid.")) - || actual.fields() != expected.fields() - { - return invalid_series("raw frame schema mismatch"); - } - let cols = batch.columns(); - let keys = down::(&cols[0], "read_key")?; - let lasts = down::(&cols[1], "is_last")?; - let time_struct = down::(&cols[2], "time")?; - let times = down::(time_struct.column(0), "timestamps")?; - let fs = down::(&cols[3], "fields")?; - let (names, labels) = if mode == RawLabelMode::Explicit { - ( - Some(down::(&cols[4], "name")?), - Some(down::(&cols[5], "labels")?), - ) - } else { - (None, None) - }; - let mut out = Vec::with_capacity(batch.num_rows()); - for row in 0..batch.num_rows() { - if keys.is_null(row) - || lasts.is_null(row) - || time_struct.is_null(row) - || times.is_null(row) - || fs.is_null(row) - { - return invalid_series("null required raw frame parent"); - } - let timestamp_values = times.value(row); - let tv = down::(×tamp_values, "timestamp values")?; - if tv.null_count() != 0 { - return invalid_series("null timestamp"); - } - let time = tv.values().clone(); - let metric = match (names, labels) { - (None, None) => None, - (Some(ns), Some(ls)) => { - if ls.is_null(row) { - return invalid_series("null labels list"); - } - let label_values = ls.value(row); - let label_struct = down::(&label_values, "labels")?; - if label_struct.null_count() != 0 { - return invalid_series("null label struct element"); - } - let ln = down::(label_struct.column(0), "label names")?; - let lv = down::(label_struct.column(1), "label values")?; - if ln.null_count() != 0 || lv.null_count() != 0 { - return invalid_series("null label element"); - } - let tags = (0..ln.len()) - .map(|i| Label { - name: Arc::from(ln.value(i)), - value: Arc::from(lv.value(i)), - }) - .collect(); - Some(SeriesLabels::try_new( - (!ns.is_null(row)).then(|| Arc::from(ns.value(row))), - tags, - )?) - } - _ => return invalid_series("incomplete explicit label columns"), - }; - let mut decoded = Vec::with_capacity(fs.num_columns()); - for col in fs.columns() { - let payload = down::(col, "field payload")?; - if payload.is_null(row) { - return invalid_series("null field payload"); - } - let floats_list = down::(payload.column(0), "float values")?; - let hist = down::(payload.column(1), "histogram payload")?; - let has_float = !floats_list.is_null(row); - let has_hist = !hist.is_null(row); - if !has_float && !has_hist { - return invalid_series("field has neither float nor histogram payload"); - } - let floats = if has_float { - down::(&floats_list.value(row), "float values")?.clone() - } else { - Float64Array::new_null(0) - }; - if has_float && floats.len() != time.len() { - return invalid_series("float payload length mismatch"); - } - let samples = if !has_hist { - Samples::from_float(floats) - } else { - let inds = down::(hist.column(0), "histogram indices")?; - let tables = down::(hist.column(1), "histogram table")?; - if inds.is_null(row) || tables.is_null(row) { - return invalid_series("null histogram indices or table"); - } - let indices = down::(&inds.value(row), "histogram indices")?.clone(); - if indices.len() != time.len() { - return invalid_series("histogram index length mismatch"); - } - let table = down::(&tables.value(row), "histogram table")?.clone(); - let batch = Arc::new(HistogramBatch::try_from_arrow(table)?); - if has_float { - Samples::try_from_mixed(floats, HistogramSamples::try_new(indices, batch)?)? - } else { - Samples::try_from_histogram(indices, batch)? - } - }; - decoded.push(samples); - } - let fields = SeriesFields::try_from_vec(decoded)?; - let key = binary_slice(keys, row)?; - let raw_time = RawTimestamps::try_new(time)?; - out.push(RawSeriesChunk::try_new( - schema, - key, - metric, - raw_time, - fields, - lasts.value(row), - )?); - } - Ok(out) + SeriesFrameDecoder::try_new(schema, mode, decode)?.decode_raw(batch) } fn binary_slice(array: &BinaryArray, row: usize) -> Result { diff --git a/src/datatypes/src/series/frame/tests.rs b/src/datatypes/src/series/frame/tests.rs index 5356ebb84dc..3afe2a9a595 100644 --- a/src/datatypes/src/series/frame/tests.rs +++ b/src/datatypes/src/series/frame/tests.rs @@ -115,7 +115,7 @@ fn histogram_batch_with_sum(sum: f64) -> Arc DataType { - raw_series_frame_schema(schema, RawLabelMode::Key) + series_frame_schema(schema, FrameLabelMode::Key, Some(&dense_description())) .unwrap() .field(3) .data_type() @@ -253,13 +253,16 @@ fn assert_float_bits(actual: &Samples, expected: &[Option]) { /// single-chunk wrapper: every batch the walk produces, then the final flush. fn pack_chunk_with_budget( schema: &SeriesSchema, - mode: RawLabelMode, + mode: FrameLabelMode, chunk: &RawSeriesChunk, max_rows: usize, max_bytes: usize, ) -> crate::error::Result> { - let mut builder = RawSeriesFrameBuilder::try_new(schema, mode, max_rows, max_bytes)?; - let mut cursor = RawSeriesChunkCursor::try_new(schema, mode, chunk.clone())?; + // Only a key frame declares a label decode description; explicit frames carry their labels. + let decode = (mode == FrameLabelMode::Key).then(dense_description); + let mut builder = + SeriesFrameBuilder::try_new(schema, mode, decode.as_ref(), max_rows, max_bytes)?; + let mut cursor = SeriesFrameCursor::try_new_raw(schema, mode, chunk.clone())?; let mut batches = Vec::new(); while let Some(batch) = cursor.append_to(&mut builder)? { batches.push(batch); @@ -273,12 +276,12 @@ fn pack_chunk_with_budget( /// Advances one owned chunk through an existing builder. A partially filled buffer stays buffered /// instead of becoming a batch of its own, which is what makes cross-chunk grouping observable. fn append_chunk( - builder: &mut RawSeriesFrameBuilder, + builder: &mut SeriesFrameBuilder, schema: &SeriesSchema, - mode: RawLabelMode, + mode: FrameLabelMode, chunk: &RawSeriesChunk, ) -> crate::error::Result> { - let mut cursor = RawSeriesChunkCursor::try_new(schema, mode, chunk.clone())?; + let mut cursor = SeriesFrameCursor::try_new_raw(schema, mode, chunk.clone())?; let mut batches = Vec::new(); while let Some(batch) = cursor.append_to(builder)? { batches.push(batch); @@ -289,13 +292,14 @@ fn append_chunk( /// Decodes every batch with one reused decoder, so repeated calls exercise the cached schema. fn decode_batches( schema: &SeriesSchema, - mode: RawLabelMode, + mode: FrameLabelMode, batches: &[RecordBatch], ) -> Vec { - let decoder = RawSeriesFrameDecoder::try_new(schema, mode).unwrap(); + let decode = (mode == FrameLabelMode::Key).then(dense_description); + let decoder = SeriesFrameDecoder::try_new(schema, mode, decode.as_ref()).unwrap(); batches .iter() - .flat_map(|batch| decoder.decode(batch).unwrap()) + .flat_map(|batch| decoder.decode_raw(batch).unwrap()) .collect() } @@ -376,16 +380,22 @@ fn sliced_histogram_frames_densify_without_duplicating_shared_table() { // paths below stay observable; the default budget coalesces the same rows later on. let frames = pack_chunk_with_budget( &schema, - RawLabelMode::Key, + FrameLabelMode::Key, &chunk, 1, - RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, ) .unwrap(); assert_eq!(frames.len(), 64); let mut decoded_chunks = Vec::with_capacity(frames.len()); for frame in &frames { - let decoded = unpack_raw_series_frame(&schema, RawLabelMode::Key, frame).unwrap(); + let decoded = unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + frame, + ) + .unwrap(); assert_eq!(decoded.len(), 1); assert_eq!(decoded[0].time.len(), 1); decoded_chunks.push(decoded.into_iter().next().unwrap()); @@ -419,10 +429,16 @@ fn sliced_histogram_frames_densify_without_duplicating_shared_table() { // The default budget groups the same 64 rows into one multirow batch; every row still decodes // to its own slice of the shared table with bit-exact values. - let coalesced = pack_raw_series_chunk(&schema, RawLabelMode::Key, &chunk).unwrap(); + let coalesced = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &chunk, + ) + .unwrap(); assert_eq!(coalesced.len(), 1); assert_eq!(coalesced[0].num_rows(), 64); - let coalesced_rows = decode_batches(&schema, RawLabelMode::Key, &coalesced); + let coalesced_rows = decode_batches(&schema, FrameLabelMode::Key, &coalesced); assert_eq!(coalesced_rows.len(), 64); for (position, row) in coalesced_rows.iter().enumerate() { assert_eq!(row.primary_key.as_slice(), &[0x42]); @@ -478,8 +494,9 @@ fn raw_frame_field_names_bind_payloads_to_their_schema() { ]) .unwrap(); - for mode in [RawLabelMode::Key, RawLabelMode::Explicit] { - let metric = (mode == RawLabelMode::Explicit).then(|| metric(Some("host"), &[])); + for mode in [FrameLabelMode::Key, FrameLabelMode::Explicit] { + let decode = (mode == FrameLabelMode::Key).then(dense_description); + let metric = (mode == FrameLabelMode::Explicit).then(|| metric(Some("host"), &[])); let chunk = RawSeriesChunk::try_new( &schema, Buffer::from(vec![0xaa_u8]), @@ -489,7 +506,7 @@ fn raw_frame_field_names_bind_payloads_to_their_schema() { true, ) .unwrap(); - let batch = &pack_raw_series_chunk(&schema, mode, &chunk).unwrap()[0]; + let batch = &pack_raw_series_chunk(&schema, mode, decode.as_ref(), &chunk).unwrap()[0]; let payloads = batch .column(3) .as_any() @@ -504,7 +521,7 @@ fn raw_frame_field_names_bind_payloads_to_their_schema() { ["cpu_usage", "cpu_idle"] ); - let decoded = unpack_raw_series_frame(&schema, mode, batch).unwrap(); + let decoded = unpack_raw_series_frame(&schema, mode, decode.as_ref(), batch).unwrap(); assert_eq!(decoded[0].primary_key.as_slice(), &[0xaa]); assert!(decoded[0].is_last); let times = decoded[0].time.as_buffer(); @@ -513,19 +530,35 @@ fn raw_frame_field_names_bind_payloads_to_their_schema() { assert_float_bits(decoded[0].fields.get(1).unwrap(), &[Some(10.0), Some(20.0)]); assert!( - unpack_raw_series_frame(&test_schema(&["cpu_idle", "cpu_usage"]), mode, batch).is_err() + unpack_raw_series_frame( + &test_schema(&["cpu_idle", "cpu_usage"]), + mode, + decode.as_ref(), + batch + ) + .is_err() ); assert!( - unpack_raw_series_frame(&test_schema(&["cpu_usage", "cpu_busy"]), mode, batch).is_err() + unpack_raw_series_frame( + &test_schema(&["cpu_usage", "cpu_busy"]), + mode, + decode.as_ref(), + batch + ) + .is_err() ); } } +/// A key frame declares the label decode description that makes its `read_key` readable, so its +/// metadata carries that text; an explicit frame declares none, and its `read_key` column is +/// nullable because an evaluated explicit row carries no storage key at all. #[test] fn schema_and_key_explicit_round_trips_preserve_identity_and_bits() { let schema = test_schema(&["value"]); - let key_schema = raw_series_frame_schema(&schema, RawLabelMode::Key).unwrap(); - let explicit_schema = raw_series_frame_schema(&schema, RawLabelMode::Explicit).unwrap(); + let key_schema = + series_frame_schema(&schema, FrameLabelMode::Key, Some(&dense_description())).unwrap(); + let explicit_schema = raw_series_frame_schema(&schema, FrameLabelMode::Explicit).unwrap(); let payload = DataType::Struct( vec![ Field::new( @@ -560,10 +593,16 @@ fn schema_and_key_explicit_round_trips_preserve_identity_and_bits() { .into(), ); let mut expected_key_metadata = std::collections::HashMap::new(); - expected_key_metadata.insert("greptime.series.layout".to_string(), "v1".to_string()); + expected_key_metadata.insert("greptime.series.layout".to_string(), "v2".to_string()); expected_key_metadata.insert("greptime.series.content".to_string(), "raw".to_string()); expected_key_metadata.insert("greptime.series.labels".to_string(), "key".to_string()); expected_key_metadata.insert("greptime.series.time".to_string(), "timestamps".to_string()); + // A key frame also declares the description that decodes its labels. Its exact text is pinned by + // the metadata fixture, so it is spelled here by the same serializer the writer uses. + expected_key_metadata.insert( + LABEL_DECODE.to_string(), + serde_json::to_string(&dense_description()).unwrap(), + ); let expected_key = Schema::new_with_metadata( vec![ Field::new("read_key", DataType::Binary, false), @@ -589,7 +628,10 @@ fn schema_and_key_explicit_round_trips_preserve_identity_and_bits() { expected_key_metadata, ); assert_eq!(key_schema.as_ref(), &expected_key); + // An explicit frame still has the key column, but only a raw explicit row carries a key: an + // evaluated one has none, so the column is nullable here while a key frame's is required. let mut explicit_fields = expected_key.fields().iter().cloned().collect::>(); + explicit_fields[0] = Field::new("read_key", DataType::Binary, true).into(); explicit_fields.push(Field::new("name", DataType::Utf8, true).into()); explicit_fields.push( Field::new( @@ -611,6 +653,8 @@ fn schema_and_key_explicit_round_trips_preserve_identity_and_bits() { ); let mut explicit_metadata = expected_key.metadata().clone(); explicit_metadata.insert("greptime.series.labels".to_string(), "explicit".to_string()); + // An explicit frame carries complete labels, so it declares no decode description. + explicit_metadata.remove(LABEL_DECODE); let expected_explicit = Schema::new_with_metadata(explicit_fields, explicit_metadata); assert_eq!(explicit_schema.as_ref(), &expected_explicit); @@ -625,9 +669,21 @@ fn schema_and_key_explicit_round_trips_preserve_identity_and_bits() { values.to_vec(), true, ); - let packed = pack_raw_series_chunk(&schema, RawLabelMode::Key, &native).unwrap(); + let packed = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &native, + ) + .unwrap(); assert_eq!(packed.len(), 1); - let decoded = unpack_raw_series_frame(&schema, RawLabelMode::Key, &packed[0]).unwrap(); + let decoded = unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &packed[0], + ) + .unwrap(); assert_eq!(decoded.len(), 1); assert_eq!(decoded[0].primary_key.as_slice(), &[0, 0xff, 0, 0x80]); assert!(decoded[0].metric.is_none()); @@ -648,20 +704,45 @@ fn schema_and_key_explicit_round_trips_preserve_identity_and_bits() { vec![Some(1.25)], false, ); - let packed = pack_raw_series_chunk(&schema, RawLabelMode::Explicit, &explicit).unwrap(); - let decoded = unpack_raw_series_frame(&schema, RawLabelMode::Explicit, &packed[0]).unwrap(); + let packed = pack_raw_series_chunk(&schema, FrameLabelMode::Explicit, None, &explicit).unwrap(); + let decoded = + unpack_raw_series_frame(&schema, FrameLabelMode::Explicit, None, &packed[0]).unwrap(); assert_eq!(decoded[0].primary_key.as_slice(), &[0x80, 0, 0xff]); assert!(!decoded[0].is_last); let labels = decoded[0].metric.as_ref().unwrap(); assert_eq!(labels.name(), None); assert_eq!(labels.get("rack"), Some("")); assert_eq!(labels.get("zone"), Some("west")); - assert!(pack_raw_series_chunk(&schema, RawLabelMode::Key, &explicit).is_err()); - assert!(pack_raw_series_chunk(&schema, RawLabelMode::Explicit, &native).is_err()); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &packed[0]).is_err()); + assert!( + pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &explicit + ) + .is_err() + ); + assert!(pack_raw_series_chunk(&schema, FrameLabelMode::Explicit, None, &native).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &packed[0] + ) + .is_err() + ); - let key_packed = pack_raw_series_chunk(&schema, RawLabelMode::Key, &native).unwrap(); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Explicit, &key_packed[0]).is_err()); + let key_packed = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &native, + ) + .unwrap(); + assert!( + unpack_raw_series_frame(&schema, FrameLabelMode::Explicit, None, &key_packed[0]).is_err() + ); for name in [None, Some(""), Some("cpu")] { let named = float_chunk( &schema, @@ -671,9 +752,10 @@ fn schema_and_key_explicit_round_trips_preserve_identity_and_bits() { vec![Some(2.0)], false, ); - let encoded = pack_raw_series_chunk(&schema, RawLabelMode::Explicit, &named).unwrap(); + let encoded = + pack_raw_series_chunk(&schema, FrameLabelMode::Explicit, None, &named).unwrap(); let restored = - unpack_raw_series_frame(&schema, RawLabelMode::Explicit, &encoded[0]).unwrap(); + unpack_raw_series_frame(&schema, FrameLabelMode::Explicit, None, &encoded[0]).unwrap(); assert_eq!(restored[0].metric.as_ref().unwrap().name(), name); assert_eq!(restored[0].metric.as_ref().unwrap().get("empty"), Some("")); } @@ -714,7 +796,13 @@ fn segmented_mixed_fields_emit_every_aligned_position_and_marker() { .unwrap(); // Variable field boundaries now group every aligned interval into one multirow batch instead // of one batch per boundary, so decode the actual rows that batch carries. - let frames = pack_raw_series_chunk(&schema, RawLabelMode::Key, &chunk).unwrap(); + let frames = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &chunk, + ) + .unwrap(); assert_eq!(frames.len(), 1); assert_eq!(frames[0].num_rows(), 4); let mut observed_times = Vec::new(); @@ -722,7 +810,13 @@ fn segmented_mixed_fields_emit_every_aligned_position_and_marker() { let mut observed_second = Vec::new(); let mut observed_keys = Vec::new(); let mut observed_markers = Vec::new(); - let rows = unpack_raw_series_frame(&schema, RawLabelMode::Key, &frames[0]).unwrap(); + let rows = unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &frames[0], + ) + .unwrap(); assert_eq!(rows.len(), 4); for (row_index, row) in rows.iter().enumerate() { assert_eq!(row.is_last, row_index == rows.len() - 1); @@ -795,12 +889,22 @@ fn segmented_mixed_fields_emit_every_aligned_position_and_marker() { true, ) .unwrap(); - let aligned_frames = - pack_raw_series_chunk(&aligned_schema, RawLabelMode::Key, &aligned_chunk).unwrap(); + let aligned_frames = pack_raw_series_chunk( + &aligned_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &aligned_chunk, + ) + .unwrap(); assert_eq!(aligned_frames.len(), 1); assert_eq!(aligned_frames[0].num_rows(), 3); - let aligned_rows = - unpack_raw_series_frame(&aligned_schema, RawLabelMode::Key, &aligned_frames[0]).unwrap(); + let aligned_rows = unpack_raw_series_frame( + &aligned_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &aligned_frames[0], + ) + .unwrap(); assert_eq!(aligned_rows.len(), 3); let mut interval_lengths = Vec::new(); let mut aligned_times = Vec::new(); @@ -896,12 +1000,23 @@ fn segmented_mixed_fields_emit_every_aligned_position_and_marker() { ) .unwrap(); distinct_rows.push( - pack_raw_series_chunk(&alternate_schema, RawLabelMode::Key, &alternate) - .unwrap() - .remove(0), + pack_raw_series_chunk( + &alternate_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &alternate, + ) + .unwrap() + .remove(0), ); let multirow = concat_batches(&frames[0].schema(), &distinct_rows).unwrap(); - let decoded = unpack_raw_series_frame(&schema, RawLabelMode::Key, &multirow).unwrap(); + let decoded = unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &multirow, + ) + .unwrap(); assert_eq!(decoded.len(), 5); assert_eq!( decoded.iter().map(|row| row.is_last).collect::>(), @@ -929,7 +1044,13 @@ fn segmented_mixed_fields_emit_every_aligned_position_and_marker() { Some(&[0xed][..]), ])); let sliced = arrow::record_batch::RecordBatch::try_new(sliced.schema(), altered).unwrap(); - let sliced_rows = unpack_raw_series_frame(&schema, RawLabelMode::Key, &sliced).unwrap(); + let sliced_rows = unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &sliced, + ) + .unwrap(); assert_eq!( sliced_rows .iter() @@ -975,10 +1096,24 @@ fn segmented_mixed_fields_emit_every_aligned_position_and_marker() { false, ) .unwrap(); - let not_last_frames = pack_raw_series_chunk(&schema, RawLabelMode::Key, ¬_last).unwrap(); + let not_last_frames = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + ¬_last, + ) + .unwrap(); let markers = not_last_frames .iter() - .flat_map(|frame| unpack_raw_series_frame(&schema, RawLabelMode::Key, frame).unwrap()) + .flat_map(|frame| { + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + frame, + ) + .unwrap() + }) .map(|row| row.is_last) .collect::>(); assert_eq!(markers, [false; 4]); @@ -1036,8 +1171,20 @@ fn histogram_payload_and_present_all_null_representation_survive() { false, ) .unwrap(); - let frame = pack_raw_series_chunk(&schema, RawLabelMode::Key, &chunk).unwrap(); - let decoded = unpack_raw_series_frame(&schema, RawLabelMode::Key, &frame[0]).unwrap(); + let frame = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &chunk, + ) + .unwrap(); + let decoded = unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &frame[0], + ) + .unwrap(); let DenseSamplesRef::Histogram(histograms) = decoded[0].fields.get(0).unwrap().dense().unwrap() else { panic!("histogram representation was normalized"); @@ -1088,10 +1235,20 @@ fn histogram_payload_and_present_all_null_representation_survive() { false, ) .unwrap(); - let mixed_frame = - pack_raw_series_chunk(&mixed_schema, RawLabelMode::Key, &mixed_chunk).unwrap(); - let mixed_decoded = - unpack_raw_series_frame(&mixed_schema, RawLabelMode::Key, &mixed_frame[0]).unwrap(); + let mixed_frame = pack_raw_series_chunk( + &mixed_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &mixed_chunk, + ) + .unwrap(); + let mixed_decoded = unpack_raw_series_frame( + &mixed_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &mixed_frame[0], + ) + .unwrap(); let DenseSamplesRef::Mixed(mixed) = mixed_decoded[0].fields.get(0).unwrap().dense().unwrap() else { panic!("all-null mixed representation was normalized"); @@ -1103,6 +1260,9 @@ fn histogram_payload_and_present_all_null_representation_survive() { ); } +/// The encoder's own declarations are refused when they contradict the frame, and the schema with +/// no fields is refused; an evaluation grid, by contrast, now describes a legal evaluated frame +/// instead of making the schema unusable here. #[test] fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() { let schema = test_schema(&["value"]); @@ -1115,13 +1275,41 @@ fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() { .unwrap(), )), }; - assert!(raw_series_frame_schema(&grid_schema, RawLabelMode::Key).is_err()); - let empty_schema = raw_series_frame_schema(&schema, RawLabelMode::Key).unwrap(); + // A schema with an evaluation grid describes evaluated samples, which is a legal frame in this + // family: the raw-only constructor still refuses it, while the one constructor accepts it as an + // evaluated frame that declares its grid and addresses time by grid spans. + assert!(raw_series_frame_schema(&grid_schema, FrameLabelMode::Explicit).is_err()); + let evaluated = series_frame_schema(&grid_schema, FrameLabelMode::Explicit, None).unwrap(); + assert_eq!( + evaluated.metadata().get(CONTENT).map(String::as_str), + Some("evaluated") + ); + assert_eq!( + evaluated.metadata().get(TIME).map(String::as_str), + Some("grid_spans") + ); + assert!(evaluated.metadata().get(GRID).is_some()); + // An evaluated frame names its series by label, so a key identity stays refused. + assert!( + series_frame_schema( + &grid_schema, + FrameLabelMode::Key, + Some(&dense_description()) + ) + .is_err() + ); + let empty_schema = + series_frame_schema(&schema, FrameLabelMode::Key, Some(&dense_description())).unwrap(); let empty = arrow::record_batch::RecordBatch::new_empty(empty_schema.clone()); assert!( - unpack_raw_series_frame(&schema, RawLabelMode::Key, &empty) - .unwrap() - .is_empty() + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &empty + ) + .unwrap() + .is_empty() ); let wrong = arrow::record_batch::RecordBatch::new_empty(Arc::new(Schema::new(vec![Field::new( @@ -1129,12 +1317,22 @@ fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() { DataType::Utf8, false, )]))); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &wrong).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &wrong + ) + .is_err() + ); let no_fields = SeriesSchema { fields: Arc::from([]), eval_grid: None, }; - assert!(raw_series_frame_schema(&no_fields, RawLabelMode::Key).is_err()); + assert!( + series_frame_schema(&no_fields, FrameLabelMode::Key, Some(&dense_description())).is_err() + ); let mut metadata = empty_schema.metadata().clone(); metadata.insert( @@ -1146,14 +1344,35 @@ fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() { metadata, )); let empty_bad = arrow::record_batch::RecordBatch::new_empty(contaminated); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &empty_bad).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &empty_bad + ) + .is_err() + ); let chunk = float_chunk(&schema, &[1], None, vec![1], vec![Some(2.0)], false); - assert!(pack_raw_series_chunk(&grid_schema, RawLabelMode::Key, &chunk).is_err()); + assert!( + pack_raw_series_chunk( + &grid_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &chunk + ) + .is_err() + ); - let valid = pack_raw_series_chunk(&schema, RawLabelMode::Key, &chunk) - .unwrap() - .remove(0); + let valid = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &chunk, + ) + .unwrap() + .remove(0); let mut extra_metadata = valid.schema().metadata().clone(); extra_metadata.insert("application.note".to_string(), "ignored".to_string()); let unrelated = arrow::record_batch::RecordBatch::try_new( @@ -1165,9 +1384,14 @@ fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() { ) .unwrap(); assert_eq!( - unpack_raw_series_frame(&schema, RawLabelMode::Key, &unrelated) - .unwrap() - .len(), + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &unrelated + ) + .unwrap() + .len(), 1 ); @@ -1181,10 +1405,23 @@ fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() { valid.columns().to_vec(), ) .unwrap(); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &wrong).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &wrong + ) + .is_err() + ); + // A raw frame must not declare an evaluation grid: `greptime.series.grid` belongs to an + // evaluated frame only. let mut grid_metadata = valid.schema().metadata().clone(); - grid_metadata.insert("greptime.series.grid.start".to_string(), "0".to_string()); + grid_metadata.insert( + "greptime.series.grid".to_string(), + "{\"start_ms\":0,\"step_ms\":1,\"len\":3}".to_string(), + ); let contaminated = arrow::record_batch::RecordBatch::try_new( Arc::new(Schema::new_with_metadata( valid.schema().fields().clone(), @@ -1193,13 +1430,32 @@ fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() { valid.columns().to_vec(), ) .unwrap(); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &contaminated).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &contaminated + ) + .is_err() + ); + + // A reserved key this revision does not know is refused when the declarations are parsed, + // rather than travelling along as dead metadata or being silently ignored. + let mut unknown_key_metadata = valid.schema().metadata().clone(); + unknown_key_metadata.insert("greptime.series.grid.start".to_string(), "0".to_string()); + let unregistered = + Schema::new_with_metadata(valid.schema().fields().clone(), unknown_key_metadata); + assert!(series_frame_label_decode(&unregistered).is_err()); + assert!(SeriesFrameDeclaration::parse(&unregistered).is_err()); for metadata_case in ["wrong_layout", "wrong_labels", "wrong_time"] { let mut metadata = valid.schema().metadata().clone(); match metadata_case { "wrong_layout" => { - metadata.insert("greptime.series.layout".to_string(), "v2".to_string()); + // `v2` is the only layout this revision reads, so the previously unreadable `v1` + // is what a wrong layout is spelled as now. + metadata.insert("greptime.series.layout".to_string(), "v1".to_string()); } "wrong_labels" => { metadata.insert("greptime.series.labels".to_string(), "explicit".to_string()); @@ -1216,7 +1472,15 @@ fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() { valid.columns().to_vec(), ) .unwrap(); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &malformed).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &malformed + ) + .is_err() + ); } } @@ -1227,12 +1491,25 @@ fn rejects_invalid_timestamps_and_empty_raw_payloads() { let schema = test_schema(&["value"]); let chunk = float_chunk(&schema, &[1], None, vec![1], vec![Some(2.0)], false); - let valid = pack_raw_series_chunk(&schema, RawLabelMode::Key, &chunk) - .unwrap() - .remove(0); + let valid = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &chunk, + ) + .unwrap() + .remove(0); for bad_timestamps in [vec![], vec![1, 1], vec![2, 1]] { let malformed = with_timestamp_values(&valid, bad_timestamps); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &malformed).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &malformed + ) + .is_err() + ); } let mut empty_columns = valid.columns().to_vec(); let empty_time = with_timestamp_values(&valid, vec![]); @@ -1267,7 +1544,15 @@ fn rejects_invalid_timestamps_and_empty_raw_payloads() { empty_columns[3] = Arc::new(empty_fields); let empty_time_and_payload = arrow::record_batch::RecordBatch::try_new(valid.schema(), empty_columns).unwrap(); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &empty_time_and_payload).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &empty_time_and_payload + ) + .is_err() + ); let two_samples = float_chunk( &schema, @@ -1277,12 +1562,25 @@ fn rejects_invalid_timestamps_and_empty_raw_payloads() { vec![Some(2.0), Some(3.0)], false, ); - let valid_two_samples = pack_raw_series_chunk(&schema, RawLabelMode::Key, &two_samples) - .unwrap() - .remove(0); + let valid_two_samples = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &two_samples, + ) + .unwrap() + .remove(0); for bad_timestamps in [vec![1, 1], vec![2, 1]] { let malformed = with_timestamp_values(&valid_two_samples, bad_timestamps); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &malformed).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &malformed + ) + .is_err() + ); } let mut empty_payload_columns = valid.columns().to_vec(); @@ -1322,7 +1620,15 @@ fn rejects_invalid_timestamps_and_empty_raw_payloads() { empty_payload_columns[3] = Arc::new(empty_fields); let empty_payload = arrow::record_batch::RecordBatch::try_new(valid.schema(), empty_payload_columns).unwrap(); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &empty_payload).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &empty_payload + ) + .is_err() + ); let mut columns = valid.columns().to_vec(); columns[3] = Arc::new(StructArray::new_null( @@ -1348,7 +1654,15 @@ fn rejects_invalid_timestamps_and_empty_raw_payloads() { let invalid_later_row = arrow::record_batch::RecordBatch::try_new(valid.schema(), bad).unwrap(); let later_invalid = concat_batches(&valid.schema(), &[valid.clone(), invalid_later_row]).unwrap(); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &later_invalid).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &later_invalid + ) + .is_err() + ); } #[test] @@ -1362,9 +1676,14 @@ fn rejects_invalid_explicit_labels() { vec![Some(1.0)], false, ); - let explicit = pack_raw_series_chunk(&explicit_schema, RawLabelMode::Explicit, &explicit_chunk) - .unwrap() - .remove(0); + let explicit = pack_raw_series_chunk( + &explicit_schema, + FrameLabelMode::Explicit, + None, + &explicit_chunk, + ) + .unwrap() + .remove(0); let name = explicit .column(4) .as_any() @@ -1427,7 +1746,10 @@ fn rejects_invalid_explicit_labels() { duplicate_labels[5] = Arc::new(duplicate_list); let duplicate = arrow::record_batch::RecordBatch::try_new(explicit.schema(), duplicate_labels).unwrap(); - assert!(unpack_raw_series_frame(&explicit_schema, RawLabelMode::Explicit, &duplicate).is_err()); + assert!( + unpack_raw_series_frame(&explicit_schema, FrameLabelMode::Explicit, None, &duplicate) + .is_err() + ); let mut reserved_labels = explicit.columns().to_vec(); let reserved_items = StructArray::try_new( @@ -1458,7 +1780,10 @@ fn rejects_invalid_explicit_labels() { reserved_labels[5] = Arc::new(reserved_list); let reserved = arrow::record_batch::RecordBatch::try_new(explicit.schema(), reserved_labels).unwrap(); - assert!(unpack_raw_series_frame(&explicit_schema, RawLabelMode::Explicit, &reserved).is_err()); + assert!( + unpack_raw_series_frame(&explicit_schema, FrameLabelMode::Explicit, None, &reserved) + .is_err() + ); } #[test] @@ -1466,7 +1791,8 @@ fn rejects_malformed_histogram_and_payload_encodings() { let hist_schema = test_schema(&["hist"]); let valid_hist = pack_raw_series_chunk( &hist_schema, - RawLabelMode::Key, + FrameLabelMode::Key, + Some(&dense_description()), &RawSeriesChunk::try_new( &hist_schema, Buffer::from(vec![1_u8]), @@ -1553,8 +1879,13 @@ fn rejects_malformed_histogram_and_payload_encodings() { let absent_hist = arrow::record_batch::RecordBatch::try_new(valid_hist.schema(), absent_hist_columns) .unwrap(); - let decoded_absent_hist = - unpack_raw_series_frame(&hist_schema, RawLabelMode::Key, &absent_hist).unwrap(); + let decoded_absent_hist = unpack_raw_series_frame( + &hist_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &absent_hist, + ) + .unwrap(); assert!( matches!(decoded_absent_hist[0].fields.get(0).unwrap().sample(0), SampleRef::Float(v) if v == 1.25) ); @@ -1606,7 +1937,15 @@ fn rejects_malformed_histogram_and_payload_encodings() { let absent_both = arrow::record_batch::RecordBatch::try_new(valid_hist.schema(), absent_both_columns) .unwrap(); - assert!(unpack_raw_series_frame(&hist_schema, RawLabelMode::Key, &absent_both).is_err()); + assert!( + unpack_raw_series_frame( + &hist_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &absent_both + ) + .is_err() + ); let DataType::List(index_item) = null_hist.fields()[0].data_type() else { panic!("expected histogram indices list"); @@ -1636,7 +1975,15 @@ fn rejects_malformed_histogram_and_payload_encodings() { bad_parent_columns[3] = Arc::new(bad_fields); let present_bad_parent = arrow::record_batch::RecordBatch::try_new(valid_hist.schema(), bad_parent_columns).unwrap(); - assert!(unpack_raw_series_frame(&hist_schema, RawLabelMode::Key, &present_bad_parent).is_err()); + assert!( + unpack_raw_series_frame( + &hist_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &present_bad_parent + ) + .is_err() + ); let valid_float = ListArray::try_new( match payload.fields()[0].data_type() { @@ -1677,7 +2024,15 @@ fn rejects_malformed_histogram_and_payload_encodings() { overlap_columns[3] = Arc::new(overlap_fields); let overlap = arrow::record_batch::RecordBatch::try_new(valid_hist.schema(), overlap_columns).unwrap(); - assert!(unpack_raw_series_frame(&hist_schema, RawLabelMode::Key, &overlap).is_err()); + assert!( + unpack_raw_series_frame( + &hist_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &overlap + ) + .is_err() + ); let short_indices = ListArray::try_new( index_item.clone(), @@ -1704,7 +2059,15 @@ fn rejects_malformed_histogram_and_payload_encodings() { short_columns[3] = Arc::new(short_fields); let short_indices = arrow::record_batch::RecordBatch::try_new(valid_hist.schema(), short_columns).unwrap(); - assert!(unpack_raw_series_frame(&hist_schema, RawLabelMode::Key, &short_indices).is_err()); + assert!( + unpack_raw_series_frame( + &hist_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &short_indices + ) + .is_err() + ); let malformed_histogram = || { let mut columns = valid_hist.columns().to_vec(); @@ -1779,7 +2142,13 @@ fn rejects_malformed_histogram_and_payload_encodings() { arrow::record_batch::RecordBatch::try_new(valid_hist.schema(), columns).unwrap() }; assert!( - unpack_raw_series_frame(&hist_schema, RawLabelMode::Key, &malformed_histogram()).is_err() + unpack_raw_series_frame( + &hist_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &malformed_histogram() + ) + .is_err() ); let mut hidden_float = valid_hist.columns().to_vec(); @@ -1820,8 +2189,13 @@ fn rejects_malformed_histogram_and_payload_encodings() { hidden_float[3] = Arc::new(fields); let absent_float = arrow::record_batch::RecordBatch::try_new(valid_hist.schema(), hidden_float).unwrap(); - let decoded_absent_float = - unpack_raw_series_frame(&hist_schema, RawLabelMode::Key, &absent_float).unwrap(); + let decoded_absent_float = unpack_raw_series_frame( + &hist_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &absent_float, + ) + .unwrap(); assert_histogram( match decoded_absent_float[0].fields.get(0).unwrap().sample(0) { SampleRef::Histogram(histogram) => histogram, @@ -1849,8 +2223,20 @@ fn decoded_key_and_payload_slices_keep_buffers_alive() { true, ) .unwrap(); - let frames = pack_raw_series_chunk(&schema, RawLabelMode::Key, &chunk).unwrap(); - let decoded = unpack_raw_series_frame(&schema, RawLabelMode::Key, &frames[0]).unwrap(); + let frames = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &chunk, + ) + .unwrap(); + let decoded = unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &frames[0], + ) + .unwrap(); (owner, chunk, frames, decoded) }; let batch = frames[0].clone(); @@ -2090,10 +2476,10 @@ fn sliced_canonical_mixed_histogram_round_trip_keeps_buffers_alive() { // observable: the default budget could coalesce several rows and copy their payloads. let frames = pack_chunk_with_budget( &schema, - RawLabelMode::Key, + FrameLabelMode::Key, &chunk, 1, - RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, ) .unwrap(); assert_eq!(frames.len(), 1); @@ -2200,7 +2586,13 @@ fn sliced_canonical_mixed_histogram_round_trip_keeps_buffers_alive() { .as_ptr() ); } - let decoded = unpack_raw_series_frame(&schema, RawLabelMode::Key, packed).unwrap(); + let decoded = unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + packed, + ) + .unwrap(); let DenseSamplesRef::Mixed(decoded_mixed) = decoded[0].fields.get(0).unwrap().dense().unwrap() else { panic!("mixed representation was normalized"); @@ -2369,7 +2761,15 @@ fn malformed_pack_input_is_revalidated_before_encoding() { fields: SeriesFields::one(Samples::from_float(Float64Array::from(vec![Some(3.0)]))), ..chunk }; - assert!(pack_raw_series_chunk(&schema, RawLabelMode::Key, &invalid).is_err()); + assert!( + pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &invalid + ) + .is_err() + ); } #[test] @@ -2476,21 +2876,23 @@ fn cross_chunk_rows_coalesce_into_one_multirow_batch() { vec![Some(3.0), Some(4.0), None], true, ); - assert_eq!(RawSeriesFrameBuilder::DEFAULT_MAX_ROWS, 256); - assert_eq!(RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, 1 << 20); - let expected = raw_series_frame_schema(&schema, RawLabelMode::Key).unwrap(); + assert_eq!(SeriesFrameBuilder::DEFAULT_MAX_ROWS, 256); + assert_eq!(SeriesFrameBuilder::DEFAULT_MAX_BYTES, 1 << 20); + let expected = + series_frame_schema(&schema, FrameLabelMode::Key, Some(&dense_description())).unwrap(); - let mut builder = RawSeriesFrameBuilder::try_new( + let mut builder = SeriesFrameBuilder::try_new( &schema, - RawLabelMode::Key, - RawSeriesFrameBuilder::DEFAULT_MAX_ROWS, - RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, + FrameLabelMode::Key, + Some(&dense_description()), + SeriesFrameBuilder::DEFAULT_MAX_ROWS, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, ) .unwrap(); assert_eq!(builder.schema(), &expected); - let mut batches = append_chunk(&mut builder, &schema, RawLabelMode::Key, &first).unwrap(); - batches.extend(append_chunk(&mut builder, &schema, RawLabelMode::Key, &second).unwrap()); + let mut batches = append_chunk(&mut builder, &schema, FrameLabelMode::Key, &first).unwrap(); + batches.extend(append_chunk(&mut builder, &schema, FrameLabelMode::Key, &second).unwrap()); if let Some(batch) = builder.flush().unwrap() { batches.push(batch); } @@ -2510,7 +2912,7 @@ fn cross_chunk_rows_coalesce_into_one_multirow_batch() { ["read_key", "is_last", "time", "fields"] ); for (key, value) in [ - ("greptime.series.layout", "v1"), + ("greptime.series.layout", "v2"), ("greptime.series.content", "raw"), ("greptime.series.labels", "key"), ("greptime.series.time", "timestamps"), @@ -2578,7 +2980,7 @@ fn cross_chunk_rows_coalesce_into_one_multirow_batch() { assert_eq!(float_values.value(2).to_bits(), 3.0_f64.to_bits()); assert_eq!(float_values.value(3).to_bits(), 4.0_f64.to_bits()); - let decoded = decode_batches(&schema, RawLabelMode::Key, &batches); + let decoded = decode_batches(&schema, FrameLabelMode::Key, &batches); assert_eq!(decoded.len(), 2); assert_eq!(decoded[0].primary_key.as_slice(), &[0xaa]); assert!(!decoded[0].is_last); @@ -2614,17 +3016,18 @@ fn raw_frame_builder_emits_at_the_row_budget_without_another_cursor() { ]; for max_rows in [2usize, 3] { - let mut builder = RawSeriesFrameBuilder::try_new( + let mut builder = SeriesFrameBuilder::try_new( &schema, - RawLabelMode::Key, + FrameLabelMode::Key, + Some(&dense_description()), max_rows, - RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, ) .unwrap(); let mut emitted_at = None; let mut batches = Vec::new(); for (index, chunk) in chunks.iter().enumerate() { - let produced = append_chunk(&mut builder, &schema, RawLabelMode::Key, chunk).unwrap(); + let produced = append_chunk(&mut builder, &schema, FrameLabelMode::Key, chunk).unwrap(); assert!(produced.len() <= 1, "one cursor emits at most one batch"); if let Some(batch) = produced.first() { assert!( @@ -2660,7 +3063,7 @@ fn raw_frame_builder_emits_at_the_row_budget_without_another_cursor() { .collect::>() ); - let decoded = decode_batches(&schema, RawLabelMode::Key, &batches); + let decoded = decode_batches(&schema, FrameLabelMode::Key, &batches); assert_eq!(decoded.len(), 3); assert_eq!( decoded @@ -2681,30 +3084,42 @@ fn raw_frame_builder_rejects_zero_budgets_and_bounds_offsets() { let schema = test_schema(&["value"]); // Zero in either budget is not a legal configuration; the literals take the parameter's type. assert!( - RawSeriesFrameBuilder::try_new( + SeriesFrameBuilder::try_new( &schema, - RawLabelMode::Key, + FrameLabelMode::Key, + Some(&dense_description()), 0, - RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, ) .is_err() ); assert!( - RawSeriesFrameBuilder::try_new( + SeriesFrameBuilder::try_new( &schema, - RawLabelMode::Key, - RawSeriesFrameBuilder::DEFAULT_MAX_ROWS, + FrameLabelMode::Key, + Some(&dense_description()), + SeriesFrameBuilder::DEFAULT_MAX_ROWS, 0, ) .is_err() ); - assert!(RawSeriesFrameBuilder::try_new(&schema, RawLabelMode::Key, 0, 0).is_err()); assert!( - RawSeriesFrameBuilder::try_new( + SeriesFrameBuilder::try_new( &schema, - RawLabelMode::Key, + FrameLabelMode::Key, + Some(&dense_description()), + 0, + 0 + ) + .is_err() + ); + assert!( + SeriesFrameBuilder::try_new( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), 1, - RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, ) .is_ok() ); @@ -2740,9 +3155,9 @@ fn byte_budget_emits_legal_oversized_rows_without_dropping_them() { // batch instead of being dropped or merged. let batches = pack_chunk_with_budget( &schema, - RawLabelMode::Key, + FrameLabelMode::Key, &chunk, - RawSeriesFrameBuilder::DEFAULT_MAX_ROWS, + SeriesFrameBuilder::DEFAULT_MAX_ROWS, 1, ) .unwrap(); @@ -2754,7 +3169,7 @@ fn byte_budget_emits_legal_oversized_rows_without_dropping_them() { "an oversized row is emitted alone, never dropped" ); - let decoded = decode_batches(&schema, RawLabelMode::Key, &batches); + let decoded = decode_batches(&schema, FrameLabelMode::Key, &batches); assert_eq!(decoded.len(), 3); for (position, row) in decoded.iter().enumerate() { assert_eq!(row.primary_key.as_slice(), &[0x9a]); @@ -2776,24 +3191,25 @@ fn byte_budget_flushes_before_exceeding_and_retries_the_pending_row() { let mut measured = Vec::with_capacity(chunks.len()); for chunk in &chunks { let mut cursor = - RawSeriesChunkCursor::try_new(&schema, RawLabelMode::Key, chunk.clone()).unwrap(); + SeriesFrameCursor::try_new_raw(&schema, FrameLabelMode::Key, chunk.clone()).unwrap(); measured.push(cursor.next_row().unwrap().unwrap().bytes); } let budget = measured[0] + measured[1] - 1; - let mut builder = RawSeriesFrameBuilder::try_new( + let mut builder = SeriesFrameBuilder::try_new( &schema, - RawLabelMode::Key, - RawSeriesFrameBuilder::DEFAULT_MAX_ROWS, + FrameLabelMode::Key, + Some(&dense_description()), + SeriesFrameBuilder::DEFAULT_MAX_ROWS, budget, ) .unwrap(); let mut first = - RawSeriesChunkCursor::try_new(&schema, RawLabelMode::Key, chunks[0].clone()).unwrap(); + SeriesFrameCursor::try_new_raw(&schema, FrameLabelMode::Key, chunks[0].clone()).unwrap(); assert!(first.append_to(&mut builder).unwrap().is_none()); let mut second = - RawSeriesChunkCursor::try_new(&schema, RawLabelMode::Key, chunks[1].clone()).unwrap(); + SeriesFrameCursor::try_new_raw(&schema, FrameLabelMode::Key, chunks[1].clone()).unwrap(); let emitted = second.append_to(&mut builder).unwrap().unwrap(); assert_eq!(emitted.num_rows(), 1, "the first row is flushed alone"); // The second row stays pending across the flush and is retried, never dropped. @@ -2801,7 +3217,7 @@ fn byte_budget_flushes_before_exceeding_and_retries_the_pending_row() { let remainder = builder.flush().unwrap().unwrap(); assert_eq!(remainder.num_rows(), 1); - let decoded = decode_batches(&schema, RawLabelMode::Key, &[emitted, remainder]); + let decoded = decode_batches(&schema, FrameLabelMode::Key, &[emitted, remainder]); assert_eq!(decoded.len(), 2); assert_eq!(decoded[0].primary_key.as_slice(), &[1]); assert_eq!(decoded[1].primary_key.as_slice(), &[2]); @@ -2816,10 +3232,11 @@ fn byte_budget_flushes_before_exceeding_and_retries_the_pending_row() { // A requested byte bound is clamped to the representable cumulative offset range, and that // range is rejected before any child array is allocated. - let clamped = RawSeriesFrameBuilder::try_new( + let clamped = SeriesFrameBuilder::try_new( &schema, - RawLabelMode::Key, - RawSeriesFrameBuilder::DEFAULT_MAX_ROWS, + FrameLabelMode::Key, + Some(&dense_description()), + SeriesFrameBuilder::DEFAULT_MAX_ROWS, usize::MAX, ) .unwrap(); @@ -2848,7 +3265,13 @@ fn multirow_histogram_rows_keep_row_local_indices_and_tables() { .unwrap(); // The public wrapper groups two independent rows into one multirow batch. - let packed = pack_raw_series_chunk(&schema, RawLabelMode::Key, &chunk).unwrap(); + let packed = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &chunk, + ) + .unwrap(); assert_eq!(packed.len(), 1); assert_eq!(packed[0].num_rows(), 2); @@ -2880,7 +3303,11 @@ fn multirow_histogram_rows_keep_row_local_indices_and_tables() { .unwrap(); assert_eq!(concatenated.len(), 4); - let decoded = decode_batches(&schema, RawLabelMode::Key, std::slice::from_ref(&packed[0])); + let decoded = decode_batches( + &schema, + FrameLabelMode::Key, + std::slice::from_ref(&packed[0]), + ); assert_row_local_histogram_rows(&decoded); // A real Arrow IPC round trip keeps the schema, the whole tables, the null index entries and @@ -2907,17 +3334,19 @@ fn multirow_histogram_rows_keep_row_local_indices_and_tables() { .value(1), 0 ); - let restored_rows = decode_batches(&schema, RawLabelMode::Key, &restored); + let restored_rows = decode_batches(&schema, FrameLabelMode::Key, &restored); assert_row_local_histogram_rows(&restored_rows); } #[test] fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { let schema = test_schema(&["value"]); - let expected = raw_series_frame_schema(&schema, RawLabelMode::Key).unwrap(); + let expected = + series_frame_schema(&schema, FrameLabelMode::Key, Some(&dense_description())).unwrap(); let first = pack_raw_series_chunk( &schema, - RawLabelMode::Key, + FrameLabelMode::Key, + Some(&dense_description()), &float_chunk( &schema, &[1], @@ -2931,14 +3360,17 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { .remove(0); let second = pack_raw_series_chunk( &schema, - RawLabelMode::Key, + FrameLabelMode::Key, + Some(&dense_description()), &float_chunk(&schema, &[2], None, vec![3], vec![Some(3.0)], true), ) .unwrap() .remove(0); - let key_decoder = RawSeriesFrameDecoder::try_new(&schema, RawLabelMode::Key).unwrap(); - let decoded = key_decoder.decode(&first).unwrap(); + let key_decoder = + SeriesFrameDecoder::try_new(&schema, FrameLabelMode::Key, Some(&dense_description())) + .unwrap(); + let decoded = key_decoder.decode_raw(&first).unwrap(); assert_eq!(decoded.len(), 1); assert_eq!(decoded[0].primary_key.as_slice(), &[1]); assert_eq!( @@ -2948,7 +3380,7 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { [1, 2] ); // The same decoder accepts a later valid batch before the malformed ones below. - let decoded = key_decoder.decode(&second).unwrap(); + let decoded = key_decoder.decode_raw(&second).unwrap(); assert_eq!(decoded.len(), 1); assert_eq!(decoded[0].primary_key.as_slice(), &[2]); assert!(decoded[0].is_last); @@ -2964,11 +3396,12 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { second.columns().to_vec(), ) .unwrap(); - assert_eq!(key_decoder.decode(&unrelated).unwrap().len(), 1); + assert_eq!(key_decoder.decode_raw(&unrelated).unwrap().len(), 1); // Reserved metadata and evolved fields are rejected, including by a zero-row batch. for (key, value) in [ - ("greptime.series.layout", "v2"), + // `v2` is the layout this decoder expects, so the unreadable `v1` is the wrong one. + ("greptime.series.layout", "v1"), ("greptime.series.content", "evaluated"), ("greptime.series.labels", "explicit"), ("greptime.series.time", "grid"), @@ -2984,28 +3417,34 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { ) .unwrap(); assert!( - key_decoder.decode(&contaminated).is_err(), + key_decoder.decode_raw(&contaminated).is_err(), "{key} must be rejected" ); } let evolved_schema = test_schema(&["other"]); let evolved = pack_raw_series_chunk( &evolved_schema, - RawLabelMode::Key, + FrameLabelMode::Key, + Some(&dense_description()), &float_chunk(&evolved_schema, &[3], None, vec![4], vec![Some(4.0)], false), ) .unwrap() .remove(0); - assert!(key_decoder.decode(&evolved).is_err()); + assert!(key_decoder.decode_raw(&evolved).is_err()); let empty_evolved = arrow::record_batch::RecordBatch::new_empty( - raw_series_frame_schema(&evolved_schema, RawLabelMode::Key).unwrap(), + series_frame_schema( + &evolved_schema, + FrameLabelMode::Key, + Some(&dense_description()), + ) + .unwrap(), ); - assert!(key_decoder.decode(&empty_evolved).is_err()); + assert!(key_decoder.decode_raw(&empty_evolved).is_err()); // Rejections leave the cached schema usable, and a valid zero-row batch stays valid. - assert_eq!(key_decoder.decode(&first).unwrap().len(), 1); + assert_eq!(key_decoder.decode_raw(&first).unwrap().len(), 1); assert!( key_decoder - .decode(&arrow::record_batch::RecordBatch::new_empty( + .decode_raw(&arrow::record_batch::RecordBatch::new_empty( expected.clone() )) .unwrap() @@ -3013,10 +3452,12 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { ); // Explicit mode reuses one decoder for a present but nameless metric and a named one. - let explicit_decoder = RawSeriesFrameDecoder::try_new(&schema, RawLabelMode::Explicit).unwrap(); + let explicit_decoder = + SeriesFrameDecoder::try_new(&schema, FrameLabelMode::Explicit, None).unwrap(); let nameless = pack_raw_series_chunk( &schema, - RawLabelMode::Explicit, + FrameLabelMode::Explicit, + None, &float_chunk( &schema, &[7], @@ -3030,7 +3471,8 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { .remove(0); let named = pack_raw_series_chunk( &schema, - RawLabelMode::Explicit, + FrameLabelMode::Explicit, + None, &float_chunk( &schema, &[8], @@ -3042,12 +3484,12 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { ) .unwrap() .remove(0); - let decoded = explicit_decoder.decode(&nameless).unwrap(); + let decoded = explicit_decoder.decode_raw(&nameless).unwrap(); let labels = decoded[0].metric.as_ref().unwrap(); assert_eq!(labels.name(), None); assert_eq!(labels.get("zone"), Some("west")); assert_eq!( - explicit_decoder.decode(&named).unwrap()[0] + explicit_decoder.decode_raw(&named).unwrap()[0] .metric .as_ref() .unwrap() @@ -3055,8 +3497,8 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { Some("cpu") ); // Label modes are not interchangeable. - assert!(explicit_decoder.decode(&first).is_err()); - assert!(key_decoder.decode(&nameless).is_err()); + assert!(explicit_decoder.decode_raw(&first).is_err()); + assert!(key_decoder.decode_raw(&nameless).is_err()); } /// A dense description carrying the ordered key columns and the logical tags the decoder reads. @@ -3087,22 +3529,26 @@ fn dense_description() -> LabelDecodeSpec { #[test] fn label_decode_description_roundtrips_through_schema_metadata() { let description = dense_description(); - let key = raw_series_frame_schema_with_label_decode( + let key = series_frame_schema( &test_schema(&["value"]), - RawLabelMode::Key, + FrameLabelMode::Key, Some(&description), ) .unwrap(); assert_eq!( - raw_series_frame_label_decode(&key).unwrap(), + series_frame_label_decode(&key).unwrap(), Some(description.clone()) ); - // The description is metadata-only: the column contract is untouched. - assert_eq!( - key.fields(), - raw_series_frame_schema(&test_schema(&["value"]), RawLabelMode::Key) - .unwrap() - .fields() + // The description is metadata-only: the column contract is untouched, so only the + // `greptime.series.label_decode` text differs from a frame declaring another description. + let mut other = dense_description(); + other.encoding = LabelDecodeEncoding::Sparse; + let other = + series_frame_schema(&test_schema(&["value"]), FrameLabelMode::Key, Some(&other)).unwrap(); + assert_eq!(key.fields(), other.fields()); + assert_ne!( + key.metadata().get(LABEL_DECODE), + other.metadata().get(LABEL_DECODE) ); assert_eq!(key.metadata().get(LABELS).map(String::as_str), Some("key")); } @@ -3110,20 +3556,19 @@ fn label_decode_description_roundtrips_through_schema_metadata() { #[test] fn explicit_frames_declare_no_label_decode_description() { let schema = test_schema(&["value"]); - let explicit = - raw_series_frame_schema_with_label_decode(&schema, RawLabelMode::Explicit, None).unwrap(); + let explicit = series_frame_schema(&schema, FrameLabelMode::Explicit, None).unwrap(); assert!(explicit.metadata().get(LABEL_DECODE).is_none()); - assert_eq!(raw_series_frame_label_decode(&explicit).unwrap(), None); + assert_eq!(series_frame_label_decode(&explicit).unwrap(), None); // A key frame without a description, or an explicit frame with one, is rejected instead of // being written as a frame the consumer cannot decode. assert!( - raw_series_frame_schema_with_label_decode(&schema, RawLabelMode::Key, None).is_err(), + series_frame_schema(&schema, FrameLabelMode::Key, None).is_err(), "key-encoded frames must describe their labels" ); assert!( - raw_series_frame_schema_with_label_decode( + series_frame_schema( &schema, - RawLabelMode::Explicit, + FrameLabelMode::Explicit, Some(&dense_description()) ) .is_err(), @@ -3133,14 +3578,19 @@ fn explicit_frames_declare_no_label_decode_description() { /// The raw frame's own columns with exactly the supplied metadata map. fn schema_with_metadata(metadata: Vec<(String, String)>) -> Schema { - let layout = raw_series_frame_schema(&test_schema(&["value"]), RawLabelMode::Key).unwrap(); + let layout = series_frame_schema( + &test_schema(&["value"]), + FrameLabelMode::Key, + Some(&dense_description()), + ) + .unwrap(); Schema::new_with_metadata(layout.fields().clone(), metadata.into_iter().collect()) } /// The four metadata entries every raw frame declares, with the given label mode. fn base_metadata(labels: &str) -> Vec<(String, String)> { [ - ("greptime.series.layout", "v1"), + ("greptime.series.layout", "v2"), ("greptime.series.content", "raw"), ("greptime.series.labels", labels), ("greptime.series.time", "timestamps"), @@ -3150,8 +3600,14 @@ fn base_metadata(labels: &str) -> Vec<(String, String)> { .collect() } +/// A description the reader would refuse cannot be written into a frame. +/// +/// The three shapes below are each a self-contradictory binding: a duplicate key column, a logical +/// tag naming a column the primary key does not list, and a tag listed twice. Writing one would +/// produce a frame no consumer could decode, so the producer is where it is refused. A frame that +/// arrives already carrying one is a different case, and the decoder rejects those on read. #[test] -fn label_decode_description_rejects_inconsistent_bindings() { +fn label_decode_description_rejects_inconsistent_bindings_on_write() { let schema = test_schema(&["value"]); let mut cases = Vec::new(); @@ -3169,20 +3625,13 @@ fn label_decode_description_rejects_inconsistent_bindings() { cases.push(("duplicate logical tag", duplicate_tag)); for (label, description) in cases { - // Writing is faithful: the description records what the producer declares. Reading is - // where a description this revision cannot honour is rejected, so a producer cannot turn a - // bad binding into a frame a consumer would decode under silently. - let frame = raw_series_frame_schema_with_label_decode( - &schema, - RawLabelMode::Key, - Some(&description), - ) - .unwrap_or_else(|error| panic!("{label}: {error}")); - // The writer records what the producer declares, so the rejection has to come from the - // reader; one assertion covers it, since the frame it reads is the text just written. + // Both directions reject: a frame is worth writing only if it can be read back, so an + // inconsistent description is refused at the producer instead of being written faithfully + // and refused only once it reaches a consumer who cannot fix it. The case that must still + // reach the wire is a hand-built frame, which the fixture below covers. assert!( - raw_series_frame_label_decode(&frame).is_err(), - "{label}: an inconsistent description is rejected when read" + series_frame_schema(&schema, FrameLabelMode::Key, Some(&description)).is_err(), + "{label}: an inconsistent description is rejected when written" ); } } @@ -3198,7 +3647,7 @@ fn label_decode_rejects_a_frame_without_a_label_mode() { .filter(|(key, _)| key != LABELS) .collect(); assert!( - raw_series_frame_label_decode(&schema_with_metadata(without_mode)).is_err(), + series_frame_label_decode(&schema_with_metadata(without_mode)).is_err(), "a frame without a label mode has no label layout to describe" ); } @@ -3216,11 +3665,16 @@ fn raw_series_label_decode_metadata_fixture() { } let cases: Vec = serde_json::from_str(include_str!( - "../../../../../tests/compatibility/wire/raw_series_label_decode/v1_metadata.json" + "../../../../../tests/compatibility/wire/raw_series_label_decode/v2_metadata.json" )) .unwrap(); for case in cases { - let layout = raw_series_frame_schema(&test_schema(&["value"]), RawLabelMode::Key).unwrap(); + let layout = series_frame_schema( + &test_schema(&["value"]), + FrameLabelMode::Key, + Some(&dense_description()), + ) + .unwrap(); let metadata = case .metadata .iter() @@ -3229,12 +3683,12 @@ fn raw_series_label_decode_metadata_fixture() { let frame_schema = Schema::new_with_metadata(layout.fields().clone(), metadata); match case.encoding.as_deref() { None => assert!( - raw_series_frame_label_decode(&frame_schema).is_err(), + series_frame_label_decode(&frame_schema).is_err(), "{}: must be rejected", case.name ), Some(expected) => { - let decode = raw_series_frame_label_decode(&frame_schema) + let decode = series_frame_label_decode(&frame_schema) .unwrap_or_else(|error| panic!("{}: {error}", case.name)) .unwrap_or_else(|| panic!("{}: no description", case.name)); let encoding = match decode.encoding { @@ -3279,9 +3733,9 @@ fn raw_series_label_decode_metadata_fixture() { if expected == "dense" { // The fixture text is the encoder's own output, byte for byte: re-encoding the // description it decodes to must reproduce it exactly. - let roundtrip = raw_series_frame_schema_with_label_decode( + let roundtrip = series_frame_schema( &test_schema(&["value"]), - RawLabelMode::Key, + FrameLabelMode::Key, Some(&decode), ) .unwrap(); @@ -3309,9 +3763,9 @@ fn label_decode_description_carries_a_timestamp_typed_tag_column() { column_id: 2, name: "observed".to_string(), }); - let frame = raw_series_frame_schema_with_label_decode( + let frame = series_frame_schema( &test_schema(&["value"]), - RawLabelMode::Key, + FrameLabelMode::Key, Some(&description), ) .unwrap(); @@ -3321,7 +3775,7 @@ fn label_decode_description_carries_a_timestamp_typed_tag_column() { text.contains("\"physical_type\":{\"Timestamp\":{\"Millisecond\":null}}"), "the encoder wrote a timestamp type: {text}" ); - let recovered = raw_series_frame_label_decode(&frame).unwrap().unwrap(); + let recovered = series_frame_label_decode(&frame).unwrap().unwrap(); assert_eq!(recovered, description); assert_eq!( recovered.primary_key[1].physical_type, @@ -3354,9 +3808,9 @@ fn label_decode_text_is_independent_of_the_callers_tag_order() { descending.primary_key.reverse(); let write = |description: &LabelDecodeSpec| { - let frame = raw_series_frame_schema_with_label_decode( + let frame = series_frame_schema( &test_schema(&["value"]), - RawLabelMode::Key, + FrameLabelMode::Key, Some(description), ) .unwrap(); @@ -3394,10 +3848,10 @@ fn label_decode_text_is_independent_of_the_callers_tag_order() { write(&descending), "the writer reordered the encoding columns" ); - let recovered = raw_series_frame_label_decode( - &raw_series_frame_schema_with_label_decode( + let recovered = series_frame_label_decode( + &series_frame_schema( &test_schema(&["value"]), - RawLabelMode::Key, + FrameLabelMode::Key, Some(&descending), ) .unwrap(), diff --git a/src/datatypes/src/series/time.rs b/src/datatypes/src/series/time.rs index 966ee2f6e15..1568ce7d24c 100644 --- a/src/datatypes/src/series/time.rs +++ b/src/datatypes/src/series/time.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use arrow::buffer::ScalarBuffer; use once_cell::sync::OnceCell; +use serde::{Deserialize, Serialize}; use snafu::OptionExt; use crate::error::{InvalidSeriesSnafu, Result}; @@ -25,7 +26,12 @@ use crate::error::{InvalidSeriesSnafu, Result}; mod tests; /// Value description of a non-empty, fixed-step evaluation grid. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +/// +/// The serde form is the wire form a series frame writes under `greptime.series.grid`: field names +/// in declaration order, no optional members. Two nodes compare that text byte for byte, so the +/// only place that spells it is a series frame's own grid metadata entry. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct GridSpec { /// First evaluation timestamp in milliseconds. pub start_ms: i64, diff --git a/src/mito-codec/src/labels/tests.rs b/src/mito-codec/src/labels/tests.rs index 88651bd1c35..2d84176d530 100644 --- a/src/mito-codec/src/labels/tests.rs +++ b/src/mito-codec/src/labels/tests.rs @@ -629,13 +629,13 @@ fn frame_label_decode_description_decodes_like_the_bound_source() { ); let description = common_query::series_frame::label_decode_spec(&bound) .expect("a primary-key binding describes its key layout"); - let frame_schema = datatypes::series::frame::raw_series_frame_schema_with_label_decode( + let frame_schema = datatypes::series::frame::series_frame_schema( &frame_series_schema(), - datatypes::series::frame::RawLabelMode::Key, + datatypes::series::frame::FrameLabelMode::Key, Some(&description), ) .unwrap(); - let recovered = datatypes::series::frame::raw_series_frame_label_decode(&frame_schema) + let recovered = datatypes::series::frame::series_frame_label_decode(&frame_schema) .unwrap() .expect("the frame declares its decode description"); assert_eq!(recovered, description); diff --git a/src/mito2/src/engine/scan_test.rs b/src/mito2/src/engine/scan_test.rs index 55f8cf80408..bfb316959a6 100644 --- a/src/mito2/src/engine/scan_test.rs +++ b/src/mito2/src/engine/scan_test.rs @@ -5093,7 +5093,7 @@ async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() { assert_eq!(raw_visible, visible); // Exercise the production table physical consumer over a fresh real Mito scanner. - use common_query::series_frame::{raw_series_frame_ordering, unpack_raw_series}; + use common_query::series_frame::{series_frame_ordering, unpack_raw_series}; use datafusion::execution::TaskContext; use datafusion::physical_plan::ExecutionPlan; use table::table::scan::RegionScanExec; @@ -5127,7 +5127,7 @@ async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() { .unwrap(); // Unpack under the frame's own declaration. A legacy producer declares `KeyThenTime`, and // only that declaration makes the decoder check that keys increase. - let frame_ordering = raw_series_frame_ordering(native_frames.schema().as_ref()).unwrap(); + let frame_ordering = series_frame_ordering(native_frames.schema().as_ref()).unwrap(); assert_eq!( SeriesOrdering::KeyThenTime, frame_ordering, @@ -5233,9 +5233,7 @@ async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() { /// `promql::source::merge::tests::rejects_unsupported_ordering_and_empty_input_without_polling`. #[tokio::test] async fn test_two_phase_native_raw_stream_declares_contiguous_frames() { - use common_query::series_frame::{ - pack_raw_series, raw_series_frame_ordering, unpack_raw_series, - }; + use common_query::series_frame::{pack_raw_series, series_frame_ordering, unpack_raw_series}; use datatypes::series::SeriesOrdering; use datatypes::series::samples::SeriesSchema; use store_api::region_engine::{BoundTag, QueryScanContext, RawLabelSource}; @@ -5374,7 +5372,7 @@ async fn test_two_phase_native_raw_stream_declares_contiguous_frames() { chunks, ))) .unwrap(); - let ordering = raw_series_frame_ordering(packed.schema().as_ref()).unwrap(); + let ordering = series_frame_ordering(packed.schema().as_ref()).unwrap(); assert_eq!(SeriesOrdering::Contiguous, ordering); let validated = unpack_raw_series(packed, Some(schema)) .unwrap() @@ -6560,7 +6558,7 @@ async fn test_mito_raw_series_int64_predicate_uses_original_precision() { assert_eq!(vec![(2, 20.0)], output_values); // Validate the typed integer gate through RegionScanExec's actual physical consumer. - use common_query::series_frame::{raw_series_frame_ordering, unpack_raw_series}; + use common_query::series_frame::{series_frame_ordering, unpack_raw_series}; use datafusion::execution::TaskContext; use datafusion::physical_plan::ExecutionPlan; use datatypes::series::SeriesOrdering; @@ -6597,7 +6595,7 @@ async fn test_mito_raw_series_int64_predicate_uses_original_precision() { .execute(0, Arc::new(TaskContext::default())) .unwrap(); // Unpack under the frame's own declaration so the decoder checks the legacy key order. - let ordering = raw_series_frame_ordering(frames.schema().as_ref()).unwrap(); + let ordering = series_frame_ordering(frames.schema().as_ref()).unwrap(); assert_eq!( SeriesOrdering::KeyThenTime, ordering, @@ -6655,7 +6653,7 @@ async fn test_mito_raw_series_int64_predicate_uses_original_precision() { let empty_frames = empty_plan .execute(0, Arc::new(TaskContext::default())) .unwrap(); - let empty_ordering = raw_series_frame_ordering(empty_frames.schema().as_ref()).unwrap(); + let empty_ordering = series_frame_ordering(empty_frames.schema().as_ref()).unwrap(); assert_eq!( SeriesOrdering::KeyThenTime, empty_ordering, diff --git a/src/promql/src/extension_plan/fused_series/tests.rs b/src/promql/src/extension_plan/fused_series/tests.rs index 30bd1983dcf..00df6f05aeb 100644 --- a/src/promql/src/extension_plan/fused_series/tests.rs +++ b/src/promql/src/extension_plan/fused_series/tests.rs @@ -63,7 +63,11 @@ use datafusion::physical_plan::{ RecordBatchStream, SendableRecordBatchStream, collect, displayable, }; use datafusion::prelude::SessionContext; -use datatypes::series::frame::{RawLabelMode, pack_raw_series_chunk, raw_series_frame_schema}; +use datatypes::prelude::ConcreteDataType; +use datatypes::series::frame::{ + FrameLabelMode, LABEL_DECODE_VERSION, LabelDecodeColumn, LabelDecodeEncoding, LabelDecodeSpec, + LabelDecodeTag, pack_raw_series_chunk, raw_series_frame_schema, series_frame_schema, +}; use datatypes::series::raw::{RawSeriesChunk, RawTimestamps}; use datatypes::series::samples::{FieldSpec, Samples, Series, SeriesFields, SeriesSchema}; use datatypes::series::time::{EvalGrid, GridSpan, GridSpec, TimeAxis}; @@ -488,7 +492,7 @@ fn binding(tree: &NativeTree) -> FusedSeriesBinding { /// The schema of a frame carrying series of `schema` with their labels explicit. fn frame_schema_of(schema: &SeriesSchema) -> SchemaRef { - raw_series_frame_schema(schema, RawLabelMode::Explicit).expect("explicit labels are a frame") + raw_series_frame_schema(schema, FrameLabelMode::Explicit).expect("explicit labels are a frame") } /// The schema of a frame carrying these series, which is what the node declares as its input. @@ -496,6 +500,30 @@ fn frame_schema() -> SchemaRef { frame_schema_of(&raw_schema()) } +/// The description a frame with its labels encoded in a dense [`HOST`] key declares. +fn host_label_decode() -> LabelDecodeSpec { + LabelDecodeSpec { + version: LABEL_DECODE_VERSION, + metric_name: METRIC.to_string(), + encoding: LabelDecodeEncoding::Dense, + primary_key: vec![LabelDecodeColumn { + column_id: 1, + name: HOST.to_string(), + physical_type: ConcreteDataType::string_datatype(), + }], + logical_tags: vec![LabelDecodeTag { + column_id: 1, + name: HOST.to_string(), + }], + } +} + +/// The schema of a frame carrying series of `schema` whose labels are encoded in their key. +fn keyed_frame_schema_of(schema: &SeriesSchema) -> SchemaRef { + series_frame_schema(schema, FrameLabelMode::Key, Some(&host_label_decode())) + .expect("key labels are a frame") +} + /// The columns this node produces: the field, the tags and the grid timestamps. fn output_schema() -> SchemaRef { Arc::new(Schema::new(vec![ @@ -975,7 +1003,7 @@ fn metric(labels: &[(&str, &str)], name: Option<&str>) -> SeriesLabels { /// One frame batch carrying `chunk` of `schema`, packed by the scan's own encoder. fn frame_of(schema: &SeriesSchema, chunk: &RawSeriesChunk) -> RecordBatch { - let mut batches = pack_raw_series_chunk(schema, RawLabelMode::Explicit, chunk) + let mut batches = pack_raw_series_chunk(schema, FrameLabelMode::Explicit, None, chunk) .expect("a chunk of this schema packs into a frame"); assert_eq!(1, batches.len(), "one small chunk is one frame batch"); batches.remove(0) @@ -2270,8 +2298,7 @@ async fn a_pending_boundary_loses_no_rows() { /// before the first batch, not when the missing labels reach the output. #[test] fn a_frame_whose_labels_are_encoded_is_rejected_when_the_plan_is_built() { - let keyed = - raw_series_frame_schema(&raw_schema(), RawLabelMode::Key).expect("key labels are a frame"); + let keyed = keyed_frame_schema_of(&raw_schema()); let error = plan(rate_tree(), memory_exec_with(keyed, Vec::new())) .expect_err("the frames carry no labels to read tags from"); assert!( @@ -2283,18 +2310,19 @@ fn a_frame_whose_labels_are_encoded_is_rejected_when_the_plan_is_built() { /// A frame that carries an evaluation grid is refused when the plan is built. /// /// A frame holds raw series: its rows are samples, not grid positions, and the grid every output -/// row is addressed on is the tree's own. A schema that declares a grid is one this node would -/// read positions out of, so it is refused rather than decoded. The source below is one partition, -/// which is the shape the node accepts, so what refuses the plan here is the frame's own -/// declaration and not the node's partition count: the refusal has to name the grid. +/// row is addressed on is the tree's own. A frame that declares a grid is one whose rows this node +/// would read as positions on it, so the shared decoder refuses it rather than reading samples out +/// of it. The source below is one partition, which is the shape the node accepts, so what refuses +/// the plan here is the frame's own declaration and not the node's partition count: the refusal has +/// to name the frame. #[test] fn a_frame_with_an_evaluation_grid_is_rejected_when_the_plan_is_built() { - let mut metadata = frame_schema().metadata().clone(); - metadata.insert("greptime.series.grid".to_string(), "v1".to_string()); - let gridded = Arc::new(Schema::new_with_metadata( - frame_schema().fields().clone(), - metadata, - )); + let gridded_schema = Arc::new(SeriesSchema { + fields: raw_schema().fields.clone(), + eval_grid: Some(grid()), + }); + let gridded = series_frame_schema(&gridded_schema, FrameLabelMode::Explicit, None) + .expect("a grid is a frame"); // One empty partition: every check but the frame's passes. let source = memory_exec_with(gridded, Vec::new()); assert_eq!( @@ -2305,7 +2333,7 @@ fn a_frame_with_an_evaluation_grid_is_rejected_when_the_plan_is_built() { let error = plan(rate_tree(), source).expect_err("the frame declares a grid"); let message = error_chain(&error); assert!( - message.contains("raw series schema cannot have an evaluation grid"), + message.contains("expected a raw frame, found an evaluated one"), "refused for another reason: {message}" ); assert!( @@ -2326,7 +2354,9 @@ fn a_frame_of_another_series_schema_is_rejected_when_the_plan_is_built() { let error = plan(rate_tree(), memory_exec_with(other, Vec::new())) .expect_err("the tree's input reads one field"); assert!( - error.to_string().contains("raw frame"), + error + .to_string() + .contains("frame fields do not match the expected series schema"), "refused for another reason: {error}" ); } diff --git a/src/promql/src/source/merge/tests.rs b/src/promql/src/source/merge/tests.rs index ff8a8ef9ad9..b7fcc0e0c69 100644 --- a/src/promql/src/source/merge/tests.rs +++ b/src/promql/src/source/merge/tests.rs @@ -24,7 +24,7 @@ use api::v1::SemanticType; use common_error::ext::{BoxedError, ErrorExt, PlainError}; use common_error::status_code::StatusCode; use common_query::series_frame::{ - label_decode_spec, pack_raw_series, raw_series_frame_ordering, unpack_raw_series, + label_decode_spec, pack_raw_series, series_frame_ordering, unpack_raw_series, }; use datatypes::arrow::array::Float64Array; use datatypes::arrow::buffer::{Buffer, ScalarBuffer}; @@ -458,7 +458,7 @@ async fn merges_two_unpacked_frames_through_the_frame_codec() { ); let schema = Arc::clone(stream.schema()); let packed = pack_raw_series(stream).unwrap(); - let ordering = raw_series_frame_ordering(packed.schema().as_ref()).unwrap(); + let ordering = series_frame_ordering(packed.schema().as_ref()).unwrap(); assert_eq!(SeriesOrdering::KeyThenTime, ordering); unpack_raw_series(packed, Some(schema)).unwrap() }; @@ -1889,7 +1889,7 @@ async fn merge_rejects_inputs_that_describe_different_decode_layouts() { /// that its chunks are undecoded key chunks, which is what a consumer decodes from. #[tokio::test] async fn merge_without_a_bound_identity_still_advertises_the_decode_layout() { - use common_query::series_frame::raw_series_frame_ordering; + use common_query::series_frame::series_frame_ordering; use common_query::series_frame::{pack_raw_series, unpack_raw_series}; use store_api::region_engine::NO_BOUND_LABEL_SOURCE; @@ -1937,11 +1937,11 @@ async fn merge_without_a_bound_identity_still_advertises_the_decode_layout() { // Packing advertises that layout under the merge's ordering, so the frame states both. let framed = pack_raw_series(merged).unwrap(); assert_eq!( - datatypes::series::frame::raw_series_frame_label_decode(framed.schema().as_ref()).unwrap(), + datatypes::series::frame::series_frame_label_decode(framed.schema().as_ref()).unwrap(), Some(carried.clone()) ); assert_eq!( - raw_series_frame_ordering(framed.schema().as_ref()).unwrap(), + series_frame_ordering(framed.schema().as_ref()).unwrap(), SeriesOrdering::KeyThenTime ); @@ -1970,7 +1970,7 @@ async fn merge_without_a_bound_identity_still_advertises_the_decode_layout() { #[tokio::test] async fn merged_frame_declares_a_label_decode_description_that_decodes_its_keys() { use common_query::series_frame::{pack_raw_series, unpack_raw_series}; - use datatypes::series::frame::raw_series_frame_label_decode; + use datatypes::series::frame::series_frame_label_decode; use mito_codec::labels::{LabelProjection, PrimaryKeyLabelDecoder}; let key = read_key("rack-a", "sensor-1"); @@ -1994,12 +1994,12 @@ async fn merged_frame_declares_a_label_decode_description_that_decodes_its_keys( let ordering = merged.ordering(); let framed = pack_raw_series(merged).unwrap(); assert_eq!( - common_query::series_frame::raw_series_frame_ordering(framed.schema().as_ref()).unwrap(), + common_query::series_frame::series_frame_ordering(framed.schema().as_ref()).unwrap(), ordering ); let declared = framed.schema(); assert_eq!( - raw_series_frame_label_decode(declared.as_ref()).unwrap(), + series_frame_label_decode(declared.as_ref()).unwrap(), Some(expected.clone()) ); diff --git a/src/promql/src/source/merge/tests/payload.rs b/src/promql/src/source/merge/tests/payload.rs index a180a01784a..459fad9ef57 100644 --- a/src/promql/src/source/merge/tests/payload.rs +++ b/src/promql/src/source/merge/tests/payload.rs @@ -19,8 +19,8 @@ use std::task::{Context, Poll}; use common_error::ext::BoxedError; use common_query::series_frame::{ - ensure_frame_matches_label_source, label_decode_spec, pack_raw_series, - raw_series_frame_ordering, unpack_raw_series, + ensure_frame_matches_label_source, label_decode_spec, pack_raw_series, series_frame_ordering, + unpack_raw_series, }; use datatypes::arrow::array::{Array, Float64Array, ListArray, UInt32Array}; use datatypes::arrow::buffer::{Buffer, ScalarBuffer}; @@ -263,7 +263,7 @@ async fn merges_typed_multifield_payload_through_frame_and_label_roundtrip() { // declares, since the merge only accepts inputs that declare one layout. ensure_frame_matches_label_source(framed.schema().as_ref(), &label_source(1)).unwrap(); assert_eq!( - raw_series_frame_ordering(framed.schema().as_ref()).unwrap(), + series_frame_ordering(framed.schema().as_ref()).unwrap(), frame_ordering, "the packed frame declares the merge's ordering" ); diff --git a/src/promql/src/source/tests.rs b/src/promql/src/source/tests.rs index 10c955dd3cf..acfd6d4107e 100644 --- a/src/promql/src/source/tests.rs +++ b/src/promql/src/source/tests.rs @@ -1073,8 +1073,7 @@ async fn materializes_labels_from_a_frame_with_no_supplied_binding() { ); let framed = pack_raw_series(raw).unwrap(); assert_eq!( - datatypes::series::frame::raw_series_frame_label_decode(framed.schema().as_ref()) - .unwrap(), + datatypes::series::frame::series_frame_label_decode(framed.schema().as_ref()).unwrap(), Some(description.clone()), "{encoding:?}: the frame schema is the producer's only label statement" ); diff --git a/src/query/src/dist_plan/merge_scan.rs b/src/query/src/dist_plan/merge_scan.rs index a9107b49375..78fb9792490 100644 --- a/src/query/src/dist_plan/merge_scan.rs +++ b/src/query/src/dist_plan/merge_scan.rs @@ -1423,7 +1423,7 @@ mod tests { }; use datatypes::prelude::{ConcreteDataType, VectorRef}; use datatypes::schema::{ColumnSchema, Schema}; - use datatypes::series::frame::{self, RawLabelMode}; + use datatypes::series::frame::{self, FrameLabelMode}; use datatypes::series::raw::{RawSeriesChunk, RawTimestamps}; use datatypes::series::samples::{ DenseSamplesRef, FieldSpec, SampleRef, Samples, SeriesFields, SeriesSchema, @@ -3361,7 +3361,8 @@ mod tests { async fn merge_scan_remote_schema_identity_rejects_empty_raw_frame_metadata_mismatch() { let region_id = RegionId::new(1024, 1); let raw = raw_test_series_schema(); - let frame = frame::raw_series_frame_schema(&raw, RawLabelMode::Key).unwrap(); + let decode = raw_test_label_decode(); + let frame = frame::series_frame_schema(&raw, FrameLabelMode::Key, Some(&decode)).unwrap(); let metadata = frame.metadata().clone(); let mut cases = Vec::new(); for (key, value) in [ @@ -3428,7 +3429,7 @@ mod tests { label_decode_spec, raw_series_frame_schema_with_ordering, }; use datatypes::series::SeriesOrdering; - use datatypes::series::frame::raw_series_frame_label_decode; + use datatypes::series::frame::series_frame_label_decode; let raw = raw_test_series_schema(); // Two bindings that are equal but not the same allocation: the expectation and the @@ -3503,7 +3504,7 @@ mod tests { spaced_metadata.insert(key.to_string(), spaced_text); let spaced = ArrowSchema::new_with_metadata(expected.fields().clone(), spaced_metadata); assert_eq!( - raw_series_frame_label_decode(&spaced) + series_frame_label_decode(&spaced) .unwrap() .map(|decode| decode.version), Some(1), @@ -3512,6 +3513,27 @@ mod tests { assert!(validate_remote_schema(&expected, &spaced, "test").is_err()); } + /// The key layout a key frame over [`raw_test_series_schema`] declares. + /// + /// A key-encoded frame cannot be declared without the description that decodes its key, and the + /// merge-scan tests never decode a key with it, so it only has to be self-consistent. + fn raw_test_label_decode() -> frame::LabelDecodeSpec { + frame::LabelDecodeSpec { + version: frame::LABEL_DECODE_VERSION, + metric_name: "cpu_usage".to_string(), + encoding: frame::LabelDecodeEncoding::Dense, + primary_key: vec![frame::LabelDecodeColumn { + column_id: 1, + name: "host".to_string(), + physical_type: ConcreteDataType::string_datatype(), + }], + logical_tags: vec![frame::LabelDecodeTag { + column_id: 1, + name: "host".to_string(), + }], + } + } + fn raw_test_series_schema() -> SeriesSchema { SeriesSchema { fields: vec![ @@ -3531,8 +3553,9 @@ mod tests { async fn merge_scan_remote_schema_identity_rejects_later_raw_frame_inner_metadata_mismatch() { let region_id = RegionId::new(1024, 1); let raw = raw_test_series_schema(); - let mode = RawLabelMode::Key; - let frame = frame::raw_series_frame_schema(&raw, mode).unwrap(); + let mode = FrameLabelMode::Key; + let decode = raw_test_label_decode(); + let frame = frame::series_frame_schema(&raw, mode, Some(&decode)).unwrap(); let mut changed_metadata = frame.metadata().clone(); changed_metadata.insert("greptime.series.marker".to_string(), "changed".to_string()); let changed_schema = Arc::new(ArrowSchema::new_with_metadata( @@ -3554,13 +3577,22 @@ mod tests { ) .unwrap() }; - let first = frame::pack_raw_series_chunk(&raw, mode, &chunk([1.0, 2.0], [10, 20], false)) - .unwrap() - .remove(0); - let mut second = - frame::pack_raw_series_chunk(&raw, mode, &chunk([3.0, 4.0], [30, 40], true)) - .unwrap() - .remove(0); + let first = frame::pack_raw_series_chunk( + &raw, + mode, + Some(&decode), + &chunk([1.0, 2.0], [10, 20], false), + ) + .unwrap() + .remove(0); + let mut second = frame::pack_raw_series_chunk( + &raw, + mode, + Some(&decode), + &chunk([3.0, 4.0], [30, 40], true), + ) + .unwrap() + .remove(0); second = DfRecordBatch::try_new(changed_schema, second.columns().to_vec()).unwrap(); let advertised = Arc::new(Schema::try_from(frame.clone()).unwrap()); let first = RecordBatch::from_df_record_batch(advertised.clone(), first); @@ -3578,7 +3610,7 @@ mod tests { ); let mut stream = exec.execute(0, Arc::new(TaskContext::default())).unwrap(); let emitted = stream.next().await.unwrap().unwrap(); - let decoded = frame::unpack_raw_series_frame(&raw, mode, &emitted).unwrap(); + let decoded = frame::unpack_raw_series_frame(&raw, mode, Some(&decode), &emitted).unwrap(); assert_eq!(decoded[0].time.timestamp(0), Some(10)); assert!(matches!( decoded[0].fields.get(0).unwrap().sample(0), @@ -3593,7 +3625,9 @@ mod tests { async fn merge_scan_remote_schema_identity_rejects_raw_frame_field_metadata_mismatch() { let region_id = RegionId::new(1024, 1); let raw = raw_test_series_schema(); - let expected = frame::raw_series_frame_schema(&raw, RawLabelMode::Key).unwrap(); + let expected = + frame::series_frame_schema(&raw, FrameLabelMode::Key, Some(&raw_test_label_decode())) + .unwrap(); let frame_metadata = expected.metadata().clone(); let fields = expected.fields().to_vec(); let mut field_cases = Vec::new(); @@ -3678,7 +3712,9 @@ mod tests { async fn merge_scan_remote_schema_identity_preserves_valid_raw_frame_payload() { let region_id = RegionId::new(1024, 1); let raw = raw_test_series_schema(); - let expected_frame = frame::raw_series_frame_schema(&raw, RawLabelMode::Key).unwrap(); + let decode = raw_test_label_decode(); + let expected_frame = + frame::series_frame_schema(&raw, FrameLabelMode::Key, Some(&decode)).unwrap(); let mut expected_metadata = expected_frame.metadata().clone(); expected_metadata.insert("application.note".to_string(), "expected".to_string()); let expected = Arc::new(ArrowSchema::new_with_metadata( @@ -3702,7 +3738,8 @@ mod tests { true, ) .unwrap(); - let packed = frame::pack_raw_series_chunk(&raw, RawLabelMode::Key, &chunk).unwrap(); + let packed = + frame::pack_raw_series_chunk(&raw, FrameLabelMode::Key, Some(&decode), &chunk).unwrap(); let inner = packed.into_iter().next().unwrap(); let original_columns = inner.columns().to_vec(); let batch = RecordBatch::from_df_record_batch(remote_schema.clone(), inner); @@ -3722,7 +3759,8 @@ mod tests { for (original, returned) in original_columns.iter().zip(result[0].columns()) { assert!(Arc::ptr_eq(original, returned)); } - let decoded = frame::unpack_raw_series_frame(&raw, RawLabelMode::Key, &result[0]); + let decoded = + frame::unpack_raw_series_frame(&raw, FrameLabelMode::Key, Some(&decode), &result[0]); let decoded = decoded.unwrap(); assert_eq!(decoded.len(), 1); assert_eq!(decoded[0].primary_key.as_slice(), &[1, 2, 3]); diff --git a/src/table/src/table/scan/series_tests.rs b/src/table/src/table/scan/series_tests.rs index f2120924877..2f1b0d50a3f 100644 --- a/src/table/src/table/scan/series_tests.rs +++ b/src/table/src/table/scan/series_tests.rs @@ -20,7 +20,7 @@ use std::task::{Context, Poll}; use api::v1::SemanticType; use common_error::ext::BoxedError; use common_query::request::RawSeriesRequest; -use common_query::series_frame::raw_series_frame_ordering; +use common_query::series_frame::series_frame_ordering; use common_recordbatch::{ QueryMemoryTracker, RecordBatch, RecordBatches, SendableRecordBatchStream, }; @@ -36,7 +36,7 @@ use datatypes::data_type::ConcreteDataType; use datatypes::error::InvalidSeriesSnafu; use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; use datatypes::series::SeriesOrdering; -use datatypes::series::frame::{LabelDecodeSpec, RawLabelMode}; +use datatypes::series::frame::{FrameLabelMode, LabelDecodeSpec}; use datatypes::series::raw::{RawSeriesChunk, RawTimestamps}; use datatypes::series::samples::{FieldSpec, Samples, SeriesFields, SeriesSchema}; use datatypes::vectors::{Int32Vector, TimestampMillisecondVector, VectorRef}; @@ -673,7 +673,8 @@ async fn native_raw_frame_executes_and_decodes_bound_data() { assert_eq!(frames[0].schema(), plan.schema()); let decoded = datatypes::series::frame::unpack_raw_series_frame( &schema, - RawLabelMode::Explicit, + FrameLabelMode::Explicit, + None, &frames[0], ) .unwrap(); @@ -765,10 +766,7 @@ async fn planned_ordering_is_preserved_for_empty_and_nonempty_frames() { .unwrap() .unwrap(); let planned_schema = plan.schema(); - assert_eq!( - ordering, - raw_series_frame_ordering(&planned_schema).unwrap() - ); + assert_eq!(ordering, series_frame_ordering(&planned_schema).unwrap()); { let state = state.lock().unwrap(); assert_eq!(0, state.raw_calls); @@ -783,7 +781,7 @@ async fn planned_ordering_is_preserved_for_empty_and_nonempty_frames() { ); assert_eq!( ordering, - raw_series_frame_ordering(stream.schema().as_ref()).unwrap() + series_frame_ordering(stream.schema().as_ref()).unwrap() ); let frames = stream.try_collect::>().await.unwrap(); assert_eq!(usize::from(nonempty), frames.len()); @@ -791,7 +789,7 @@ async fn planned_ordering_is_preserved_for_empty_and_nonempty_frames() { assert_eq!(planned_schema.as_ref(), frame.schema().as_ref()); assert_eq!( ordering, - raw_series_frame_ordering(frame.schema().as_ref()).unwrap() + series_frame_ordering(frame.schema().as_ref()).unwrap() ); } let state = state.lock().unwrap(); @@ -839,7 +837,7 @@ async fn planned_and_stream_ordering_mismatches_fail_without_legacy_fallback() { .unwrap(); assert_eq!( planned, - raw_series_frame_ordering(plan.schema().as_ref()).unwrap() + series_frame_ordering(plan.schema().as_ref()).unwrap() ); state.lock().unwrap().stream_ordering = Some(actual); let error = plan @@ -1171,7 +1169,8 @@ async fn error_after_emitted_frame_is_not_replayed() { assert_eq!(frame.schema(), plan.schema()); let decoded = datatypes::series::frame::unpack_raw_series_frame( plan.series_schema(), - RawLabelMode::Explicit, + FrameLabelMode::Explicit, + None, &frame, ) .unwrap(); @@ -1354,7 +1353,8 @@ async fn tracked_native_frame_holds_and_releases_quota_on_cancel() { assert_eq!( datatypes::series::frame::unpack_raw_series_frame( plan.series_schema(), - RawLabelMode::Explicit, + FrameLabelMode::Explicit, + None, &frame, ) .unwrap()[0] @@ -1484,7 +1484,7 @@ async fn raw_series_admission_captures_scanner_ordering_before_execution() { .unwrap(); assert_eq!(plan.raw_series_ordering(), ordering); assert_eq!( - raw_series_frame_ordering(plan.schema().as_ref()).unwrap(), + series_frame_ordering(plan.schema().as_ref()).unwrap(), ordering ); { diff --git a/tests/compatibility/wire/raw_series_label_decode/README.md b/tests/compatibility/wire/raw_series_label_decode/README.md index a7d5c36858f..048b5bf54d9 100644 --- a/tests/compatibility/wire/raw_series_label_decode/README.md +++ b/tests/compatibility/wire/raw_series_label_decode/README.md @@ -1,13 +1,14 @@ # Raw-series label decode description compatibility -`v1_metadata.json` is a hand-authored, frozen set of Arrow schema metadata maps -for the v1 raw-series frame. It records the established layout/content/labels/ -time metadata plus the new `greptime.series.label_decode` entry, whose value is a -JSON label decode description with an explicit `version`. The Rust unit test -parses these literal maps through `raw_series_frame_label_decode`; the fixture is -not produced by the schema builder. `v1_physical_types.json`, described under +`v2_metadata.json` is a hand-authored, frozen set of Arrow schema metadata maps +for the v2 series frame. It records the established layout/content/labels/time +metadata plus the `greptime.series.label_decode` entry, whose value is a JSON +label decode description with an explicit `version`. The Rust unit test parses +these literal maps through `series_frame_label_decode`; the fixture is not +produced by the schema builder. `v1_physical_types.json`, described under [Physical types](#physical-types), is the one generated fixture here: it records -the serde text of every `ConcreteDataType` variant. +the serde text of every `ConcreteDataType` variant, which the frame layout version +does not change, so it keeps its original name. Run the focused check: @@ -22,6 +23,17 @@ column, and that the reader rejects a key frame without a description, an explic frame that declares one, an unknown version, a missing or unknown field, a wrong field type, and a non-JSON value. +It also pins the declarations the unified frame family refuses. Layout is now +`v2`: a frame declaring `v1` or any other value is refused, and so is a +`greptime.series.*` key this revision does not register. The three modes are not +independent - `content` fixes `time`, and `labels` decides whether a description +must be present - so `content = evaluated` with `time = timestamps`, `content = +evaluated` with `labels = key`, and `content = raw` with a +`greptime.series.grid` are all contradictions and are refused rather than +half-trusted. An evaluated frame is therefore always `explicit` + `grid_spans` and +always declares its grid; a raw frame is always `timestamps` and never declares +one. + A timestamp-typed tag column has its own case in the crate's unit tests rather than here, so this fixture stays a single fixed layout. diff --git a/tests/compatibility/wire/raw_series_label_decode/v1_metadata.json b/tests/compatibility/wire/raw_series_label_decode/v2_metadata.json similarity index 62% rename from tests/compatibility/wire/raw_series_label_decode/v1_metadata.json rename to tests/compatibility/wire/raw_series_label_decode/v2_metadata.json index 2e4e6d7f24b..ee32ec14c49 100644 --- a/tests/compatibility/wire/raw_series_label_decode/v1_metadata.json +++ b/tests/compatibility/wire/raw_series_label_decode/v2_metadata.json @@ -2,7 +2,7 @@ { "name": "key_frame_declares_dense_description", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps", @@ -13,7 +13,7 @@ { "name": "key_frame_declares_sparse_description", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps", @@ -24,7 +24,7 @@ { "name": "missing_description_rejected", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps" @@ -34,7 +34,7 @@ { "name": "unknown_version_rejected", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps", @@ -45,7 +45,7 @@ { "name": "missing_field_rejected", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps", @@ -56,7 +56,7 @@ { "name": "unknown_field_rejected", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps", @@ -67,7 +67,7 @@ { "name": "wrong_field_type_rejected", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps", @@ -78,7 +78,7 @@ { "name": "unlisted_logical_tag_rejected", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps", @@ -89,7 +89,7 @@ { "name": "explicit_frame_with_description_rejected", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "explicit", "greptime.series.time": "timestamps", @@ -100,12 +100,69 @@ { "name": "malformed_json_rejected", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps", "greptime.series.label_decode": "greptime" }, "encoding": null + }, + { + "name": "wrong_layout_rejected", + "metadata": { + "greptime.series.layout": "v3", + "greptime.series.content": "raw", + "greptime.series.labels": "key", + "greptime.series.time": "timestamps", + "greptime.series.label_decode": "{\"version\":1,\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}" + }, + "encoding": null + }, + { + "name": "unregistered_reserved_metadata_rejected", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "raw", + "greptime.series.labels": "key", + "greptime.series.time": "timestamps", + "greptime.series.label_decode": "{\"version\":1,\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}", + "greptime.series.future": "x" + }, + "encoding": null + }, + { + "name": "evaluated_with_timestamps_rejected", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "evaluated", + "greptime.series.labels": "explicit", + "greptime.series.time": "timestamps", + "greptime.series.grid": "{\"start_ms\":0,\"step_ms\":10000,\"len\":4}" + }, + "encoding": null + }, + { + "name": "evaluated_with_key_labels_rejected", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "evaluated", + "greptime.series.labels": "key", + "greptime.series.time": "grid_spans", + "greptime.series.label_decode": "{\"version\":1,\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}", + "greptime.series.grid": "{\"start_ms\":0,\"step_ms\":10000,\"len\":4}" + }, + "encoding": null + }, + { + "name": "raw_with_grid_rejected", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "raw", + "greptime.series.labels": "explicit", + "greptime.series.time": "timestamps", + "greptime.series.grid": "{\"start_ms\":0,\"step_ms\":10000,\"len\":4}" + }, + "encoding": null } ] diff --git a/tests/compatibility/wire/raw_series_ordering/README.md b/tests/compatibility/wire/raw_series_ordering/README.md index 630dc25e2a2..877f811d3f7 100644 --- a/tests/compatibility/wire/raw_series_ordering/README.md +++ b/tests/compatibility/wire/raw_series_ordering/README.md @@ -1,10 +1,11 @@ # Raw-series ordering schema metadata compatibility -`v1_metadata.json` is a hand-authored, frozen set of Arrow schema metadata maps -for the v1 raw-series frame. It records the established layout/content/labels/ -time metadata and the new optional ordering entry. The Rust unit test parses -these literal maps through `raw_series_frame_ordering`; the fixture is not -produced by the schema builder. +`v2_metadata.json` is a hand-authored, frozen set of Arrow schema metadata maps +for the v2 series frame. It records the established layout/content/labels/time +metadata and the optional ordering entry, for raw frames and for evaluated ones. +The Rust unit test parses these literal maps through `series_frame_ordering` and +through `SeriesFrameDeclaration::parse`; the fixture is not produced by the +schema builder. Run the focused check: @@ -13,7 +14,24 @@ cargo nextest run -p common-query -E 'test(raw_series_ordering_metadata_fixture) ``` The fixture proves that absent ordering metadata defaults to `Contiguous`, the -two known values decode, and an unknown value is rejected. It does not prove -interoperability with an old binary, nor does it test an end-to-end client or -service. Separate pack/IPC tests check the declared ordering schema and batch -schema agreement for both orderings, including empty streams. +two known values decode, an unknown value is rejected, and that a whole +declaration is accepted or refused consistently with its layout and content: a +`v3` layout, an unregistered `greptime.series.*` key, a raw frame that declares a +grid, and any content/time or content/labels contradiction are refusals. The +evaluated cases are read as declarations only; the streaming round trips for +evaluated frames are covered by the crate's evaluated tests. + +Because the declaration is cross-checked, a frame that names `evaluated` content +is only an evaluated frame when it also declares `explicit` labels, `grid_spans` +time, and a `greptime.series.grid`. `read_key_then_time` parses as a declaration +(a raw frame may legitimately claim it), but no writer produces one on an +evaluated frame: computed values have no storage key to be ordered by, so both the +schema builder and `unpack_evaluated_series` refuse it. The fixture keeps a +hand-written evaluated frame that states it, because a reader must still refuse +such a frame rather than decode it under an ordering its producer could not have +guaranteed. + +The fixture does not prove interoperability with an old binary, nor does it test +an end-to-end client or service. Separate pack/IPC tests check the declared +ordering schema and batch schema agreement for both orderings, including empty +streams, and check the same for the evaluated frame across Arrow IPC. diff --git a/tests/compatibility/wire/raw_series_ordering/v1_metadata.json b/tests/compatibility/wire/raw_series_ordering/v1_metadata.json deleted file mode 100644 index c5b3c0ea69b..00000000000 --- a/tests/compatibility/wire/raw_series_ordering/v1_metadata.json +++ /dev/null @@ -1,45 +0,0 @@ -[ - { - "name": "missing_defaults_to_contiguous", - "metadata": { - "greptime.series.layout": "v1", - "greptime.series.content": "raw", - "greptime.series.labels": "explicit", - "greptime.series.time": "timestamps" - }, - "ordering": "contiguous" - }, - { - "name": "known_contiguous", - "metadata": { - "greptime.series.layout": "v1", - "greptime.series.content": "raw", - "greptime.series.labels": "explicit", - "greptime.series.time": "timestamps", - "greptime.series.ordering": "contiguous" - }, - "ordering": "contiguous" - }, - { - "name": "known_read_key_then_time", - "metadata": { - "greptime.series.layout": "v1", - "greptime.series.content": "raw", - "greptime.series.labels": "explicit", - "greptime.series.time": "timestamps", - "greptime.series.ordering": "read_key_then_time" - }, - "ordering": "read_key_then_time" - }, - { - "name": "unknown_ordering_rejected", - "metadata": { - "greptime.series.layout": "v1", - "greptime.series.content": "raw", - "greptime.series.labels": "explicit", - "greptime.series.time": "timestamps", - "greptime.series.ordering": "future_value" - }, - "ordering": null - } -] diff --git a/tests/compatibility/wire/raw_series_ordering/v2_metadata.json b/tests/compatibility/wire/raw_series_ordering/v2_metadata.json new file mode 100644 index 00000000000..8dd827b17d1 --- /dev/null +++ b/tests/compatibility/wire/raw_series_ordering/v2_metadata.json @@ -0,0 +1,98 @@ +[ + { + "name": "missing_defaults_to_contiguous", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "raw", + "greptime.series.labels": "explicit", + "greptime.series.time": "timestamps" + }, + "ordering": "contiguous", + "declaration": "accepted" + }, + { + "name": "known_contiguous", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "raw", + "greptime.series.labels": "explicit", + "greptime.series.time": "timestamps", + "greptime.series.ordering": "contiguous" + }, + "ordering": "contiguous", + "declaration": "accepted" + }, + { + "name": "known_read_key_then_time", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "raw", + "greptime.series.labels": "explicit", + "greptime.series.time": "timestamps", + "greptime.series.ordering": "read_key_then_time" + }, + "ordering": "read_key_then_time", + "declaration": "accepted" + }, + { + "name": "unknown_ordering_rejected", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "raw", + "greptime.series.labels": "explicit", + "greptime.series.time": "timestamps", + "greptime.series.ordering": "future_value" + }, + "ordering": null, + "declaration": "accepted" + }, + { + "name": "evaluated_contiguous", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "evaluated", + "greptime.series.labels": "explicit", + "greptime.series.time": "grid_spans", + "greptime.series.grid": "{\"start_ms\":0,\"step_ms\":10000,\"len\":4}", + "greptime.series.ordering": "contiguous" + }, + "ordering": "contiguous", + "declaration": "accepted" + }, + { + "name": "evaluated_read_key_then_time_still_reads", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "evaluated", + "greptime.series.labels": "explicit", + "greptime.series.time": "grid_spans", + "greptime.series.grid": "{\"start_ms\":0,\"step_ms\":10000,\"len\":4}", + "greptime.series.ordering": "read_key_then_time" + }, + "ordering": "read_key_then_time", + "declaration": "accepted" + }, + { + "name": "wrong_layout_rejected", + "metadata": { + "greptime.series.layout": "v3", + "greptime.series.content": "raw", + "greptime.series.labels": "explicit", + "greptime.series.time": "timestamps" + }, + "ordering": "contiguous", + "declaration": "rejected" + }, + { + "name": "unregistered_reserved_metadata_rejected", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "raw", + "greptime.series.labels": "explicit", + "greptime.series.time": "timestamps", + "greptime.series.future": "x" + }, + "ordering": "contiguous", + "declaration": "rejected" + } +]