diff --git a/src/common/query/src/series_frame.rs b/src/common/query/src/series_frame.rs index d18cdab86f1..04096c2b903 100644 --- a/src/common/query/src/series_frame.rs +++ b/src/common/query/src/series_frame.rs @@ -27,8 +27,12 @@ //! preserves //! scan candidates only; it is not a residual filter, authorization check, remote transport, or //! compaction boundary. Only -//! [`SeriesOrdering::Contiguous`] and [`SeriesOrdering::KeyThenTime`] are accepted; -//! `Interleaved` is rejected before any poll. +//! [`SeriesOrdering::Contiguous`] and [`SeriesOrdering::KeyThenTime`] are accepted for raw frames and +//! only `Contiguous` for evaluated ones; `Interleaved` is rejected before any poll. +//! +//! An evaluated frame is the same physical family: [`pack_evaluated_series`] and +//! [`unpack_evaluated_series`] carry computed values under complete labels and grid positions, so a +//! compute consumer needs neither a storage key nor a scan binding to read them. use std::pin::Pin; use std::sync::Arc; @@ -40,13 +44,12 @@ use datafusion_common::DataFusionError; use datatypes::arrow::datatypes::{Schema as ArrowSchema, SchemaRef as ArrowSchemaRef}; use datatypes::error::InvalidSeriesSnafu; use datatypes::series::frame::{ - LABEL_DECODE_VERSION, LabelDecodeColumn, LabelDecodeEncoding, LabelDecodeSpec, LabelDecodeTag, - RawLabelMode, RawSeriesChunkCursor, RawSeriesFrameBuilder, RawSeriesFrameDecoder, - raw_series_frame_label_decode, raw_series_frame_schema_with_label_decode, - raw_series_frame_series_schema, + FrameContent, FrameLabelMode, LABEL_DECODE_VERSION, LabelDecodeColumn, LabelDecodeEncoding, + LabelDecodeSpec, LabelDecodeTag, SeriesFrameBuilder, SeriesFrameCursor, SeriesFrameDeclaration, + SeriesFrameDecoder, series_frame_label_decode, series_frame_schema, }; use datatypes::series::raw::RawSeriesChunk; -use datatypes::series::samples::SeriesSchema; +use datatypes::series::samples::{Series, SeriesSchema}; use datatypes::series::{SeriesLabels, SeriesOrdering}; use futures_util::Stream; use store_api::codec::PrimaryKeyEncoding; @@ -63,10 +66,19 @@ mod tests; const ORDERING_KEY: &str = "greptime.series.ordering"; +/// A stream of evaluated series chunks and their finality. +/// +/// The pair is what an evaluated producer has and an evaluated consumer needs: the values on their +/// absolute grid positions, and whether this chunk closed the chain it belongs to. It is deliberately +/// not a third runtime chunk type - `Series` is the runtime value, and finality is the one fact a +/// chunk has to carry besides it. +pub type SendableEvaluatedSeriesStream = + Pin> + Send>>; + /// Builds the canonical raw frame schema with the producer's label binding and ordering. /// /// Ordering records a producer guarantee; it is not inferred from frame contents. `Interleaved` -/// has no v1 wire representation and is rejected before the schema is assembled. The label binding +/// has no wire representation and is rejected before the schema is assembled. The label binding /// determines both the label mode and, for a primary-key binding, the self-describing label decode /// description written into the schema, so a frame can never declare a label layout it does not /// describe. @@ -75,6 +87,11 @@ pub fn raw_series_frame_schema_with_ordering( label_source: &RawLabelSource, ordering: SeriesOrdering, ) -> datafusion_common::Result { + if schema.eval_grid.is_some() { + return Err(invalid( + "a raw frame cannot carry a schema with an evaluation grid", + )); + } frame_schema_with( schema, label_mode(label_source), @@ -85,26 +102,30 @@ pub fn raw_series_frame_schema_with_ordering( /// Assembles a declared frame schema from an explicit label mode and decode description. /// -/// The mode and description pair is checked by -/// [`raw_series_frame_schema_with_label_decode`], so a key mode without a description and an -/// explicit mode with one are both rejected here instead of reaching the wire. +/// The mode and description pair is checked by [`series_frame_schema`], so a key mode without a +/// description and an explicit mode with one are both rejected here instead of reaching the wire. fn frame_schema_with( schema: &SeriesSchema, - mode: RawLabelMode, + mode: FrameLabelMode, decode: Option<&LabelDecodeSpec>, ordering: SeriesOrdering, ) -> datafusion_common::Result { + // An evaluated frame has no storage key to be ordered by, so the only claim it can honour is + // that each chain is contiguous. Writing a stronger one would produce a frame the reader refuses + // and a plan that fails where it could have been refused. + if schema.eval_grid.is_some() { + ensure_contiguous_ordering(ordering)?; + } let ordering = match ordering { SeriesOrdering::Contiguous => "contiguous", SeriesOrdering::KeyThenTime => "read_key_then_time", SeriesOrdering::Interleaved => { return Err(invalid( - "interleaved raw series ordering has no wire representation", + "interleaved series ordering has no wire representation", )); } }; - let schema = - raw_series_frame_schema_with_label_decode(schema, mode, decode).map_err(codec_error)?; + let schema = series_frame_schema(schema, mode, decode).map_err(codec_error)?; let mut metadata = schema.metadata().clone(); metadata.insert(ORDERING_KEY.to_string(), ordering.to_string()); Ok(Arc::new(ArrowSchema::new_with_metadata( @@ -163,28 +184,26 @@ pub fn label_decode_spec(source: &RawLabelSource) -> Option { Some(description) } -/// Reads the declared producer ordering, conservatively treating legacy frames as contiguous. +/// Reads the producer ordering declared by a complete v2 frame schema. /// -/// Only the two orderings representable on the wire can be returned. -pub fn raw_series_frame_ordering( - schema: &ArrowSchema, -) -> datafusion_common::Result { +/// Ordering is mandatory stream metadata; it is never inferred from frame contents. +pub fn series_frame_ordering(schema: &ArrowSchema) -> datafusion_common::Result { match schema.metadata().get(ORDERING_KEY).map(String::as_str) { - None | Some("contiguous") => Ok(SeriesOrdering::Contiguous), + Some("contiguous") => Ok(SeriesOrdering::Contiguous), Some("read_key_then_time") => Ok(SeriesOrdering::KeyThenTime), - Some(_) => Err(invalid("unknown raw series ordering")), + None | Some(_) => Err(invalid("missing or unknown series ordering")), } } /// Packs a storage raw-series stream into DataFusion record batches incrementally. /// -/// Only [`SeriesOrdering::Contiguous`] and [`SeriesOrdering::KeyThenTime`] streams are -/// accepted; an `Interleaved` declaration is rejected before the first poll, including for an -/// empty stream. Rows accumulate across chunks into batches bounded by the builder's row and -/// estimated payload limits; one legal oversized row may be emitted alone, and these limits are a -/// work bound, not a resident-memory cap. A ready batch is yielded without polling the source -/// again. Producers must keep each series' chunks contiguous. Validation checks adjacent chunk -/// conditions but does not certify nonadjacent recurrence for `Contiguous` ordering. +/// Only [`SeriesOrdering::Contiguous`] and [`SeriesOrdering::KeyThenTime`] streams are accepted; an +/// `Interleaved` declaration is rejected before the first poll, including for an empty stream. Rows +/// accumulate across chunks into batches bounded by the builder's row and estimated payload limits; +/// one legal oversized row may be emitted alone, and these limits are a work bound, not a +/// resident-memory cap. A ready batch is yielded without polling the source again. Producers must +/// keep each series' chunks contiguous. Validation checks adjacent chunk conditions but does not +/// certify nonadjacent recurrence for `Contiguous` ordering. /// /// The stream's own declaration is the only decode input: a frame unpacked without a scan-bound /// [`RawLabelSource`] round trips unchanged instead of being re-declared from a binding it does not @@ -197,27 +216,71 @@ pub fn pack_raw_series( ensure_supported_ordering(ordering)?; let schema = source.schema().clone(); let decode = source.label_decode_spec().cloned(); - let mode = declared_label_mode(&decode); + let labels = declared_label_mode(&decode); // The declared frame schema carries the producer ordering and the label decode description; - // the shared builder's schema intentionally omits both, so every ready batch is declared under - // this schema before it is yielded, including when no row is ever produced. - let frame_schema = frame_schema_with(&schema, mode, decode.as_ref(), ordering)?; - let builder = RawSeriesFrameBuilder::try_new( + // every ready batch is published under it, including when no row is ever produced. + let frame_schema = frame_schema_with(&schema, labels, decode.as_ref(), ordering)?; + let builder = SeriesFrameBuilder::try_new( &schema, - mode, - RawSeriesFrameBuilder::DEFAULT_MAX_ROWS, - RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, + labels, + decode.as_ref(), + SeriesFrameBuilder::DEFAULT_MAX_ROWS, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, ) .map_err(codec_error)?; Ok(Box::pin(PackStream { source: Some(source), schema, - mode, - frame_schema, + labels, + batches: PackBatches::new(builder, frame_schema), ordering, validator: SequenceValidator::new(ordering), - builder: Some(builder), - cursor: None, + })) +} + +/// Packs a stream of evaluated chunks into DataFusion record batches incrementally. +/// +/// An evaluated frame names its series by their complete labels and addresses time on the evaluation +/// grid `schema` declares, so there is nothing to decode and no storage key to carry: the frame +/// declares `explicit` identity and `grid_spans` time, and the grid is written once instead of +/// repeating timestamps on every row. +/// +/// Only [`SeriesOrdering::Contiguous`] is declared, because that is the one ordering this layer can +/// promise and check: each chunk chain is contiguous with itself. A caller that has reason to claim +/// [`SeriesOrdering::KeyThenTime`] about computed values would have to justify it, and a consumer +/// must never infer it from the presence of labels. +/// +/// Bounds, backpressure, and error handling are the raw path's: rows accumulate into batches bounded +/// by the builder's row and estimated payload limits, a ready batch is yielded before the source is +/// polled again, and a `Pending` source still delivers what is buffered. +pub fn pack_evaluated_series( + source: SendableEvaluatedSeriesStream, + schema: Arc, +) -> datafusion_common::Result { + if schema.eval_grid.is_none() { + return Err(invalid( + "an evaluated frame needs a schema with an evaluation grid", + )); + } + let frame_schema = frame_schema_with( + &schema, + FrameLabelMode::Explicit, + None, + SeriesOrdering::Contiguous, + )?; + let builder = SeriesFrameBuilder::try_new( + &schema, + FrameLabelMode::Explicit, + None, + SeriesFrameBuilder::DEFAULT_MAX_ROWS, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, + ) + .map_err(codec_error)?; + Ok(Box::pin(PackEvaluatedStream { + source: Some(source), + schema, + batches: PackBatches::new(builder, frame_schema), + validator: EvaluatedSequenceValidator::new(), })) } @@ -233,31 +296,40 @@ pub fn pack_raw_series( /// /// Every frame is validated here, unconditionally. The declared layout must be this revision's, the /// label decode description must parse, name a known version, and agree with the declared label -/// mode — a key frame must declare one and an explicit frame must not, since explicit labels are +/// mode - a key frame must declare one and an explicit frame must not, since explicit labels are /// already complete. The declared fields and the empty batch are then decoded once, so a stream that /// could not be decoded is rejected before it is returned. /// -/// `expected` is a check, never an input. When it is given it must match the schema the frame -/// declares — the same logical fields in the same order, and no evaluation grid — and a mismatch is -/// an error. The decoder and the returned stream's schema are built from the frame either way, so a -/// caller cannot supply a field layout the frame does not declare. +/// A frame that declares evaluated content is refused here, before anything is read: a raw consumer +/// reads timestamps rather than grid positions, so the two contents are not interchangeable. +/// +/// `expected` is a check, never an input. When it is given it must name the same logical fields in +/// the same order and the same evaluation grid as the frame declares - which for a raw frame is that +/// both sides carry none - and a mismatch is an error. The decoder and the returned stream's schema +/// are built from the frame either way, so a caller cannot supply a field layout the frame does not +/// declare. pub fn unpack_raw_series( source: DfSendableRecordBatchStream, expected: Option>, ) -> datafusion_common::Result { let frame_schema = source.schema(); - let declared = raw_series_frame_label_decode(frame_schema.as_ref()).map_err(codec_error)?; - let mode = declared_label_mode(&declared); - let schema = - Arc::new(raw_series_frame_series_schema(frame_schema.as_ref()).map_err(codec_error)?); + let declaration = SeriesFrameDeclaration::parse(frame_schema.as_ref()).map_err(codec_error)?; + if declaration.content() != FrameContent::Raw { + return Err(invalid("expected a raw frame, found an evaluated one")); + } + let labels = declaration.labels(); + let declared = declaration.label_decode().cloned(); + let schema = Arc::new(declaration.series().clone()); if let Some(expected) = &expected { ensure_declared_schema(&schema, expected)?; } - let ordering = raw_series_frame_ordering(frame_schema.as_ref())?; - let decoder = RawSeriesFrameDecoder::try_new(&schema, mode).map_err(codec_error)?; + let ordering = series_frame_ordering(frame_schema.as_ref())?; + ensure_supported_ordering(ordering)?; + let decoder = + SeriesFrameDecoder::try_new(&schema, labels, declared.as_ref()).map_err(codec_error)?; // Validate even empty streams' declared schema and layout before returning the stream. let empty = DfRecordBatch::new_empty(frame_schema.clone()); - decoder.decode(&empty).map_err(codec_error)?; + decoder.decode_raw(&empty).map_err(codec_error)?; Ok(Box::pin(UnpackStream { source: Some(source), schema, @@ -270,20 +342,53 @@ pub fn unpack_raw_series( })) } +/// Unpacks DataFusion batches declared by [`pack_evaluated_series`]. +/// +/// The frame declares its grid, its field list, and its label mode, so the returned stream carries +/// the [`SeriesSchema`] the rows are addressed on and needs no planning-side copy of it. The grid is +/// the frame's own declaration: the timestamps stay unmaterialized until a consumer asks for one. +/// +/// A frame that declares raw samples, keyed identity, or an ordering other than `contiguous` is +/// refused before the stream is returned, as is a grid the declaration check cannot make sense of. +/// `expected` is a check and never an input, exactly as on the raw path: when given it must name the +/// same fields and the same grid the frame declares. +pub fn unpack_evaluated_series( + source: DfSendableRecordBatchStream, + expected: Option>, +) -> datafusion_common::Result { + let frame_schema = source.schema(); + let declaration = SeriesFrameDeclaration::parse(frame_schema.as_ref()).map_err(codec_error)?; + if declaration.content() != FrameContent::Evaluated { + return Err(invalid("expected an evaluated frame, found a raw one")); + } + ensure_contiguous_ordering(series_frame_ordering(frame_schema.as_ref())?)?; + let schema = Arc::new(declaration.series().clone()); + if let Some(expected) = &expected { + ensure_declared_schema(&schema, expected)?; + } + let decoder = SeriesFrameDecoder::try_new(&schema, FrameLabelMode::Explicit, None) + .map_err(codec_error)?; + let empty = DfRecordBatch::new_empty(frame_schema.clone()); + decoder.decode_evaluated(&empty).map_err(codec_error)?; + Ok(Box::pin(UnpackEvaluatedStream { + source: Some(source), + frame_schema, + validator: EvaluatedSequenceValidator::new(), + pending: Vec::new().into_iter(), + decoder, + })) +} + /// Rejects an `expected` series schema the frame does not declare. /// /// Only the two things a caller can get wrong are checked, because those are the only two things -/// [`SeriesSchema`] holds: the logical fields and their order, and an evaluation grid a raw frame -/// cannot carry. +/// [`SeriesSchema`] holds: the logical fields and their order, and the evaluation grid. A raw frame +/// carries no grid and an evaluated one carries exactly its own, so the grid comparison is also what +/// keeps a raw expectation from decoding an evaluated frame. fn ensure_declared_schema( declared: &SeriesSchema, expected: &SeriesSchema, ) -> datafusion_common::Result<()> { - if expected.eval_grid.is_some() { - return Err(invalid( - "expected raw series schema declares an evaluation grid the frame cannot carry", - )); - } if expected.fields.len() != declared.fields.len() || expected .fields @@ -292,10 +397,16 @@ fn ensure_declared_schema( .any(|(expected, declared)| expected.name != declared.name) { return Err(invalid( - "raw frame fields do not match the expected series schema", + "frame fields do not match the expected series schema", )); } - Ok(()) + match (&expected.eval_grid, &declared.eval_grid) { + (None, None) => Ok(()), + (Some(expected), Some(declared)) if expected.spec() == declared.spec() => Ok(()), + _ => Err(invalid( + "frame evaluation grid does not match the expected series schema", + )), + } } /// Checks that a frame declares the label decode layout a scan-bound binding describes. @@ -313,15 +424,14 @@ fn ensure_declared_schema( /// [`unpack_raw_series`] decodes it without a binding, and a caller that received the frame from a /// peer compares the declaration here instead of decoding under whichever side it trusts. /// -/// The comparison is plain equality. It used to canonicalize logical tag order on both sides, which -/// gave local comparisons one rule and the byte-exact remote comparison another; the writer now -/// canonicalizes instead (see [`LabelDecodeSpec::canonicalize`]), so a description that reaches a -/// reader already has one spelling and equality says everything there is to say about it. +/// The comparison is plain equality. The writer canonicalizes the description (see +/// [`LabelDecodeSpec::canonicalize`]), so a description that reaches a reader already has one +/// spelling and equality says everything there is to say about it. pub fn ensure_frame_matches_label_source( frame: &ArrowSchema, label_source: &RawLabelSource, ) -> datafusion_common::Result<()> { - let declared = raw_series_frame_label_decode(frame).map_err(codec_error)?; + let declared = series_frame_label_decode(frame).map_err(codec_error)?; if declared != label_decode_spec(label_source) { return Err(invalid( "raw frame label decode description does not match the supplied binding", @@ -330,19 +440,19 @@ pub fn ensure_frame_matches_label_source( Ok(()) } -fn label_mode(source: &RawLabelSource) -> RawLabelMode { +fn label_mode(source: &RawLabelSource) -> FrameLabelMode { match source { - RawLabelSource::PrimaryKey { .. } => RawLabelMode::Key, - RawLabelSource::Explicit => RawLabelMode::Explicit, + RawLabelSource::PrimaryKey { .. } => FrameLabelMode::Key, + RawLabelSource::Explicit => FrameLabelMode::Explicit, } } /// Returns the label mode a frame declares: a key frame describes its key, an explicit frame /// carries complete labels and declares no description. -fn declared_label_mode(declared: &Option) -> RawLabelMode { +fn declared_label_mode(declared: &Option) -> FrameLabelMode { match declared { - Some(_) => RawLabelMode::Key, - None => RawLabelMode::Explicit, + Some(_) => FrameLabelMode::Key, + None => FrameLabelMode::Explicit, } } @@ -356,6 +466,21 @@ fn ensure_supported_ordering(ordering: SeriesOrdering) -> datafusion_common::Res } } +/// Rejects an evaluated frame that claims more order than its producer promised. +/// +/// Computed values have no storage key to be ordered by, so `read_key_then_time` is not a claim this +/// layer can honour; `Interleaved` is not consumable at all. Treating either as `contiguous` would +/// silently weaken a guarantee a consumer may rely on. +fn ensure_contiguous_ordering(ordering: SeriesOrdering) -> datafusion_common::Result<()> { + match ordering { + SeriesOrdering::Contiguous => Ok(()), + SeriesOrdering::KeyThenTime => Err(invalid( + "an evaluated frame cannot claim key-then-time ordering", + )), + SeriesOrdering::Interleaved => Err(invalid("interleaved series ordering is not supported")), + } +} + fn codec_error(error: datatypes::error::Error) -> DataFusionError { DataFusionError::External(Box::new(BoxedError::new(error))) } @@ -373,9 +498,10 @@ fn invalid(reason: impl Into) -> DataFusionError { ))) } -/// Cross-chunk validation state. The producer must keep each series' chunks contiguous; this -/// validator checks adjacent chunks only and never sees `Interleaved` streams. It retains only -/// the previous complete owned key (not an Arrow buffer), timestamp, metric, and finality. +/// Cross-chunk validation state for raw chains. The producer must keep each series' chunks +/// contiguous; this validator checks adjacent chunks only and never sees `Interleaved` streams. It +/// retains only the previous complete owned key (not an Arrow buffer), timestamp, metric, and +/// finality. struct SequenceValidator { ordering: SeriesOrdering, previous_key: Option>, @@ -447,30 +573,95 @@ impl SequenceValidator { } } -struct PackStream { - source: Option, - schema: Arc, - mode: RawLabelMode, - frame_schema: datatypes::arrow::datatypes::SchemaRef, - ordering: SeriesOrdering, - validator: SequenceValidator, - builder: Option, - cursor: Option, +/// Cross-chunk validation state for evaluated chains. +/// +/// `is_last` closes one chunk chain and nothing more. It is not the end of a label set: a range +/// function or a label rewrite can make two input series show the same complete labels, and each of +/// them closes itself, so a closed chain may be followed by another chain whose labels happen to be +/// equal. Only the still-open chain is remembered, which is what keeps such a contribution from being +/// refused as a re-opened series while still refusing a chain that changes identity before it closes. +struct EvaluatedSequenceValidator { + /// The labels of the open chain and the last grid position it carried. + open: Option<(SeriesLabels, usize)>, } -impl PackStream { - /// Releases the source and drops every buffered owner, leaving the stream terminal; batches - /// already yielded are unaffected. - fn finish(&mut self) { - self.source = None; - self.cursor = None; - self.builder = None; - self.validator = SequenceValidator::new(self.ordering); +impl EvaluatedSequenceValidator { + fn new() -> Self { + Self { open: None } + } + + fn finish(&self) -> datafusion_common::Result<()> { + if self.open.is_some() { + return Err(invalid( + "evaluated series stream ended before a chain closed", + )); + } + Ok(()) + } + + fn validate(&mut self, series: &Series, is_last: bool) -> datafusion_common::Result<()> { + let (Some(first), Some(last)) = (series.time.first_position(), series.time.last_position()) + else { + // No carried position: only a closing chunk may be empty, and it closes the chain it + // belongs to without advancing it. + if !is_last { + return Err(invalid( + "an evaluated chunk without positions must close its series", + )); + } + match &self.open { + Some((labels, _)) if labels == &series.metric => self.open = None, + Some(_) => { + return Err(invalid( + "evaluated series identity changed before its chain closed", + )); + } + None => {} + } + return Ok(()); + }; + if let Some((labels, previous)) = &self.open { + if labels != &series.metric { + return Err(invalid( + "evaluated series identity changed before its chain closed", + )); + } + if first <= *previous { + return Err(invalid( + "evaluated series positions are not increasing across chunks", + )); + } + } + self.open = (!is_last).then(|| (series.metric.clone(), last)); + Ok(()) + } +} + +/// The batching half of a pack stream: the builder, the cursor installed in it, and the declared +/// schema every ready batch is published under. +struct PackBatches { + frame_schema: ArrowSchemaRef, + builder: Option, + cursor: Option, +} + +impl PackBatches { + fn new(builder: SeriesFrameBuilder, frame_schema: ArrowSchemaRef) -> Self { + Self { + frame_schema, + builder: Some(builder), + cursor: None, + } + } + + /// Installs the cursor of the chunk just read, replacing whatever was left of the previous one. + fn install(&mut self, cursor: SeriesFrameCursor) { + self.cursor = Some(cursor); } /// Advances the installed cursor until it yields one ready batch or is exhausted, dropping an /// exhausted cursor. `None` means no batch is ready, so the source may be polled again. - fn advance_cursor(&mut self) -> datatypes::error::Result> { + fn advance(&mut self) -> datatypes::error::Result> { let ready = match (self.cursor.as_mut(), self.builder.as_mut()) { (Some(cursor), Some(builder)) => cursor.append_to(builder)?, _ => return Ok(None), @@ -485,38 +676,63 @@ impl PackStream { } /// Flushes the rows buffered so far, if any, so a pending source can still deliver data. - fn flush_buffered(&mut self) -> datatypes::error::Result> { + fn flush(&mut self) -> datatypes::error::Result> { match self.builder.as_mut() { Some(builder) => builder.flush(), None => Ok(None), } } - /// Publishes one ready builder batch under the declared frame schema, which is the shared - /// builder's schema plus the producer's ordering metadata. A batch whose fields cannot be - /// declared releases the stream instead of yielding rows with an undeclared layout. - fn declare_batch( + /// Releases the buffered owners, leaving the stream terminal; batches already yielded are + /// unaffected. + fn release(&mut self) { + self.cursor = None; + self.builder = None; + } + + /// Publishes one ready batch under the declared frame schema, which is the builder's own layout + /// plus the producer's ordering metadata. A batch whose fields cannot be declared releases the + /// stream instead of yielding rows with an undeclared layout. + fn declare( &mut self, batch: DfRecordBatch, ) -> Poll>> { - match batch.with_schema(self.frame_schema.clone()) { + match DfRecordBatch::try_new(self.frame_schema.clone(), batch.columns().to_vec()) { Ok(batch) => Poll::Ready(Some(Ok(batch))), Err(error) => { - self.finish(); + self.release(); Poll::Ready(Some(Err(DataFusionError::from(error)))) } } } } +struct PackStream { + source: Option, + schema: Arc, + labels: FrameLabelMode, + batches: PackBatches, + ordering: SeriesOrdering, + validator: SequenceValidator, +} + +impl PackStream { + /// Releases the source and drops every buffered owner, leaving the stream terminal. + fn finish(&mut self) { + self.source = None; + self.batches.release(); + self.validator = SequenceValidator::new(self.ordering); + } +} + impl Stream for PackStream { type Item = datafusion_common::Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { let this = &mut *self; loop { - match this.advance_cursor() { - Ok(Some(batch)) => return this.declare_batch(batch), + match this.batches.advance() { + Ok(Some(batch)) => return this.batches.declare(batch), Ok(None) => {} Err(error) => { this.finish(); @@ -529,9 +745,9 @@ impl Stream for PackStream { }; match polled { Poll::Pending => { - let flushed = this.flush_buffered(); + let flushed = this.batches.flush(); return match flushed { - Ok(Some(batch)) => this.declare_batch(batch), + Ok(Some(batch)) => this.batches.declare(batch), Ok(None) => Poll::Pending, Err(error) => { this.finish(); @@ -544,10 +760,18 @@ impl Stream for PackStream { this.finish(); return Poll::Ready(Some(Err(error))); } - let flushed = this.flush_buffered(); + let flushed = this.batches.flush(); + // The declared schema is read before the release below, so the last batch is + // published under the same schema as every batch before it. + let declared = this.batches.frame_schema.clone(); this.finish(); return match flushed { - Ok(Some(batch)) => this.declare_batch(batch), + Ok(Some(batch)) => { + match DfRecordBatch::try_new(declared, batch.columns().to_vec()) { + Ok(batch) => Poll::Ready(Some(Ok(batch))), + Err(error) => Poll::Ready(Some(Err(DataFusionError::from(error)))), + } + } Ok(None) => Poll::Ready(None), Err(error) => Poll::Ready(Some(Err(codec_error(error)))), }; @@ -561,8 +785,8 @@ impl Stream for PackStream { this.finish(); return Poll::Ready(Some(Err(error))); } - match RawSeriesChunkCursor::try_new(&this.schema, this.mode, chunk) { - Ok(cursor) => this.cursor = Some(cursor), + match SeriesFrameCursor::try_new_raw(&this.schema, this.labels, chunk) { + Ok(cursor) => this.batches.install(cursor), Err(error) => { this.finish(); return Poll::Ready(Some(Err(codec_error(error)))); @@ -575,21 +799,117 @@ impl Stream for PackStream { } impl datafusion::physical_plan::RecordBatchStream for PackStream { - fn schema(&self) -> datatypes::arrow::datatypes::SchemaRef { - self.frame_schema.clone() + fn schema(&self) -> ArrowSchemaRef { + self.batches.frame_schema.clone() + } +} + +/// The pack stream of [`pack_evaluated_series`]. +struct PackEvaluatedStream { + source: Option, + schema: Arc, + batches: PackBatches, + validator: EvaluatedSequenceValidator, +} + +impl PackEvaluatedStream { + /// Releases the source and drops every buffered owner, leaving the stream terminal. + fn finish(&mut self) { + self.source = None; + self.batches.release(); + self.validator = EvaluatedSequenceValidator::new(); + } +} + +impl Stream for PackEvaluatedStream { + type Item = datafusion_common::Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = &mut *self; + loop { + match this.batches.advance() { + Ok(Some(batch)) => return this.batches.declare(batch), + Ok(None) => {} + Err(error) => { + this.finish(); + return Poll::Ready(Some(Err(codec_error(error)))); + } + } + let polled = match this.source.as_mut() { + Some(source) => source.as_mut().poll_next(cx), + None => return Poll::Ready(None), + }; + match polled { + Poll::Pending => { + let flushed = this.batches.flush(); + return match flushed { + Ok(Some(batch)) => this.batches.declare(batch), + Ok(None) => Poll::Pending, + Err(error) => { + this.finish(); + Poll::Ready(Some(Err(codec_error(error)))) + } + }; + } + Poll::Ready(None) => { + if let Err(error) = this.validator.finish() { + this.finish(); + return Poll::Ready(Some(Err(error))); + } + let flushed = this.batches.flush(); + // The declared schema is read before the release below, so the last batch is + // published under the same schema as every batch before it. + let declared = this.batches.frame_schema.clone(); + this.finish(); + return match flushed { + Ok(Some(batch)) => { + match DfRecordBatch::try_new(declared, batch.columns().to_vec()) { + Ok(batch) => Poll::Ready(Some(Ok(batch))), + Err(error) => Poll::Ready(Some(Err(DataFusionError::from(error)))), + } + } + Ok(None) => Poll::Ready(None), + Err(error) => Poll::Ready(Some(Err(codec_error(error)))), + }; + } + Poll::Ready(Some(Err(error))) => { + this.finish(); + return Poll::Ready(Some(Err(upstream_error(error)))); + } + Poll::Ready(Some(Ok((series, is_last)))) => { + if let Err(error) = this.validator.validate(&series, is_last) { + this.finish(); + return Poll::Ready(Some(Err(error))); + } + match SeriesFrameCursor::try_new_evaluated(&this.schema, series, is_last) { + Ok(cursor) => this.batches.install(cursor), + Err(error) => { + this.finish(); + return Poll::Ready(Some(Err(codec_error(error)))); + } + } + } + } + } + } +} + +impl datafusion::physical_plan::RecordBatchStream for PackEvaluatedStream { + fn schema(&self) -> ArrowSchemaRef { + self.batches.frame_schema.clone() } } struct UnpackStream { source: Option, schema: Arc, - frame_schema: datatypes::arrow::datatypes::SchemaRef, + frame_schema: ArrowSchemaRef, /// The description parsed from the frame schema; the only input decoding needs. label_decode_spec: Option, ordering: SeriesOrdering, validator: SequenceValidator, pending: std::vec::IntoIter, - decoder: RawSeriesFrameDecoder, + decoder: SeriesFrameDecoder, } impl RawSeriesStream for UnpackStream { @@ -628,7 +948,6 @@ impl Stream for UnpackStream { type Item = Result; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - const MAX_EMPTY_BATCHES_PER_POLL: usize = 32; let this = &mut *self; let mut empty_batches = 0; loop { @@ -644,12 +963,7 @@ impl Stream for UnpackStream { Poll::Ready(None) => { let result = this.validator.finish(); this.finish(); - return Poll::Ready(result.err().map(|error| { - Err(BoxedError::new(crate::error::Error::GeneralDataFusion { - error, - location: snafu::Location::default(), - })) - })); + return Poll::Ready(result.err().map(|error| Err(unpack_error(error)))); } Poll::Ready(Some(Err(error))) => { this.finish(); @@ -666,7 +980,7 @@ impl Stream for UnpackStream { return Err(invalid("raw frame schema mismatch")); } this.decoder - .decode(&batch) + .decode_raw(&batch) .map_err(codec_error) .and_then(|chunks| { for chunk in &chunks { @@ -689,12 +1003,7 @@ impl Stream for UnpackStream { } Err(error) => { this.finish(); - return Poll::Ready(Some(Err(BoxedError::new( - crate::error::Error::GeneralDataFusion { - error, - location: snafu::Location::default(), - }, - )))); + return Poll::Ready(Some(Err(unpack_error(error)))); } } } @@ -702,3 +1011,99 @@ impl Stream for UnpackStream { } } } + +/// The unpack stream of [`unpack_evaluated_series`]. +struct UnpackEvaluatedStream { + source: Option, + frame_schema: ArrowSchemaRef, + validator: EvaluatedSequenceValidator, + pending: std::vec::IntoIter<(Series, bool)>, + decoder: SeriesFrameDecoder, +} + +impl UnpackEvaluatedStream { + fn finish(&mut self) { + self.source = None; + self.pending = Vec::new().into_iter(); + self.validator = EvaluatedSequenceValidator::new(); + } +} + +impl Stream for UnpackEvaluatedStream { + type Item = Result<(Series, bool), BoxedError>; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = &mut *self; + let mut empty_batches = 0; + loop { + if let Some(chunk) = this.pending.next() { + return Poll::Ready(Some(Ok(chunk))); + } + let polled = match this.source.as_mut() { + Some(source) => source.as_mut().poll_next(cx), + None => return Poll::Ready(None), + }; + match polled { + Poll::Pending => return Poll::Pending, + Poll::Ready(None) => { + let result = this.validator.finish(); + this.finish(); + return Poll::Ready(result.err().map(|error| Err(unpack_error(error)))); + } + Poll::Ready(Some(Err(error))) => { + this.finish(); + return Poll::Ready(Some(Err(BoxedError::new( + crate::error::Error::GeneralDataFusion { + error, + location: snafu::Location::default(), + }, + )))); + } + Poll::Ready(Some(Ok(batch))) => { + let result = (|| { + if batch.schema() != this.frame_schema { + return Err(invalid("evaluated frame schema mismatch")); + } + this.decoder + .decode_evaluated(&batch) + .map_err(codec_error) + .and_then(|chunks| { + for (series, is_last) in &chunks { + this.validator.validate(series, *is_last)?; + } + Ok(chunks) + }) + })(); + match result { + Ok(chunks) => { + if chunks.is_empty() { + empty_batches += 1; + if empty_batches == MAX_EMPTY_BATCHES_PER_POLL { + cx.waker().wake_by_ref(); + return Poll::Pending; + } + } else { + this.pending = chunks.into_iter(); + } + } + Err(error) => { + this.finish(); + return Poll::Ready(Some(Err(unpack_error(error)))); + } + } + } + } + } + } +} + +/// How many empty batches one poll reads before yielding, so a stream of them cannot hold the task. +const MAX_EMPTY_BATCHES_PER_POLL: usize = 32; + +/// Wraps a decode or validation error as the boxed error an unpack stream reports. +fn unpack_error(error: DataFusionError) -> BoxedError { + BoxedError::new(crate::error::Error::GeneralDataFusion { + error, + location: snafu::Location::default(), + }) +} diff --git a/src/common/query/src/series_frame/tests.rs b/src/common/query/src/series_frame/tests.rs index 61509c4f12c..1ed826e3ce4 100644 --- a/src/common/query/src/series_frame/tests.rs +++ b/src/common/query/src/series_frame/tests.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod evaluated; mod lifecycle; use std::collections::{HashMap, VecDeque}; @@ -34,7 +35,9 @@ use datatypes::arrow::record_batch::RecordBatch; use datatypes::series::Label; use datatypes::series::SeriesLabels; use datatypes::series::SeriesOrdering; -use datatypes::series::frame::{LabelDecodeSpec, RawLabelMode, raw_series_frame_schema}; +use datatypes::series::frame::{ + FrameLabelMode, LabelDecodeSpec, raw_series_frame_schema, series_frame_schema, +}; use datatypes::series::histogram::{BucketSideView, CounterResetHint, HistogramBatchBuilder}; use datatypes::series::raw::{RawSeriesChunk, RawTimestamps}; use datatypes::series::samples::{ @@ -46,8 +49,8 @@ use store_api::region_engine::{ }; use super::{ - ensure_frame_matches_label_source, label_decode_spec, pack_raw_series, - raw_series_frame_ordering, unpack_raw_series, + ensure_frame_matches_label_source, label_decode_spec, pack_raw_series, series_frame_ordering, + unpack_raw_series, }; fn schema() -> Arc { @@ -59,6 +62,28 @@ fn schema() -> Arc { eval_grid: None, }) } + +/// The one grid every evaluated fixture is addressed on: 512 positions, 1000 ms apart. +fn evaluated_grid() -> Arc { + Arc::new( + datatypes::series::time::EvalGrid::try_new( + datatypes::series::time::GridSpec::try_new(0, 1000, 512).unwrap(), + ) + .unwrap(), + ) +} + +/// A schema with one `value` field addressed on [`evaluated_grid`], which is what makes it an +/// evaluated declaration. +fn evaluated_schema() -> Arc { + Arc::new(SeriesSchema { + fields: vec![FieldSpec { + name: Arc::from("value"), + }] + .into(), + eval_grid: Some(evaluated_grid()), + }) +} fn metric() -> SeriesLabels { SeriesLabels::try_new(Some(Arc::from("cpu")), vec![]).unwrap() } @@ -629,7 +654,8 @@ async fn invalid_sequence_reports_valid_prefix_then_one_terminal_error() { assert_eq!(batch.num_rows(), case.prefix.len(), "{}", case.name); let chunks = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &batch, ) .unwrap(); @@ -656,7 +682,8 @@ async fn invalid_sequence_reports_valid_prefix_then_one_terminal_error() { frames.extend( datatypes::series::frame::pack_raw_series_chunk( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, item, ) .unwrap(), @@ -672,7 +699,9 @@ async fn invalid_sequence_reports_valid_prefix_then_one_terminal_error() { .unwrap(); let frames = frames .into_iter() - .map(|frame| frame.with_schema(frame_schema.clone()).unwrap()) + .map(|frame| { + RecordBatch::try_new(frame_schema.clone(), frame.columns().to_vec()).unwrap() + }) .collect(); let (source, _, _) = df_source(frames, frame_schema); let mut decoded = unpack_raw_series(source, Some(schema())).unwrap(); @@ -692,7 +721,8 @@ async fn invalid_sequence_reports_valid_prefix_then_one_terminal_error() { // missing final key is reported once, at EOF. let frame = datatypes::series::frame::pack_raw_series_chunk( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &chunk(b"unfinished", &[7], false), ) .unwrap(); @@ -923,7 +953,8 @@ async fn explicit_metric_must_remain_stable_within_key() { ); let chunks = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &batch, ) .unwrap(); @@ -975,7 +1006,7 @@ async fn empty_streams_keep_self_describing_schema() { assert_eq!( frame_schema.field(0).name(), "read_key", - "the v1 wire column name is unchanged" + "the wire column name is unchanged" ); assert!(packed.next().await.is_none()); // A declared-but-empty stream is still validated against the expected schema before any @@ -1018,7 +1049,7 @@ async fn upstream_error_status_is_preserved_once_and_releases_source() { assert_eq!(drops.load(Ordering::SeqCst), 1); let batch_schema = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let plain = PlainError::new("upstream df".into(), status); @@ -1062,7 +1093,8 @@ async fn dropping_adapter_releases_upstream_and_pending_item_is_resumed() { ); let chunks = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &first, ) .unwrap(); @@ -1076,7 +1108,8 @@ async fn dropping_adapter_releases_upstream_and_pending_item_is_resumed() { ); let chunks = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &second, ) .unwrap(); @@ -1094,18 +1127,25 @@ async fn unpack_rejects_mismatched_batch_schema_before_polling() { assert_eq!(drops.load(Ordering::SeqCst), 1); } +/// A schema with an evaluation grid is a legal *evaluated* frame declaration, so it is no longer a +/// reason to refuse a stream at construction; it is still no schema to carry *raw* chunks under, so +/// the first raw chunk is rejected instead of the stream. #[tokio::test] -async fn invalid_grid_schema_rejected_for_empty_raw_stream() { - let mut invalid = (*schema()).clone(); - invalid.eval_grid = Some(Arc::new( +async fn grid_schema_is_legal_but_cannot_carry_a_raw_chunk() { + let mut grid_schema = (*schema()).clone(); + grid_schema.eval_grid = Some(Arc::new( datatypes::series::time::EvalGrid::try_new( datatypes::series::time::GridSpec::try_new(0, 10, 1).unwrap(), ) .unwrap(), )); + let grid_schema = Arc::new(grid_schema); + + // An empty stream under a grid schema constructs: the declaration is legal and no row disagrees + // with it. let polls = Arc::new(AtomicUsize::new(0)); let source = Box::pin(RawMock { - schema: Arc::new(invalid), + schema: grid_schema.clone(), items: VecDeque::new(), binding: RawLabelSource::Explicit, label_decode_spec: None, @@ -1117,8 +1157,33 @@ async fn invalid_grid_schema_rejected_for_empty_raw_stream() { panic_on_poll: false, emitted: 0, }); - assert!(pack_raw_series(source).is_err()); - assert_eq!(polls.load(Ordering::SeqCst), 0); + let mut packed = pack_raw_series(source).unwrap(); + assert!(packed.next().await.is_none()); + assert_eq!(polls.load(Ordering::SeqCst), 1); + + // A raw chunk cannot be addressed on a grid, so the chunk that arrives under this declaration is + // what fails. + let polls = Arc::new(AtomicUsize::new(0)); + let source = Box::pin(RawMock { + schema: grid_schema, + items: VecDeque::from([Ok(chunk(b"a", &[1], true))]), + binding: RawLabelSource::Explicit, + label_decode_spec: None, + ordering: SeriesOrdering::Contiguous, + polls: polls.clone(), + drops: Arc::new(AtomicUsize::new(0)), + pending_after: None, + pending_reported: false, + panic_on_poll: false, + emitted: 0, + }); + let mut packed = pack_raw_series(source).unwrap(); + let error = packed.next().await.unwrap().unwrap_err(); + assert_eq!( + crate::error::datafusion_status_code::(&error, None), + StatusCode::InvalidArguments + ); + assert!(packed.next().await.is_none()); } #[tokio::test] @@ -1150,19 +1215,21 @@ async fn unpack_batch_sequence_error_is_atomic_for_all_rows() { async fn unpack_rejects_late_invalid_frame_row_before_prefix() { let frame_schema = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let first = datatypes::series::frame::pack_raw_series_chunk( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &chunk(b"a", &[1], true), ) .unwrap() .remove(0); let mut second = datatypes::series::frame::pack_raw_series_chunk( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &chunk(b"b", &[2], true), ) .unwrap() @@ -1211,7 +1278,7 @@ async fn unpack_rejects_late_invalid_frame_row_before_prefix() { async fn frame_application_metadata_is_stable_and_schema_change_rejected() { let base_schema = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let mut application_metadata = base_schema.metadata().clone(); @@ -1223,7 +1290,8 @@ async fn frame_application_metadata_is_stable_and_schema_change_rejected() { ); let first = datatypes::series::frame::pack_raw_series_chunk( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &chunk(b"a", &[1], true), ) .unwrap() @@ -1232,7 +1300,8 @@ async fn frame_application_metadata_is_stable_and_schema_change_rejected() { .unwrap(); let stable_second = datatypes::series::frame::pack_raw_series_chunk( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &chunk(b"b", &[2], true), ) .unwrap() @@ -1267,7 +1336,8 @@ async fn frame_application_metadata_is_stable_and_schema_change_rejected() { let changed_schema = Arc::new((*base_schema).clone().with_metadata(changed)); let bad_batch = datatypes::series::frame::pack_raw_series_chunk( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &chunk(b"b", &[2], true), ) .unwrap() @@ -1380,7 +1450,7 @@ fn unpack_yields_at_empty_batch_budget_before_observing_eof() { const BUDGET: usize = 32; let frame_schema = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let batches = vec![RecordBatch::new_empty(frame_schema.clone()); BUDGET]; @@ -1406,7 +1476,7 @@ fn unpack_yields_at_empty_batch_budget_before_observing_eof() { fn unpack_propagates_upstream_pending_without_extra_wake() { let frame_schema = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let (source, polls, _) = df_pending( @@ -1431,7 +1501,7 @@ fn unpack_reports_bad_batch_at_budget_boundary_before_yielding() { const BUDGET: usize = 32; let frame_schema = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let mut batches = vec![RecordBatch::new_empty(frame_schema.clone()); BUDGET - 1]; @@ -1460,7 +1530,7 @@ fn unpack_surfaces_upstream_error_after_budget_yield_once() { const BUDGET: usize = 32; let frame_schema = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let error = datafusion_common::DataFusionError::External(Box::new(BoxedError::new( @@ -1537,6 +1607,10 @@ async fn unpack_drains_decoded_rows_before_polling_another_batch() { assert!(unpacked.next().await.is_none()); } +/// An `expected` schema the raw frame does not declare is rejected before any poll, whether it names +/// no field at all or carries an evaluation grid. A grid schema is a legal *evaluated* declaration +/// now, so it is refused here because a raw frame carries no grid, not because the schema itself is +/// impossible. #[tokio::test] async fn unpack_preflights_empty_and_grid_schemas_without_polling() { let invalid_schemas = [ @@ -1557,7 +1631,7 @@ async fn unpack_preflights_empty_and_grid_schemas_without_polling() { for expected in invalid_schemas { let frame_schema = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let (source, polls, drops) = df_source(vec![], frame_schema); @@ -1749,7 +1823,8 @@ async fn pack_bounds_ready_rows_at_default_row_limit_before_eof() { for batch in [&first, &second] { let decoded = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, batch, ) .unwrap(); @@ -1824,7 +1899,8 @@ async fn pack_splits_one_chunk_into_bounded_batches_without_repolling_the_source for batch in [&first, &second] { let decoded = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, batch, ) .unwrap(); @@ -1877,7 +1953,8 @@ async fn dropping_pack_mid_chunk_keeps_last_batch_and_releases_source() { drop(owners); let decoded = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &batch, ) .unwrap(); @@ -1926,7 +2003,8 @@ async fn pack_flushes_ready_rows_at_pending_then_remainder_at_eof() { assert_eq!(wake.0.load(Ordering::SeqCst), 1, "upstream wakes the task"); let decoded = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &first, ) .unwrap(); @@ -1943,7 +2021,8 @@ async fn pack_flushes_ready_rows_at_pending_then_remainder_at_eof() { assert_eq!(drops.load(Ordering::SeqCst), 1); let decoded = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &second, ) .unwrap(); @@ -1997,7 +2076,8 @@ async fn pack_emits_oversized_legal_rows_whole() { ] { let chunks = datatypes::series::frame::unpack_raw_series_frame( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, batch, ) .unwrap(); @@ -2095,7 +2175,7 @@ async fn multirow_ipc_roundtrip_preserves_cross_chunk_variable_segments() { assert_eq!( reader.schema(), batches[0].schema(), - "IPC preserves the v1 layout, metadata, and field order" + "IPC preserves the layout, metadata, and field order" ); let frame_schema = reader.schema(); let ipc_batches = reader.map(Result::unwrap).collect::>(); @@ -2285,8 +2365,8 @@ async fn public_chunk_with_empty_times_is_rejected_without_panic() { } /// `Interleaved` is rejected by the constructor before any source poll, for a pack source and for -/// an empty unpack stream; the wire-schema helper rejects it before assembling a schema, since no -/// v1 metadata value represents it. No adjacent-only fallback is installed. +/// an empty unpack stream; the wire-schema helper rejects it before assembling a schema, since the +/// wire carries only the two representable spellings. No adjacent-only fallback is installed. #[tokio::test] async fn interleaved_ordering_is_rejected_before_any_poll() { assert!( @@ -2319,7 +2399,7 @@ async fn interleaved_ordering_is_rejected_before_any_poll() { // a frame that declares one; that spelling has no wire representation and is rejected. let frame = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let mut metadata = frame.metadata().clone(); @@ -2343,17 +2423,20 @@ async fn interleaved_ordering_is_rejected_before_any_poll() { /// The unpack stream rejects a batch whose schema differs from the stream's declared schema, /// including drift in metadata the codec itself does not interpret and an empty batch that -/// carries the drift; the reserved layout metadata is rejected as well. +/// carries the drift; the reserved layout metadata is rejected as well. Since the current layout is +/// `v2`, the drift case rewrites the layout to `v1`: any value other than this revision's is refused, +/// so a stale or a future spelling cannot be decoded under the current columns. #[tokio::test] async fn unpack_rejects_unrelated_metadata_drift_and_empty_schema_changes() { let base = datatypes::series::frame::raw_series_frame_schema( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, ) .unwrap(); let first = datatypes::series::frame::pack_raw_series_chunk( &schema(), - datatypes::series::frame::RawLabelMode::Explicit, + datatypes::series::frame::FrameLabelMode::Explicit, + None, &chunk(b"a", &[1], true), ) .unwrap() @@ -2361,8 +2444,9 @@ async fn unpack_rejects_unrelated_metadata_drift_and_empty_schema_changes() { let mut unrelated = base.metadata().clone(); unrelated.insert("app.unrelated".into(), "x".into()); + // The current layout is `v2`, so a frame that still declares `v1` is the drift here. let mut reserved = base.metadata().clone(); - reserved.insert("greptime.series.layout".into(), "v2".into()); + reserved.insert("greptime.series.layout".into(), "v1".into()); let mut renamed = base.metadata().clone(); renamed.insert("app.renamed".into(), "y".into()); @@ -2408,6 +2492,9 @@ async fn unpack_rejects_unrelated_metadata_drift_and_empty_schema_changes() { } } +/// The ordering and layout fixture: literal metadata maps are read the way a receiver reads them, +/// so the same text pins both what `series_frame_ordering` answers and whether the declaration as a +/// whole is accepted. #[test] fn raw_series_ordering_metadata_fixture() { #[derive(serde::Deserialize)] @@ -2415,10 +2502,12 @@ fn raw_series_ordering_metadata_fixture() { name: String, metadata: HashMap, ordering: Option, + /// `accepted` or `rejected`: whether the whole declaration is one this revision reads. + declaration: String, } let cases: Vec = serde_json::from_str(include_str!( - "../../../../../tests/compatibility/wire/raw_series_ordering/v1_metadata.json" + "../../../../../tests/compatibility/wire/raw_series_ordering/v2_metadata.json" )) .unwrap(); for case in cases { @@ -2428,22 +2517,52 @@ fn raw_series_ordering_metadata_fixture() { Some(value) => panic!("{} has invalid fixture ordering {value}", case.name), None => None, }; - let frame_schema = raw_series_frame_schema(&schema(), RawLabelMode::Explicit).unwrap(); + // The columns have to match the content the case declares, so an evaluated case is built on + // an evaluated layout rather than read through a raw one. + let evaluated = case + .metadata + .get("greptime.series.content") + .is_some_and(|content| content == "evaluated"); + let frame_schema = if evaluated { + series_frame_schema(&evaluated_schema(), FrameLabelMode::Explicit, None).unwrap() + } else { + raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap() + }; let frame_schema = frame_schema.as_ref().clone().with_metadata(case.metadata); match literal_ordering { Some(expected) => assert_eq!( - raw_series_frame_ordering(&frame_schema) + series_frame_ordering(&frame_schema) .unwrap_or_else(|error| panic!("{}: {error}", case.name)), expected, "{}", case.name ), None => assert!( - raw_series_frame_ordering(&frame_schema).is_err(), + series_frame_ordering(&frame_schema).is_err(), "{}", case.name ), } + let declared = datatypes::series::frame::SeriesFrameDeclaration::parse(&frame_schema); + match case.declaration.as_str() { + "accepted" => { + let declared = declared.unwrap_or_else(|error| panic!("{}: {error}", case.name)); + if evaluated { + assert_eq!( + declared.content(), + datatypes::series::frame::FrameContent::Evaluated, + "{}", + case.name + ); + } + } + "rejected" => assert!( + declared.is_err(), + "{}: the declaration must be refused", + case.name + ), + other => panic!("{} has invalid fixture declaration {other}", case.name), + } } } @@ -2463,14 +2582,14 @@ async fn pack_declares_ordering_on_every_batch_and_ipc_roundtrips_it() { // replaced single-row packing emitted one batch per chunk. let expected_batch_count = if is_empty { 0 } else { 1 }; let canonical_schema = - raw_series_frame_schema(&schema(), RawLabelMode::Explicit).unwrap(); + raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap(); let mut expected_metadata = canonical_schema.metadata().clone(); expected_metadata.insert( "greptime.series.ordering".to_string(), match ordering { SeriesOrdering::Contiguous => "contiguous", SeriesOrdering::KeyThenTime => "read_key_then_time", - // The v1 wire carries exactly these two spellings, so the loop above + // The wire carries exactly these two spellings, so the loop above // admits only the two representable modes; `Interleaved` is rejected // before any poll (see `interleaved_ordering_is_rejected_before_any_poll`). SeriesOrdering::Interleaved => { @@ -2484,7 +2603,7 @@ async fn pack_declares_ordering_on_every_batch_and_ipc_roundtrips_it() { let declared = packed.schema(); assert_eq!(declared.fields(), canonical_schema.fields()); assert_eq!(declared.metadata(), &expected_metadata); - assert_eq!(raw_series_frame_ordering(&declared).unwrap(), ordering); + assert_eq!(series_frame_ordering(&declared).unwrap(), ordering); let mut batches = Vec::new(); while let Some(batch) = packed.next().await { let batch = batch.unwrap(); @@ -2502,10 +2621,7 @@ async fn pack_declares_ordering_on_every_batch_and_ipc_roundtrips_it() { } let reader = StreamReader::try_new(Cursor::new(ipc), None).unwrap(); assert_eq!(reader.schema(), declared); - assert_eq!( - raw_series_frame_ordering(&reader.schema()).unwrap(), - ordering - ); + assert_eq!(series_frame_ordering(&reader.schema()).unwrap(), ordering); let ipc_batches = reader.map(Result::unwrap).collect::>(); assert_eq!(ipc_batches.len(), batches.len()); assert_eq!(ipc_batches.len(), expected_batch_count); @@ -2621,17 +2737,33 @@ fn two_tag_binding(metric: &str) -> RawLabelSource { } } -/// A key frame whose metadata carries exactly the supplied description and nothing else. -fn key_frame_with(description: &datatypes::series::frame::LabelDecodeSpec) -> SchemaRef { - let layout = raw_series_frame_schema(&schema(), RawLabelMode::Key).unwrap(); +/// A raw key frame declaring the supplied description, as a producer with a primary-key binding +/// writes it. +fn key_layout() -> SchemaRef { + let description = + label_decode_spec(&key_binding("cpu")).expect("a primary-key binding describes its key"); + series_frame_schema(&schema(), FrameLabelMode::Key, Some(&description)).unwrap() +} + +/// A key frame whose metadata carries exactly the supplied description text and nothing else. +/// +/// The text is written verbatim, not canonicalized, so a case can pin what a writer must have +/// ordered and what a reader refuses to reorder. +fn key_frame_with_encoded(encoded: &str) -> SchemaRef { + let layout = key_layout(); let mut metadata = layout.metadata().clone(); metadata.insert( "greptime.series.label_decode".to_string(), - serde_json::to_string(description).unwrap(), + encoded.to_string(), ); Arc::new(Schema::new_with_metadata(layout.fields().clone(), metadata)) } +/// A key frame whose `greptime.series.label_decode` entry is the supplied description. +fn key_frame_with(description: &datatypes::series::frame::LabelDecodeSpec) -> SchemaRef { + key_frame_with_encoded(&serde_json::to_string(description).unwrap()) +} + /// Logical tag order is fixed by the writer, so the wire form has one spelling and the comparison /// is plain equality. The ordered key columns are the encoding, so the same columns in another /// order are a different layout and are rejected. @@ -2720,7 +2852,7 @@ async fn frame_check_rejects_a_binding_the_frame_does_not_describe() { "an explicit identity expects a frame that carries complete labels" ); - let explicit = raw_series_frame_schema(&schema(), RawLabelMode::Explicit).unwrap(); + let explicit = raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap(); assert!( ensure_frame_matches_label_source(explicit.as_ref(), &key_binding("cpu")).is_err(), "a frame declaring no layout cannot satisfy a primary-key identity" @@ -2746,7 +2878,7 @@ async fn unpack_retains_the_declared_description_without_a_binding() { "no physical region key layout is invented for a frame that carries none" ); - let explicit = raw_series_frame_schema(&schema(), RawLabelMode::Explicit).unwrap(); + let explicit = raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap(); let (source, _, _) = df_source(vec![], explicit); let unpacked = unpack_raw_series(source, None).unwrap(); assert_eq!( @@ -2756,17 +2888,6 @@ async fn unpack_retains_the_declared_description_without_a_binding() { ); } -/// A key frame whose `greptime.series.label_decode` entry is exactly the supplied text. -fn key_frame_with_encoded(encoded: &str) -> SchemaRef { - let layout = raw_series_frame_schema(&schema(), RawLabelMode::Key).unwrap(); - let mut metadata = layout.metadata().clone(); - metadata.insert( - "greptime.series.label_decode".to_string(), - encoded.to_string(), - ); - Arc::new(Schema::new_with_metadata(layout.fields().clone(), metadata)) -} - /// The description is validated whether or not a binding is supplied, so trusting the frame never /// means skipping validation: an unknown version, an unknown field, a malformed entry, a logical /// tag outside the key, a key frame without a description, and an explicit frame carrying one are @@ -2810,14 +2931,21 @@ async fn unpack_rejects_invalid_descriptions_without_a_binding() { ); } - let key_without_description = raw_series_frame_schema(&schema(), RawLabelMode::Key).unwrap(); + // A key frame nobody can decode: the description is removed from the frame the producer wrote. + // Construction cannot express this, which is the point - the reader is what refuses it. + let mut without_description = key_layout().metadata().clone(); + without_description.remove("greptime.series.label_decode"); + let key_without_description = Arc::new(Schema::new_with_metadata( + key_layout().fields().clone(), + without_description, + )); let (source, _, _) = df_source(vec![], key_without_description); assert!( unpack_raw_series(source, None).is_err(), "a key frame nobody can decode is rejected instead of being trusted" ); - let explicit = raw_series_frame_schema(&schema(), RawLabelMode::Explicit).unwrap(); + let explicit = raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap(); let mut metadata = explicit.metadata().clone(); metadata.insert( "greptime.series.label_decode".to_string(), @@ -2850,14 +2978,15 @@ async fn pack_declares_the_label_decode_description_of_its_binding() { ); let mut packed = pack_raw_series(input).unwrap(); let declared = packed.schema(); - let canonical = raw_series_frame_schema(&schema(), RawLabelMode::Key).unwrap(); + // A key frame's canonical layout carries its description and explicit default ordering. + let canonical = key_layout(); assert_eq!( declared.metadata().len(), - canonical.metadata().len() + 2, - "a key frame adds the producer ordering and the label decode description" + canonical.metadata().len(), + "the stream schema preserves the complete canonical declaration" ); assert_eq!( - datatypes::series::frame::raw_series_frame_label_decode(&declared).unwrap(), + datatypes::series::frame::series_frame_label_decode(&declared).unwrap(), label_decode_spec(&binding), "the declared description is derived from the binding, not from the first row" ); @@ -2872,11 +3001,16 @@ async fn pack_declares_the_label_decode_description_of_its_binding() { let packed = pack_raw_series(input).unwrap(); let declared = packed.schema(); assert_eq!( - datatypes::series::frame::raw_series_frame_label_decode(&declared).unwrap(), + datatypes::series::frame::series_frame_label_decode(&declared).unwrap(), None, "explicit labels are already complete" ); - assert_eq!(declared.metadata().len(), canonical.metadata().len() + 1); + let canonical_explicit = raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap(); + assert_eq!( + declared.metadata().len(), + canonical_explicit.metadata().len(), + "an explicit frame preserves the complete canonical declaration" + ); } /// A key frame that omits its description is not decodable, and unpacking says so instead of falling @@ -2899,13 +3033,11 @@ async fn unpack_round_trips_a_key_frame_and_rejects_one_without_a_description() let chunk = decoded.next().await.unwrap().unwrap(); assert_eq!(chunk.primary_key.as_slice(), b"encoded-complete-key"); - // A frame that omits the description is not decodable from the schema alone. - let canonical = raw_series_frame_schema(&schema(), RawLabelMode::Key).unwrap(); - let mut metadata = canonical.metadata().clone(); - metadata.insert( - "greptime.series.ordering".to_string(), - "contiguous".to_string(), - ); + // A frame that omits the description is not decodable from the schema alone. The producer + // cannot write one, so the entry is dropped from the frame it did write: the reader is what has + // to refuse it. + let mut metadata = declared.metadata().clone(); + metadata.remove("greptime.series.label_decode"); let stripped = Arc::new(datatypes::arrow::datatypes::Schema::new_with_metadata( declared.fields().clone(), metadata, @@ -2922,8 +3054,18 @@ async fn unpack_round_trips_a_key_frame_and_rejects_one_without_a_description() ); } -/// The ordering a frame declares is the only ordering a consumer has: unpacking a `KeyThenTime` -/// frame reports `KeyThenTime` although nobody passed it in. +/// An evaluated stream with no ordering declaration is rejected before the empty source is polled. +#[tokio::test] +async fn unpack_rejects_missing_ordering_on_empty_evaluated_stream() { + let frame = series_frame_schema(&evaluated_schema(), FrameLabelMode::Explicit, None).unwrap(); + let mut metadata = frame.metadata().clone(); + metadata.remove("greptime.series.ordering"); + let frame = Arc::new(Schema::new_with_metadata(frame.fields().clone(), metadata)); + let (source, polls, _) = df_source(vec![], frame); + assert!(super::unpack_evaluated_series(source, None).is_err()); + assert_eq!(polls.load(Ordering::SeqCst), 0); +} + #[tokio::test] async fn unpack_reports_the_ordering_the_frame_declares() { let (input, _, _) = raw_custom( @@ -2933,7 +3075,7 @@ async fn unpack_reports_the_ordering_the_frame_declares() { ); let packed = pack_raw_series(input).unwrap(); assert_eq!( - raw_series_frame_ordering(packed.schema().as_ref()).unwrap(), + series_frame_ordering(packed.schema().as_ref()).unwrap(), SeriesOrdering::KeyThenTime, "the producer declaration reaches the wire" ); @@ -2945,27 +3087,28 @@ async fn unpack_reports_the_ordering_the_frame_declares() { "unpacking derives the ordering from the frame, not from a caller" ); - // A frame that declares no ordering is a legacy contiguous frame. - let frame = raw_series_frame_schema(&schema(), RawLabelMode::Explicit).unwrap(); + let frame = raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap(); + let mut metadata = frame.metadata().clone(); + metadata.remove("greptime.series.ordering"); + let frame = Arc::new(Schema::new_with_metadata(frame.fields().clone(), metadata)); let (source, _, _) = df_source(vec![], frame); - assert_eq!( - unpack_raw_series(source, None).unwrap().ordering(), - SeriesOrdering::Contiguous - ); + assert!(unpack_raw_series(source, None).is_err()); } /// Reserved layout metadata this revision does not write is rejected instead of being read as the -/// current layout, so a frame cannot be decoded under a contract it did not declare. +/// current layout, so a frame cannot be decoded under a contract it did not declare. The current +/// layout is `v2`; a `v1` frame and every unknown value are refused alike. #[tokio::test] async fn unpack_rejects_unknown_layout_metadata() { let cases = [ - ("greptime.series.layout", "v2"), + ("greptime.series.layout", "v1"), + ("greptime.series.layout", "v3"), ("greptime.series.content", "series"), ("greptime.series.time", "timestamps_v2"), ("greptime.series.ordering", "hash_then_time"), ]; for (key, value) in cases { - let frame = raw_series_frame_schema(&schema(), RawLabelMode::Explicit).unwrap(); + let frame = raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap(); let mut metadata = frame.metadata().clone(); metadata.insert(key.to_string(), value.to_string()); let frame = Arc::new(Schema::new_with_metadata(frame.fields().clone(), metadata)); @@ -2982,7 +3125,7 @@ async fn unpack_rejects_unknown_layout_metadata() { /// same frame unpacks without it because the frame is the only source of decode information. #[tokio::test] async fn unpack_rejects_an_expected_schema_the_frame_does_not_declare() { - let frame = raw_series_frame_schema(&schema(), RawLabelMode::Explicit).unwrap(); + let frame = raw_series_frame_schema(&schema(), FrameLabelMode::Explicit).unwrap(); let other_field = Arc::new(SeriesSchema { fields: vec![FieldSpec { @@ -3049,7 +3192,7 @@ async fn unpack_pack_unpack_preserves_the_description_and_the_ordering() { let (input, _, _) = raw_custom(vec![key_chunk()], SeriesOrdering::KeyThenTime, binding); let packed = pack_raw_series(input).unwrap(); assert_eq!( - datatypes::series::frame::raw_series_frame_label_decode(packed.schema().as_ref()).unwrap(), + datatypes::series::frame::series_frame_label_decode(packed.schema().as_ref()).unwrap(), Some(description.clone()) ); @@ -3059,13 +3202,12 @@ async fn unpack_pack_unpack_preserves_the_description_and_the_ordering() { let repacked = pack_raw_series(unpacked).unwrap(); assert_eq!( - datatypes::series::frame::raw_series_frame_label_decode(repacked.schema().as_ref()) - .unwrap(), + datatypes::series::frame::series_frame_label_decode(repacked.schema().as_ref()).unwrap(), Some(description.clone()), "the repacked frame declares what the unpacked stream declared" ); assert_eq!( - raw_series_frame_ordering(repacked.schema().as_ref()).unwrap(), + series_frame_ordering(repacked.schema().as_ref()).unwrap(), SeriesOrdering::KeyThenTime, "the repacked frame keeps the ordering it was unpacked under" ); @@ -3078,3 +3220,42 @@ async fn unpack_pack_unpack_preserves_the_description_and_the_ordering() { assert!(chunk.metric.is_none(), "key chunks stay undecoded"); assert!(again.next().await.is_none()); } + +/// Each typed schema entry point refuses the content it cannot describe. +/// +/// The raw entry point is handed an evaluated series schema and the shared builder is handed an +/// evaluated schema with a key-then-time ordering. Both would otherwise produce a frame whose +/// declaration contradicts its own columns, and the reader would be the first to notice — after the +/// producer had already published it. +#[test] +fn typed_schema_entry_points_refuse_the_content_they_cannot_describe() { + assert!( + super::raw_series_frame_schema_with_ordering( + &evaluated_schema(), + &RawLabelSource::Explicit, + SeriesOrdering::Contiguous, + ) + .is_err(), + "a raw frame cannot carry a schema with an evaluation grid" + ); + assert!( + super::frame_schema_with( + &evaluated_schema(), + FrameLabelMode::Explicit, + None, + SeriesOrdering::KeyThenTime, + ) + .is_err(), + "an evaluated frame has no storage key to be ordered by" + ); + assert!( + super::frame_schema_with( + &evaluated_schema(), + FrameLabelMode::Explicit, + None, + SeriesOrdering::Contiguous, + ) + .is_ok(), + "contiguous is the one ordering an evaluated frame can honour" + ); +} diff --git a/src/common/query/src/series_frame/tests/evaluated.rs b/src/common/query/src/series_frame/tests/evaluated.rs new file mode 100644 index 00000000000..5a2336a4707 --- /dev/null +++ b/src/common/query/src/series_frame/tests/evaluated.rs @@ -0,0 +1,1250 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Streaming round trips through an evaluated series frame. +//! +//! An evaluated frame names its series by their complete labels and addresses time on the grid its +//! schema declares, so every test here drives the real path - pack, a record batch stream, unpack - +//! and compares what a consumer receives against what a producer sent. A schema snapshot would +//! prove the declaration; these prove the values. +//! +//! A frame row is one run of positions on the shared grid, so unpacking yields one [`Series`] per +//! row rather than one per source chunk. A consumer rebuilds a chunk chain by concatenating rows up +//! to the one that carries `is_last`, which is what these tests do. + +use std::io::Cursor; +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use common_error::ext::{BoxedError, ErrorExt}; +use common_error::status_code::StatusCode; +use common_recordbatch::DfSendableRecordBatchStream; +use datatypes::arrow::array::{ + Array, BooleanArray, Float64Array, ListArray, StructArray, UInt32Array, UInt64Array, +}; +use datatypes::arrow::buffer::OffsetBuffer; +use datatypes::arrow::datatypes::{DataType, SchemaRef}; +use datatypes::arrow::ipc::{reader::StreamReader, writer::StreamWriter}; +use datatypes::arrow::record_batch::{RecordBatch, RecordBatchOptions}; +use datatypes::series::frame::{FrameLabelMode, SeriesFrameBuilder, SeriesFrameCursor}; +use datatypes::series::histogram::{ + BucketSideView, CounterResetHint, HistogramBatch, HistogramBatchBuilder, HistogramView, +}; +use datatypes::series::samples::{ + DenseSamplesRef, FieldSpec, SampleRef, Samples, Series, SeriesFields, SeriesSchema, +}; +use datatypes::series::time::{GridSpan, TimeAxis}; +use datatypes::series::{Label, SeriesLabels, SeriesOrdering}; +use futures_util::StreamExt; + +use super::super::{ + SendableEvaluatedSeriesStream, pack_evaluated_series, series_frame_ordering, + unpack_evaluated_series, unpack_raw_series, +}; +use super::{df_source, evaluated_grid, evaluated_schema, schema}; + +/// Complete labels of the `cpu` metric on one host. +fn labels(host: &str) -> SeriesLabels { + SeriesLabels::try_new( + Some(Arc::from("cpu")), + vec![Label { + name: Arc::from("host"), + value: Arc::from(host), + }], + ) + .unwrap() +} + +/// One validated evaluated chunk: `values` on the runs `spans` of the shared test grid. +fn series(spans: &[(usize, usize)], values: Vec>, metric: SeriesLabels) -> Series { + let time = TimeAxis::from_grid_spans( + evaluated_grid(), + spans + .iter() + .map(|(start, len)| GridSpan::new(*start, *len)) + .collect(), + ) + .unwrap(); + let fields = SeriesFields::one(Samples::from_float(Float64Array::from(values))); + Series::try_new(&evaluated_schema(), metric, time, fields).unwrap() +} + +/// An evaluated producer stream over the supplied chunks and their finality. +fn source(items: Vec<(Series, bool)>) -> SendableEvaluatedSeriesStream { + Box::pin(futures_util::stream::iter( + items.into_iter().map(Ok::<_, BoxedError>), + )) +} + +/// Packs the supplied chunks, keeping the stream so a test can observe its errors. +fn pack(items: Vec<(Series, bool)>) -> DfSendableRecordBatchStream { + pack_evaluated_series(source(items), evaluated_schema()).unwrap() +} + +/// Packs the supplied chunks and drains every ready batch. +async fn packed(items: Vec<(Series, bool)>) -> Vec { + let mut stream = pack(items); + let mut out = Vec::new(); + while let Some(batch) = stream.next().await { + out.push(batch.unwrap()); + } + out +} + +/// Unpacks the batches a producer published, under the schema they declare. +fn unpack(batches: Vec, declared: SchemaRef) -> SendableEvaluatedSeriesStream { + let (source, _, _) = df_source(batches, declared); + unpack_evaluated_series(source, Some(evaluated_schema())).unwrap() +} + +/// Drains one unpacked stream, so a test can assert on the whole sequence at once. +async fn drain(mut stream: SendableEvaluatedSeriesStream) -> Vec<(Series, bool)> { + let mut out = Vec::new(); + while let Some(chunk) = stream.next().await { + out.push(chunk.unwrap()); + } + out +} + +/// The absolute grid positions a decoded chunk carries, in order. +fn positions(series: &Series) -> Vec { + series + .time + .positions() + .map(|(position, _)| position) + .collect() +} + +/// The float payload of the one `value` field, in compact-row order. +fn values(series: &Series) -> Vec> { + series + .fields + .as_slice() + .first() + .expect("the fixture has one value field") + .iter() + .map(|sample| match sample { + SampleRef::Missing => None, + SampleRef::Float(value) => Some(value), + _ => panic!("the fixture carries float samples"), + }) + .collect() +} + +/// The positions and values of every decoded row, concatenated in stream order. +fn flat(rows: &[(Series, bool)]) -> (Vec, Vec>) { + let mut all_positions = Vec::new(); + let mut all_values = Vec::new(); + for (row, _) in rows { + all_positions.extend(positions(row)); + all_values.extend(values(row)); + } + (all_positions, all_values) +} + +/// Rewrites the `is_last` column of a frame, so a test can produce a row a producer refuses to write +/// and check that the reader refuses it too. +fn with_is_last(batch: &RecordBatch, is_last: bool) -> RecordBatch { + let mut columns = batch.columns().to_vec(); + columns[1] = Arc::new(BooleanArray::from(vec![is_last; batch.num_rows()])); + RecordBatch::try_new(batch.schema(), columns).unwrap() +} + +/// Rewrites the `read_key` column of a frame, so a test can attach the storage key an evaluated row +/// must not carry. +fn with_read_key(batch: &RecordBatch, key: &[u8]) -> RecordBatch { + let keys = vec![Some(key); batch.num_rows()]; + let mut columns = batch.columns().to_vec(); + columns[0] = Arc::new(datatypes::arrow::array::BinaryArray::from(keys)); + RecordBatch::try_new(batch.schema(), columns).unwrap() +} + +/// The frame schema a producer's batches carry, with one metadata entry changed the way a divergent +/// producer would write it. +fn relabelled(declared: &SchemaRef, key: &str, value: Option<&str>) -> SchemaRef { + let mut metadata = declared.metadata().clone(); + match value { + Some(value) => metadata.insert(key.to_string(), value.to_string()), + None => metadata.remove(key), + }; + Arc::new(datatypes::arrow::datatypes::Schema::new_with_metadata( + declared.fields().clone(), + metadata, + )) +} + +/// One dense chunk keeps every position, label, and value, and its finality. +#[tokio::test] +async fn dense_roundtrip_preserves_positions_labels_and_values() { + let metric = labels("a"); + let expected: Vec> = (0..8) + .map(|row| (row % 3 != 0).then_some(row as f64)) + .collect(); + let batches = packed(vec![( + series(&[(0, 8)], expected.clone(), metric.clone()), + true, + )]) + .await; + assert_eq!(batches.len(), 1, "eight positions are one row"); + assert_eq!(batches[0].num_rows(), 1); + + let declared = batches[0].schema(); + assert_eq!( + series_frame_ordering(declared.as_ref()).unwrap(), + SeriesOrdering::Contiguous, + "an evaluated frame declares contiguous ordering" + ); + let declaration = datatypes::series::frame::SeriesFrameDeclaration::parse(declared.as_ref()) + .expect("an evaluated frame declares itself"); + assert_eq!( + declaration.content(), + datatypes::series::frame::FrameContent::Evaluated + ); + assert_eq!( + declaration.labels(), + datatypes::series::frame::FrameLabelMode::Explicit + ); + assert_eq!( + declared + .metadata() + .get("greptime.series.time") + .map(String::as_str), + Some("grid_spans"), + "an evaluated frame addresses time on its grid" + ); + assert!(declaration.label_decode().is_none(), "no key is decoded"); + + let rows = drain(unpack(batches, declared)).await; + assert_eq!(rows.len(), 1); + let (chunk, is_last) = &rows[0]; + assert!(is_last, "the source closed the chain"); + assert_eq!(chunk.metric, metric, "labels round trip"); + assert_eq!(positions(chunk), (0..8).collect::>()); + assert_eq!(values(chunk), expected, "every value, including nulls"); + assert!( + chunk + .time + .grid() + .is_some_and(|grid| grid.spec() == evaluated_grid().spec()), + "the decoded chunk is addressed on the frame's own grid" + ); +} + +/// A sparse chunk keeps its holes: the decoded positions are exactly the ones the producer sent, and +/// every field stays aligned with them. +#[tokio::test] +async fn sparse_roundtrip_keeps_holes_and_field_alignment() { + let metric = labels("b"); + let expected = vec![Some(1.0), None, Some(3.0), Some(4.0), None]; + let batches = packed(vec![( + series(&[(0, 2), (5, 3)], expected.clone(), metric.clone()), + true, + )]) + .await; + assert_eq!(batches.len(), 1); + let declared = batches[0].schema(); + let rows = drain(unpack(batches, declared)).await; + + let (decoded_positions, decoded_values) = flat(&rows); + assert_eq!( + decoded_positions, + vec![0, 1, 5, 6, 7], + "the hole between the runs is not filled" + ); + assert_eq!( + decoded_values, expected, + "values stay aligned to positions, hole included" + ); + for (row, _) in &rows { + assert_eq!(row.metric, metric); + for hole in [2, 3, 4] { + assert!( + row.time.row_of(hole).is_none(), + "no row exists at a position the chunk does not hold" + ); + } + } + assert!( + rows.last().expect("at least one row").1, + "only the last row of the chunk closes the chain" + ); + assert_eq!( + rows.iter().filter(|(_, is_last)| *is_last).count(), + 1, + "one chain, one close" + ); +} + +/// A chunk with no positions that closes its chain is one frame row, not a dropped chunk: a consumer +/// has to see the finality it carries. +#[tokio::test] +async fn zero_sample_closing_chunk_is_kept_as_one_row() { + let metric = labels("c"); + let batches = packed(vec![ + ( + series(&[(3, 2)], vec![Some(1.0), Some(2.0)], metric.clone()), + false, + ), + (series(&[], vec![], metric.clone()), true), + ]) + .await; + assert_eq!(batches.len(), 1); + assert_eq!( + batches[0].num_rows(), + 2, + "the closing chunk keeps its own row" + ); + + let declared = batches[0].schema(); + let rows = drain(unpack(batches, declared)).await; + assert_eq!(rows.len(), 2); + let (carried, carried_is_last) = &rows[0]; + assert!(!carried_is_last, "the first chunk stays open"); + assert_eq!(positions(carried), vec![3, 4]); + assert_eq!(values(carried), vec![Some(1.0), Some(2.0)]); + let (closing, closing_is_last) = &rows[1]; + assert!(closing_is_last, "the closing chunk keeps its finality"); + assert_eq!( + positions(closing), + Vec::::new(), + "and carries no position" + ); + assert_eq!(values(closing), Vec::>::new()); + assert_eq!(closing.metric, metric, "labels survive on a closing chunk"); +} + +/// One grid position per run, so a chunk with `count` positions starting at `start` becomes `count` +/// frame rows: one row per run, which is what a batch bound can split. +fn single_position_spans(start: usize, count: usize) -> Vec<(usize, usize)> { + (0..count).map(|offset| (start + offset, 1)).collect() +} + +/// Rows split across batches keep their order and finality: every position appears once and in +/// order, and only the last row of the source's final chunk closes the chain. +#[tokio::test] +async fn multi_chunk_multi_batch_stream_preserves_order_and_finality() { + let metric = labels("d"); + let first: Vec> = (0..300).map(|row| Some(row as f64)).collect(); + let second: Vec> = (0..10).map(|row| Some(row as f64)).collect(); + let batches = packed(vec![ + ( + series(&single_position_spans(0, 300), first, metric.clone()), + false, + ), + ( + series(&single_position_spans(300, 10), second, metric.clone()), + true, + ), + ]) + .await; + assert!( + batches.len() > 1, + "the default row bound splits 310 rows into several batches" + ); + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + 310 + ); + + let declared = batches[0].schema(); + assert!(batches.iter().all(|batch| batch.schema() == declared)); + let rows = drain(unpack(batches, declared)).await; + let (decoded_positions, _) = flat(&rows); + assert_eq!( + decoded_positions, + (0..310).collect::>(), + "no position is repeated, reordered, or dropped across batches" + ); + assert!(rows.iter().all(|(row, _)| row.metric == metric)); + let closes = rows + .iter() + .enumerate() + .filter(|(_, (_, is_last))| *is_last) + .map(|(index, _)| index) + .collect::>(); + assert_eq!( + closes, + vec![rows.len() - 1], + "exactly the last row closes the chain, inherited from the source's final chunk" + ); +} + +/// Two chains whose labels happen to be equal are two chains: each closes itself, and the second is +/// not mistaken for a reopened series. +#[tokio::test] +async fn consecutive_chains_with_equal_labels_each_close_independently() { + let metric = labels("e"); + let batches = packed(vec![ + ( + series(&[(0, 2)], vec![Some(1.0), Some(2.0)], metric.clone()), + true, + ), + ( + series(&[(2, 2)], vec![Some(3.0), Some(4.0)], metric.clone()), + true, + ), + ]) + .await; + + let declared = batches[0].schema(); + let rows = drain(unpack(batches, declared)).await; + assert_eq!(rows.len(), 2, "one row per source chunk"); + assert!( + rows.iter().all(|(_, is_last)| *is_last), + "each chain closes itself, so neither is refused as a reopened series" + ); + assert!(rows.iter().all(|(row, _)| row.metric == metric)); + assert_eq!(positions(&rows[0].0), vec![0, 1]); + assert_eq!(values(&rows[0].0), vec![Some(1.0), Some(2.0)]); + assert_eq!(positions(&rows[1].0), vec![2, 3]); + assert_eq!(values(&rows[1].0), vec![Some(3.0), Some(4.0)]); +} + +/// A chain still open at EOF is an error on both sides: a producer that never closed a series cannot +/// finish, and a receiver cannot pretend the stream ended cleanly. +#[tokio::test] +async fn eof_with_an_open_chain_is_an_error() { + let metric = labels("f"); + let mut stream = pack(vec![( + series(&[(0, 2)], vec![Some(1.0), Some(2.0)], metric.clone()), + false, + )]); + // The rows are buffered until EOF, where the missing close is reported instead of a batch. + let error = loop { + match stream.next().await { + Some(Ok(_)) => continue, + Some(Err(error)) => break error, + None => panic!("an open chain at EOF must be reported"), + } + }; + assert_eq!( + crate::error::datafusion_status_code::(&error, None), + StatusCode::InvalidArguments + ); + assert!(stream.next().await.is_none(), "the error is reported once"); + + // The receiving side reaches the same conclusion from the frame alone. + let closing = packed(vec![( + series(&[(0, 2)], vec![Some(1.0), Some(2.0)], metric), + true, + )]) + .await; + let declared = closing[0].schema(); + let opened = with_is_last(&closing[0], false); + let (source, _, _) = df_source(vec![opened], declared); + let mut unpacked = unpack_evaluated_series(source, Some(evaluated_schema())).unwrap(); + assert!(!unpacked.next().await.unwrap().unwrap().1); + let error = unpacked.next().await.unwrap().unwrap_err(); + assert_eq!(error.status_code(), StatusCode::InvalidArguments); + assert!(unpacked.next().await.is_none()); +} + +/// A chunk with no position that does not close its chain can never make progress, so both sides +/// refuse it. +#[tokio::test] +async fn zero_sample_chunk_that_does_not_close_is_rejected() { + let metric = labels("g"); + let mut stream = pack(vec![(series(&[], vec![], metric.clone()), false)]); + let error = stream.next().await.unwrap().unwrap_err(); + assert_eq!( + crate::error::datafusion_status_code::(&error, None), + StatusCode::InvalidArguments + ); + assert!(stream.next().await.is_none()); + + let closing = packed(vec![(series(&[], vec![], metric), true)]).await; + assert_eq!(closing[0].num_rows(), 1); + let declared = closing[0].schema(); + let not_closing = with_is_last(&closing[0], false); + let (source, _, _) = df_source(vec![not_closing], declared); + let mut unpacked = unpack_evaluated_series(source, Some(evaluated_schema())).unwrap(); + let error = unpacked.next().await.unwrap().unwrap_err(); + assert_eq!(error.status_code(), StatusCode::InvalidArguments); + assert!(unpacked.next().await.is_none()); +} + +/// Computed values have no storage key to be ordered by, so an evaluated frame that declares any +/// other ordering, or a spelling the wire does not define, is refused before the stream is returned. +#[tokio::test] +async fn evaluated_frame_rejects_non_contiguous_ordering() { + let batches = packed(vec![( + series(&[(0, 1)], vec![Some(1.0)], labels("h")), + true, + )]) + .await; + let declared = batches[0].schema(); + for ordering in ["read_key_then_time", "interleaved"] { + let schema = relabelled(&declared, "greptime.series.ordering", Some(ordering)); + let (source, polls, _) = df_source(vec![], schema); + assert!( + unpack_evaluated_series(source, Some(evaluated_schema())).is_err(), + "{ordering} is not an ordering computed values can claim" + ); + assert_eq!(polls.load(Ordering::SeqCst), 0); + } +} + +/// Every contradictory or unreadable declaration is refused instead of decoded: a missing or +/// malformed grid, a time mode that contradicts the content, and keyed identity on computed values. +#[tokio::test] +async fn evaluated_frame_rejects_contradictory_declarations() { + let batches = packed(vec![( + series(&[(0, 1)], vec![Some(1.0)], labels("i")), + true, + )]) + .await; + let declared = batches[0].schema(); + let cases: [(&str, SchemaRef); 6] = [ + ( + "no grid", + relabelled(&declared, "greptime.series.grid", None), + ), + ( + "malformed grid", + relabelled(&declared, "greptime.series.grid", Some("grid")), + ), + ( + "incomplete grid", + relabelled(&declared, "greptime.series.grid", Some(r#"{"start_ms":0}"#)), + ), + ( + "evaluated with timestamps", + relabelled(&declared, "greptime.series.time", Some("timestamps")), + ), + ( + "evaluated with key labels", + relabelled(&declared, "greptime.series.labels", Some("key")), + ), + ( + "raw content", + relabelled(&declared, "greptime.series.content", Some("raw")), + ), + ]; + for (label, schema) in cases { + let (source, polls, _) = df_source(vec![], schema); + assert!( + unpack_evaluated_series(source, None).is_err(), + "{label}: must be rejected" + ); + assert_eq!( + polls.load(Ordering::SeqCst), + 0, + "{label}: a rejected frame is never polled" + ); + } + + // `expected` is a check, never an input: a schema without the frame's grid is a disagreement. + let (source, _, _) = df_source(vec![], declared); + assert!( + unpack_evaluated_series(source, Some(schema())).is_err(), + "a raw expectation cannot decode an evaluated frame" + ); +} + +/// The two contents are not interchangeable: each unpack entry point refuses the other's frame. +#[tokio::test] +async fn each_unpack_entry_point_refuses_the_other_content() { + let evaluated = packed(vec![( + series(&[(0, 1)], vec![Some(1.0)], labels("j")), + true, + )]) + .await; + let declared = evaluated[0].schema(); + let (source, _, _) = df_source(evaluated, declared); + assert!( + unpack_raw_series(source, None).is_err(), + "a raw reader cannot decode grid-addressed rows" + ); + + let raw_frame = datatypes::series::frame::raw_series_frame_schema( + &schema(), + datatypes::series::frame::FrameLabelMode::Explicit, + ) + .unwrap(); + let (source, _, _) = df_source(vec![], raw_frame); + assert!( + unpack_evaluated_series(source, None).is_err(), + "an evaluated reader cannot decode timestamp-addressed rows" + ); +} + +/// An evaluated row names its series by label, so a row that also carries a storage key is a +/// contradiction: the reader refuses it instead of decoding an identity it cannot check. +#[tokio::test] +async fn evaluated_frame_rejects_a_storage_key_on_a_row() { + let batches = packed(vec![( + series(&[(0, 2)], vec![Some(1.0), Some(2.0)], labels("l")), + true, + )]) + .await; + let declared = batches[0].schema(); + let keyed = with_read_key(&batches[0], b"encoded-key"); + let (source, _, _) = df_source(vec![keyed], declared); + let mut unpacked = unpack_evaluated_series(source, Some(evaluated_schema())).unwrap(); + let error = unpacked.next().await.unwrap().unwrap_err(); + assert_eq!(error.status_code(), StatusCode::InvalidArguments); + assert!(unpacked.next().await.is_none()); +} + +/// An evaluated frame is self-describing across Arrow IPC: the schema, its metadata, and its rows +/// survive serialization, and a consumer decodes them with no planning-side copy of the grid. +#[tokio::test] +async fn evaluated_frame_roundtrips_through_arrow_ipc() { + let metric = labels("k"); + let expected: Vec> = vec![Some(1.5), None, Some(3.5)]; + let batches = packed(vec![( + series(&[(0, 2), (4, 1)], expected.clone(), metric.clone()), + true, + )]) + .await; + let declared = batches[0].schema(); + + let mut ipc = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut ipc, &declared).unwrap(); + for batch in &batches { + writer.write(batch).unwrap(); + } + writer.finish().unwrap(); + } + let reader = StreamReader::try_new(Cursor::new(ipc), None).unwrap(); + let read_schema = reader.schema(); + assert_eq!(read_schema, declared); + assert_eq!( + datatypes::series::frame::SeriesFrameDeclaration::parse(read_schema.as_ref()) + .unwrap() + .series() + .eval_grid + .as_ref() + .expect("the frame still declares its grid") + .spec(), + evaluated_grid().spec() + ); + + let read: Vec = reader.map(Result::unwrap).collect(); + assert_eq!(read.len(), batches.len()); + let (source, _, _) = df_source(read, read_schema); + // No `expected` schema is supplied: the frame alone says what its rows mean. + let rows = drain(unpack_evaluated_series(source, None).unwrap()).await; + let (decoded_positions, decoded_values) = flat(&rows); + assert_eq!(decoded_positions, vec![0, 1, 4]); + assert_eq!(decoded_values, expected); + assert!(rows.iter().all(|(row, _)| row.metric == metric)); + assert!(rows.last().expect("at least one row").1); +} + +/// A schema with two value fields addressed on [`evaluated_grid`], so a fixture can carry more than +/// the single float field [`series`] declares. +fn pair_schema() -> Arc { + Arc::new(SeriesSchema { + fields: vec![ + FieldSpec { + name: Arc::from("value"), + }, + FieldSpec { + name: Arc::from("score"), + }, + ] + .into(), + eval_grid: Some(evaluated_grid()), + }) +} + +/// One validated evaluated chunk: `fields` on the runs `spans` of the shared test grid, declared +/// under `schema`. +fn chunk_on( + schema: &SeriesSchema, + spans: &[(usize, usize)], + fields: SeriesFields, + metric: SeriesLabels, +) -> Series { + let time = TimeAxis::from_grid_spans( + evaluated_grid(), + spans + .iter() + .map(|(start, len)| GridSpan::new(*start, *len)) + .collect(), + ) + .unwrap(); + Series::try_new(schema, metric, time, fields).unwrap() +} + +/// Packs the supplied chunks under `schema` and drains every ready batch, for a fixture whose field +/// list is not the single `value` field [`evaluated_schema`] declares. +async fn packed_under(schema: Arc, items: Vec<(Series, bool)>) -> Vec { + let mut stream = pack_evaluated_series(source(items), schema).unwrap(); + let mut out = Vec::new(); + while let Some(batch) = stream.next().await { + out.push(batch.unwrap()); + } + out +} + +/// Unpacks a producer's batches as a consumer that planned the same series schema would, which is +/// how a fixture with more than the one `value` field is read back. +fn unpack_as( + batches: Vec, + declared: SchemaRef, + expected: Arc, +) -> SendableEvaluatedSeriesStream { + let (source, _, _) = df_source(batches, declared); + unpack_evaluated_series(source, Some(expected)).unwrap() +} + +/// The float payload of field `index`, in compact-row order. +fn field_values(series: &Series, index: usize) -> Vec> { + series + .fields + .get(index) + .expect("the fixture reads a field it declared") + .iter() + .map(|sample| match sample { + SampleRef::Missing => None, + SampleRef::Float(value) => Some(value), + _ => panic!("the fixture carries float samples"), + }) + .collect() +} + +/// Compares two per-position float payloads, with `NaN` comparing equal to itself. +/// +/// `Option` cannot be compared with `==` here: the fixtures carry `NaN` as a *value*, and +/// `NaN != NaN`, so equality would report a faithful round trip as a mismatch. Comparing bit +/// patterns keeps a value distinct from a missing position, and `None` still has to meet `None`. +fn assert_float_payload_eq(actual: &[Option], expected: &[Option], field: &str) { + assert_eq!(actual.len(), expected.len(), "{field}: position count"); + for (position, (lhs, rhs)) in actual.iter().zip(expected).enumerate() { + match (lhs, rhs) { + (None, None) => {} + (Some(lhs), Some(rhs)) => assert_eq!( + lhs.to_bits(), + rhs.to_bits(), + "{field}: position {position} must keep its value" + ), + _ => panic!("{field}: position {position} is {lhs:?}, expected {rhs:?}"), + } + } +} + +/// How many positions carry a `NaN` value and how many carry nothing, counted separately because +/// the two must not be confused on the way through the codec. +fn nan_and_null_counts(values: &[Option]) -> (usize, usize) { + let nan = values + .iter() + .filter(|value| value.is_some_and(|value| value.is_nan())) + .count(); + let null = values.iter().filter(|value| value.is_none()).count(); + (nan, null) +} + +/// The native histogram table the histogram fixture references: two rows with distinct sums, the +/// first with a start timestamp and the second without. +/// +/// The shape is the one `datatypes::series::frame::tests::histogram_batch_with_sum` builds, so this +/// test compares against the same payload the frame layer's own tests use instead of inventing a +/// second fixture. +fn histogram_batch() -> Arc { + let custom = [1.5, 4.5]; + let positive_offsets = [1]; + let positive_lengths = [2]; + let positive_buckets = [3.25, 7.5]; + let negative_offsets = [-2]; + let negative_lengths = [1]; + let negative_buckets = [0.75]; + let view = |sum, start_timestamp| HistogramView { + schema: -7, + zero_threshold: 0.125, + sum, + count: 17.25, + zero_count: 2.5, + reset_hint: CounterResetHint::Unrecognized(37), + start_timestamp, + custom_values: &custom, + positive: BucketSideView { + span_offsets: &positive_offsets, + span_lengths: &positive_lengths, + buckets: &positive_buckets, + }, + negative: BucketSideView { + span_offsets: &negative_offsets, + span_lengths: &negative_lengths, + buckets: &negative_buckets, + }, + }; + let mut builder = HistogramBatchBuilder::new(); + builder.push(view(-9.5, Some(-123))).unwrap(); + builder.push(view(90.5, None)).unwrap(); + builder.finish().unwrap() +} + +/// The payload struct of one field of a frame batch: the float list and the histogram payload. +/// +/// A field is one column of the frame's `fields` struct, so this is how a test can tell which +/// representation the codec put on the wire instead of only what came back out of it. +fn frame_field_payload(batch: &RecordBatch, field: usize) -> &StructArray { + batch + .column(3) + .as_any() + .downcast_ref::() + .expect("a frame carries its fields as a struct") + .column(field) + .as_any() + .downcast_ref::() + .expect("a field payload is a struct") +} + +/// A multi-field chunk keeps each field's missing positions independent: one position carries a +/// value in one field and nothing in the other, and a `NaN` value never collapses into a missing one. +/// +/// This is the spec's `evaluated/multi-field` case. A schema snapshot cannot stand in for it, because +/// `NaN` and a missing value are both "no ordinary number" once the payload is only a list: which of +/// the two a position holds is visible only after a round trip. +#[tokio::test] +async fn multi_field_partial_nulls_are_not_nan() { + // Position 0 carries only `value` and position 2 only `score`. `NaN` is a value in both fields, + // and the position next to it in the other field is missing. + let value: Vec> = vec![Some(1.0), Some(f64::NAN), None, Some(f64::NAN), Some(4.0)]; + let score: Vec> = vec![None, None, Some(3.0), Some(5.0), Some(f64::NAN)]; + let metric = labels("n"); + let schema = pair_schema(); + let fields = SeriesFields::try_from_vec(vec![ + Samples::from_float(Float64Array::from(value.clone())), + Samples::from_float(Float64Array::from(score.clone())), + ]) + .unwrap(); + let batches = packed_under( + schema.clone(), + vec![( + chunk_on(&schema, &[(0, 2), (5, 3)], fields, metric.clone()), + true, + )], + ) + .await; + // One row per carried run, so the hole between the runs is not what the frame trades away. + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 2); + // Both fields are columns of every row, and both carry float values of their own. + for row in 0..batches[0].num_rows() { + for field in [0, 1] { + let payload = frame_field_payload(&batches[0], field); + assert!( + !payload.column(0).is_null(row), + "field {field} carries a float payload on row {row}" + ); + } + } + + let declared = batches[0].schema(); + let rows = drain(unpack_as(batches, declared, schema)).await; + assert_eq!(rows.len(), 2, "one decoded chunk per frame row"); + assert!( + rows.iter().all(|(row, _)| row.metric == metric), + "both rows name the same series" + ); + assert!(!rows[0].1, "only the last row closes the chain"); + assert!(rows[1].1); + // Concatenated in stream order, exactly as the producer's single chunk was laid out. + let decoded_positions: Vec = rows.iter().flat_map(|(row, _)| positions(row)).collect(); + assert_eq!( + decoded_positions, + vec![0, 1, 5, 6, 7], + "the hole between the two runs is not filled" + ); + let decoded_value: Vec> = rows + .iter() + .flat_map(|(row, _)| field_values(row, 0)) + .collect(); + let decoded_score: Vec> = rows + .iter() + .flat_map(|(row, _)| field_values(row, 1)) + .collect(); + // Per position, because each field decides for itself whether it holds a value. + assert_float_payload_eq(&decoded_value, &value, "value"); + assert_float_payload_eq(&decoded_score, &score, "score"); + // `NaN` and null are counted apart: a codec that turned one into the other would keep the + // position count and the alignment while losing the distinction this test is about. + assert_eq!( + nan_and_null_counts(&decoded_value), + nan_and_null_counts(&value), + "value: the NaN and null counts survive" + ); + assert_eq!( + nan_and_null_counts(&decoded_score), + nan_and_null_counts(&score), + "score: the NaN and null counts survive" + ); + assert_eq!( + nan_and_null_counts(&decoded_value), + (2, 1), + "two NaN values and one null, not three nulls" + ); + assert_eq!( + nan_and_null_counts(&decoded_score), + (1, 2), + "one NaN value and two nulls, not three values" + ); + // The same position carries a value in one field and nothing in the other, in both directions. + assert_eq!(decoded_value[0], Some(1.0)); + assert_eq!(decoded_score[0], None); + assert_eq!(decoded_value[2], None); + assert_eq!(decoded_score[2], Some(3.0)); + // And a NaN is a value: it keeps its own position, where the other field is missing. + assert!(decoded_value[1].is_some_and(f64::is_nan)); + assert_eq!(decoded_score[1], None); +} + +/// A histogram field keeps its native payload across the wire: the frame carries the table and the +/// per-position references to it, so what a consumer decodes is still a histogram. +/// +/// This is the spec's `native histogram/mixed` case. A length check cannot stand in for it, because a +/// silent float rendition preserves the position count; what rules the conversion out is the payload +/// kind on both sides of the wire and the values behind the references. +#[tokio::test] +async fn histogram_payload_roundtrips_without_float_conversion() { + let table = histogram_batch(); + let indices = UInt32Array::from(vec![Some(0), None, Some(1)]); + let samples = Samples::try_from_histogram(indices.clone(), table).unwrap(); + let metric = labels("m"); + let batches = packed(vec![( + chunk_on( + &evaluated_schema(), + &[(3, 3)], + SeriesFields::one(samples), + metric.clone(), + ), + true, + )]) + .await; + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 1, "one run is one frame row"); + + // On the wire the field is a payload struct that carries the histogram and no float list, so the + // codec never filled a float rendition in on the way out. + let payload = frame_field_payload(&batches[0], 0); + assert!( + payload.column(0).is_null(0), + "a histogram field has no float payload" + ); + assert!( + !payload.column(1).is_null(0), + "the histogram payload is what the frame carries" + ); + + let declared = batches[0].schema(); + let rows = drain(unpack(batches, declared)).await; + assert_eq!(rows.len(), 1); + let (decoded, is_last) = &rows[0]; + assert!(is_last); + assert_eq!(decoded.metric, metric); + assert_eq!(positions(decoded), vec![3, 4, 5]); + let field = decoded.fields.get(0).expect("the fixture has one field"); + let DenseSamplesRef::Histogram(histograms) = field.dense().expect("the field is dense") else { + panic!("a histogram field must decode as a histogram, not as a float or mixed segment"); + }; + assert_eq!(histograms.len(), 3, "every position is still there"); + assert_eq!( + histograms.batch().len(), + 2, + "both input histograms are still referenced" + ); + let decoded_indices = histograms.indices(); + assert_eq!(decoded_indices.len(), indices.len()); + for position in 0..indices.len() { + assert_eq!( + decoded_indices.is_valid(position), + indices.is_valid(position), + "position {position}: a null reference stays null" + ); + if indices.is_valid(position) { + assert_eq!(decoded_indices.value(position), indices.value(position)); + } + } + // The values behind the references are the input payload, not a lossy rendition of it. + let first = histograms + .view_at(0) + .expect("position 0 references a histogram"); + assert_eq!(first.count.to_bits(), 17.25_f64.to_bits()); + assert_eq!(first.sum.to_bits(), (-9.5_f64).to_bits()); + assert_eq!(first.start_timestamp, Some(-123)); + assert_eq!( + first + .positive + .buckets + .iter() + .map(|bucket| bucket.to_bits()) + .collect::>(), + [3.25_f64.to_bits(), 7.5_f64.to_bits()] + ); + assert!( + histograms.view_at(1).is_none(), + "a null reference stays missing instead of resolving to a neighbour's payload" + ); + let second = histograms + .view_at(2) + .expect("position 2 references a histogram"); + assert_eq!(second.count.to_bits(), 17.25_f64.to_bits()); + assert_eq!(second.sum.to_bits(), 90.5_f64.to_bits()); + assert_eq!(second.start_timestamp, None); + // The per-position kind survives as well: a referenced position is a histogram sample, never a + // float one. + assert!(matches!(field.sample(0), SampleRef::Histogram(_))); + assert!(matches!(field.sample(1), SampleRef::Missing)); + assert!(matches!(field.sample(2), SampleRef::Histogram(_))); +} + +/// One field can hold floats and histograms at different positions, and the codec keeps them apart: +/// each position decodes as the kind it was written as. +/// +/// This is the `native histogram/mixed` case in its harder form. A codec could keep the position +/// count and still promote one kind into the other in a single position, which only a per-position +/// kind comparison catches. +#[tokio::test] +async fn mixed_payload_roundtrips_without_dropping_either_kind() { + let table = histogram_batch(); + // Position 0 is a float, 1 a histogram, 2 nothing, and 3 a NaN float - which must not be read as + // a missing position, nor as the neighbouring histogram. + let floats = Float64Array::from(vec![Some(1.5), None, None, Some(f64::NAN)]); + let indices = UInt32Array::from(vec![None, Some(0), None, None]); + let histograms = datatypes::series::samples::HistogramSamples::try_new(indices, table).unwrap(); + let samples = Samples::try_from_mixed(floats, histograms).unwrap(); + let metric = labels("o"); + let batches = packed(vec![( + chunk_on( + &evaluated_schema(), + &[(6, 4)], + SeriesFields::one(samples), + metric.clone(), + ), + true, + )]) + .await; + assert_eq!(batches.len(), 1); + assert_eq!(batches[0].num_rows(), 1); + + // A mixed field is the one payload shape that carries both halves, so the frame neither dropped + // the float positions nor flattened the histogram ones. + let payload = frame_field_payload(&batches[0], 0); + assert!( + !payload.column(0).is_null(0), + "a mixed field carries its floats" + ); + assert!( + !payload.column(1).is_null(0), + "a mixed field carries its histograms" + ); + + let declared = batches[0].schema(); + let rows = drain(unpack(batches, declared)).await; + assert_eq!(rows.len(), 1); + let (decoded, is_last) = &rows[0]; + assert!(is_last); + assert_eq!(decoded.metric, metric); + assert_eq!(positions(decoded), vec![6, 7, 8, 9]); + let field = decoded.fields.get(0).expect("the fixture has one field"); + let DenseSamplesRef::Mixed(mixed) = field.dense().expect("the field is dense") else { + panic!("a mixed field must decode as mixed, not as one of its two halves"); + }; + assert_eq!(mixed.floats().len(), 4); + assert_eq!(mixed.histograms().len(), 4); + assert_eq!( + mixed.histograms().batch().len(), + 2, + "the referenced table survives whole" + ); + match field.sample(0) { + SampleRef::Float(value) => assert_eq!(value.to_bits(), 1.5_f64.to_bits()), + _ => panic!("position 0 was a float"), + } + match field.sample(1) { + SampleRef::Histogram(histogram) => { + assert_eq!(histogram.count.to_bits(), 17.25_f64.to_bits()); + assert_eq!(histogram.sum.to_bits(), (-9.5_f64).to_bits()); + } + _ => panic!("position 1 was a histogram"), + } + assert!(matches!(field.sample(2), SampleRef::Missing)); + match field.sample(3) { + // A NaN float stays a float value, and its histogram slot stays empty. + SampleRef::Float(value) => assert!(value.is_nan()), + _ => panic!("position 3 was a NaN float"), + } +} + +/// A frame whose spans add up beyond `usize` is refused, not answered with a wrapped count. +/// +/// The count of a row's positions is the first arithmetic a decoded row does, and it happens before +/// the row is checked against the grid it claims: two spans of `usize::MAX / 2 + 1` each are neither +/// ordered nor inside any grid, but summing them is what a decoder would do first. The sum is +/// therefore checked, so the frame is refused with an error instead of panicking in a debug build or +/// wrapping in a release one. +#[tokio::test] +async fn spans_that_overflow_the_position_count_are_refused() { + let batches = packed(vec![( + series(&[(0, 2)], vec![Some(1.0), Some(2.0)], labels("v")), + true, + )]) + .await; + let declared = batches[0].schema(); + let huge = u64::try_from(usize::MAX / 2 + 1).expect("a stride that leaves usize behind"); + // Two spans, because one is already representable: only their sum leaves `usize`, and that sum + // is what a decoder must check before it can report which span was wrong. + let hostile = with_spans(&batches[0], &[(0, huge), (huge, huge)], true); + let mut stream = unpack(vec![hostile], declared); + let error = stream + .next() + .await + .expect("a refused frame still yields its error") + .expect_err("the position count overflows"); + assert!( + error.to_string().contains("grid span length overflow"), + "refused for another reason: {error}" + ); +} + +/// An empty row that closes nothing is refused by the decoder itself, not only by the stream. +/// +/// A row covering no position is the closing chunk's shape: it exists to carry `is_last`. One that +/// carries `is_last = false` opens a chain it can never advance, so a consumer would wait for a +/// close that no later row owes. The stream validator refuses this, and so does the public decoder, +/// because a caller driving the decoder directly must not be handed a chunk no producer could have +/// written. +#[tokio::test] +async fn an_empty_row_that_closes_nothing_is_refused_by_the_decoder() { + // A legal empty closing chunk is packed first: the illegal shape cannot be produced, which is + // the point of the check below. Making it illegal means clearing its finality and nothing else, + // so the emptiness and the payload lengths stay consistent and only finality can refuse it. + let batches = packed(vec![(series(&[], vec![], labels("w")), true)]).await; + let declared = batches[0].schema(); + assert_eq!(1, batches[0].num_rows(), "the closing chunk is one row"); + let emptied = with_spans(&batches[0], &[], false); + let mut stream = unpack(vec![emptied], declared); + let error = stream + .next() + .await + .expect("a refused frame still yields its error") + .expect_err("an empty row that closes nothing is refused"); + assert!( + error + .to_string() + .contains("an evaluated chunk without positions must close its series"), + "refused for another reason: {error}" + ); +} + +/// Rewrites the span list of every row in `batch` to `spans`, and its finality to `is_last`. +/// +/// Both are rewritten together because a test that breaks a row's shape usually breaks its finality +/// as well: a producer cannot write an empty row that closes nothing, so such a row only exists once +/// a test builds it. +fn with_spans(batch: &RecordBatch, spans: &[(u64, u64)], is_last: bool) -> RecordBatch { + let declared = batch.schema(); + let DataType::Struct(time_members) = declared.field(2).data_type() else { + panic!("time is a struct"); + }; + let DataType::List(span_item) = time_members[0].data_type() else { + panic!("spans are a list"); + }; + let DataType::Struct(span_fields) = span_item.data_type() else { + panic!("a span is a struct"); + }; + let starts: Vec = spans.iter().map(|(start, _)| *start).collect(); + let lengths: Vec = spans.iter().map(|(_, len)| *len).collect(); + let values = StructArray::try_new( + span_fields.clone(), + vec![ + Arc::new(UInt64Array::from(starts)), + Arc::new(UInt64Array::from(lengths)), + ], + None, + ) + .unwrap(); + let lengths_per_row = vec![spans.len(); batch.num_rows()]; + let list = ListArray::try_new( + span_item.clone(), + OffsetBuffer::new(offset_buffer(&lengths_per_row).into()), + Arc::new(values), + None, + ) + .unwrap(); + let time = StructArray::try_new(time_members.clone(), vec![Arc::new(list)], None).unwrap(); + let mut columns = batch.columns().to_vec(); + columns[1] = Arc::new(BooleanArray::from(vec![is_last; batch.num_rows()])); + columns[2] = Arc::new(time); + RecordBatch::try_new_with_options( + declared, + columns, + &RecordBatchOptions::new().with_row_count(Some(batch.num_rows())), + ) + .unwrap() +} + +/// The cumulative list offsets of the given row lengths. +fn offset_buffer(lengths: &[usize]) -> Vec { + let mut offsets = Vec::with_capacity(lengths.len() + 1); + let mut total = 0i32; + offsets.push(0); + for length in lengths { + total += i32::try_from(*length).expect("a test row fits an Arrow offset"); + offsets.push(total); + } + offsets +} + +/// A cursor holding positions on one grid cannot fill a builder that declares another. +/// +/// Both grids describe the same number of points, so every position the cursor writes is in range +/// for the builder and the frame would decode cleanly. What it would decode to is a different set of +/// instants, which no later check could detect: the numbers are the only thing on the wire. The +/// builder therefore compares the grids themselves, not whether a grid is present. +#[test] +fn a_cursor_on_one_grid_cannot_fill_a_builder_declaring_another() { + // Both grids have four points, so the cursor's positions fit the builder exactly. + let other_grid = || { + Arc::new( + datatypes::series::time::EvalGrid::try_new( + datatypes::series::time::GridSpec::try_new(60_000, 10_000, 4) + .expect("a valid grid"), + ) + .expect("a valid grid"), + ) + }; + let other_schema = SeriesSchema { + fields: evaluated_schema().fields.clone(), + eval_grid: Some(other_grid()), + }; + // A chunk that is legal on its own grid: the series and the schema agree, so nothing but the + // builder's own declaration can refuse it. + let fields = SeriesFields::one(Samples::from_float(Float64Array::from(vec![Some(1.0)]))); + let time = TimeAxis::from_grid_spans(other_grid(), vec![GridSpan::new(0, 1)]) + .expect("one span on its own grid"); + let chunk = Series::try_new(&other_schema, labels("g"), time, fields).expect("a legal chunk"); + let mut cursor = + SeriesFrameCursor::try_new_evaluated(&other_schema, chunk, true).expect("a legal cursor"); + + let mut builder = SeriesFrameBuilder::try_new( + &evaluated_schema(), + FrameLabelMode::Explicit, + None, + SeriesFrameBuilder::DEFAULT_MAX_ROWS, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, + ) + .expect("the schema is a frame"); + let error = cursor + .append_to(&mut builder) + .expect_err("the grids differ"); + assert!( + error.to_string().contains("schema mismatch"), + "refused for another reason: {error}" + ); +} diff --git a/src/datatypes/src/series/frame.rs b/src/datatypes/src/series/frame.rs index cf1b5a41492..4271c6c6f02 100644 --- a/src/datatypes/src/series/frame.rs +++ b/src/datatypes/src/series/frame.rs @@ -12,14 +12,31 @@ // See the License for the specific language governing permissions and // limitations under the License. -//! Finite Arrow batches for raw series chunks. Field names preserve the logical series schema in order. +//! One Arrow frame family for raw and evaluated series chunks. //! -//! Packing accumulates bounded multi-row batches: [`RawSeriesChunkCursor`] walks one validated -//! chunk and [`RawSeriesFrameBuilder`] emits rows directly as Arrow arrays once a row or estimated -//! byte bound is reached. Decoding reuses one prepared schema per stream through -//! [`RawSeriesFrameDecoder`]. Column layout, metadata, and wire strings stay on the frozen `v1` -//! contract; a key-encoded frame additionally declares a versioned [`LabelDecodeSpec`] so its -//! labels can be decoded from the received schema alone. +//! A frame carries the same thing whichever sample shape it holds: one row per chunk of one series, +//! with that series' identity, its finality, its time axis, and one column of samples per logical +//! field. Field names preserve the logical series schema in order. What varies is declared rather +//! than inferred, because the receiver of a frame has no scanner, region, or planning context to +//! infer it from: +//! +//! | Content | Identity | Time | Producer | +//! | --- | --- | --- | --- | +//! | `raw` | `key` | `timestamps` | a storage raw-series scan | +//! | `raw` | `explicit` | `timestamps` | a raw input whose labels are already decoded | +//! | `evaluated` | `explicit` | `grid_spans` | a compute node's evaluated output | +//! +//! Any other combination is rejected here, because no producer can mean it: an evaluated value's +//! labels may be rewritten by the computation that produced it, so an evaluated frame cannot claim +//! a storage key as its identity, and raw samples have no evaluation grid to be addressed on. +//! +//! Packing accumulates bounded multi-row batches: [`SeriesFrameCursor`] walks one validated chunk +//! and [`SeriesFrameBuilder`] emits rows directly as Arrow arrays once a row or estimated byte bound +//! is reached. Decoding reuses one prepared +//! schema per stream through [`SeriesFrameDecoder`]. Column layout, metadata, and wire strings stay +//! on the frozen `v2` contract; a key-encoded frame additionally declares a versioned +//! [`LabelDecodeSpec`] so its labels can be decoded from the received schema alone, and an evaluated +//! frame declares its [`GridSpec`] once instead of repeating timestamps. //! //! # Label decode description versioning //! @@ -61,7 +78,7 @@ use std::sync::Arc; use arrow::array::{ Array, ArrayRef, BinaryArray, BooleanArray, Float64Array, Int64Array, ListArray, StringArray, - StructArray, UInt32Array, + StructArray, UInt32Array, UInt64Array, }; use arrow::buffer::{Buffer, NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow::compute::concat; @@ -74,8 +91,9 @@ use crate::error::{InvalidSeriesSnafu, Result}; use crate::series::histogram::{HistogramBatch, fields as histogram_fields}; use crate::series::raw::{RawSeriesChunk, RawTimestamps}; use crate::series::samples::{ - DenseSamplesRef, FieldSpec, HistogramSamples, Samples, SeriesFields, SeriesSchema, + DenseSamplesRef, FieldSpec, HistogramSamples, Samples, Series, SeriesFields, SeriesSchema, }; +use crate::series::time::{EvalGrid, GridRun, GridSpan, GridSpec, TimeAxis}; use crate::series::{Label, SeriesLabels, invalid_series}; const LAYOUT: &str = "greptime.series.layout"; @@ -85,19 +103,107 @@ const TIME: &str = "greptime.series.time"; const GRID: &str = "greptime.series.grid"; /// Metadata key of the versioned label decode description carried by key-encoded frames. const LABEL_DECODE: &str = "greptime.series.label_decode"; +/// Metadata key of the producer ordering carried by a complete frame declaration. +const ORDERING: &str = "greptime.series.ordering"; + +/// Every `greptime.series.*` key a frame this revision writes may carry. +/// +/// Frame schemas carry an explicit `contiguous` default; streaming producers may replace it with +/// their stronger guarantee. +const RESERVED_KEYS: [&str; 7] = [LAYOUT, CONTENT, LABELS, TIME, GRID, LABEL_DECODE, ORDERING]; + +/// The only layout this revision writes and reads. +/// +/// A frame declaring any other value is refused instead of decoded: the sample representation of an +/// unreleased format is not worth a second decoder, and silently guessing would turn a format change +/// into wrong values. +pub const FRAME_LAYOUT_VERSION: &str = "v2"; /// Version of the label decode description this revision writes and reads. +/// +/// This is independent of [`FRAME_LAYOUT_VERSION`]: the description versions its own text, which a +/// reader refuses to guess at, while the layout versions the column and metadata set around it. pub const LABEL_DECODE_VERSION: u32 = 1; -/// Whether a raw frame carries only the complete key or explicit metric labels as well. +/// What one frame's rows hold: raw samples with their own timestamps, or evaluated samples +/// addressed on the grid the frame declares. #[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum RawLabelMode { - /// Labels remain encoded solely in `read_key`. +pub enum FrameContent { + /// Samples read from storage, each row carrying the timestamps it was read at. + Raw, + /// Values computed on an evaluation grid, each row carrying the grid positions it holds. + Evaluated, +} + +impl FrameContent { + /// The `greptime.series.content` text of this content. + pub const fn as_str(self) -> &'static str { + match self { + Self::Raw => "raw", + Self::Evaluated => "evaluated", + } + } + + /// Parses a `greptime.series.content` value this revision implements. + const fn parse(text: &str) -> Option { + match text.as_bytes() { + b"raw" => Some(Self::Raw), + b"evaluated" => Some(Self::Evaluated), + _ => None, + } + } + + /// The `greptime.series.time` text this content's rows are addressed by. + /// + /// Time mode is a function of content and never an independent choice: raw samples carry their + /// own timestamps and evaluated ones sit on a grid, so a frame has no way to hold one without + /// the other. Writing it as a derived text rather than a second enum keeps the two from ever + /// disagreeing. + const fn time_text(self) -> &'static str { + match self { + Self::Raw => "timestamps", + Self::Evaluated => "grid_spans", + } + } + + /// The struct member a row's time column carries for this content. + const fn time_member(self) -> &'static str { + match self { + Self::Raw => "timestamps", + Self::Evaluated => "spans", + } + } +} + +/// How a frame names the series of its rows. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FrameLabelMode { + /// Identity is only the encoded storage key in `read_key`, decoded through the frame's own + /// [`LabelDecodeSpec`]. Key, - /// Complete metric name and ordinary labels are present in the frame. + /// The complete metric name and ordinary labels are present in the frame as well. Explicit, } +impl FrameLabelMode { + /// The `greptime.series.labels` text of this mode. + pub const fn as_str(self) -> &'static str { + match self { + Self::Key => "key", + Self::Explicit => "explicit", + } + } + + /// Parses a `greptime.series.labels` value this revision implements. + const fn parse(text: &str) -> Option { + match text.as_bytes() { + b"key" => Some(Self::Key), + b"explicit" => Some(Self::Explicit), + _ => None, + } + } +} + /// Primary-key encoding of the labels a key-encoded frame carries. /// /// This mirrors the storage contract's `PrimaryKeyEncoding` without making this crate depend on @@ -148,7 +254,7 @@ pub struct LabelDecodeTag { /// a silent fallback. /// /// It is JSON-encoded in one schema metadata entry, always through -/// [`raw_series_frame_schema_with_label_decode`], so both the producing and the expecting side of a +/// [`series_frame_schema`], so both the producing and the expecting side of a /// `greptime.series.*` exact metadata comparison see byte-identical text. The text is /// self-describing but not a general interchange format: readers reject unknown fields here only /// at this structure's own levels, while [`LabelDecodeColumn::physical_type`] is decoded by @@ -188,7 +294,7 @@ pub struct LabelDecodeSpec { impl LabelDecodeSpec { /// Puts the order-insensitive part of the description in its canonical order. /// - /// [`raw_series_frame_schema_with_label_decode`] applies this to a copy before serializing, and + /// [`series_frame_schema`] applies this to a copy before serializing, and /// so does the description a binding derives (in `common_query::series_frame`), so both sides of /// a text comparison produce the same bytes and a reader can compare descriptions structurally. /// `primary_key` is deliberately untouched: its order is the encoding order. @@ -206,11 +312,9 @@ impl LabelDecodeSpec { .sort_unstable_by(|a, b| (a.column_id, &a.name).cmp(&(b.column_id, &b.name))); } - /// Checks the version and the structural invariants a decoder relies on. + /// Checks the version and structural invariants of the description. /// - /// Semantic binding checks (reserved names, internal columns, name equality against the bound - /// column) stay in the decoder so both a typed binding and a decoded description fail with the - /// same errors. + /// Storage-region binding semantics are deliberately outside this schema-level validation. pub fn validate(&self) -> Result<()> { if self.version != LABEL_DECODE_VERSION { return Err(err(format!( @@ -253,27 +357,76 @@ impl LabelDecodeSpec { } } -/// Builds the typed schema for a raw frame, preserving logical field names and order. +/// Builds the one frame schema family: raw or evaluated, keyed or explicit. /// -/// This is the column layout shared with the frame builder and decoder; it deliberately carries no -/// label decode description. A key-encoded stream must be declared through -/// [`raw_series_frame_schema_with_label_decode`], which refuses such a layout without one. -pub fn raw_series_frame_schema(schema: &SeriesSchema, mode: RawLabelMode) -> Result { - if schema.eval_grid.is_some() { - return invalid_series("raw series schema cannot have an evaluation grid"); - } +/// Everything the wire states is derived from the two things a producer actually holds. The +/// evaluation grid comes from `schema.eval_grid`, where a schema without one describes raw samples +/// and a schema with one describes values addressed on that grid, so content and time mode can never +/// contradict the columns they describe. `labels` decides identity, and with it whether the frame +/// carries the `name`/`labels` columns: a key frame keeps its labels encoded in `read_key` and must +/// declare the [`LabelDecodeSpec`] that decodes them, while an explicit frame carries complete +/// labels and must not declare one. +/// +/// This is the only place the reserved `greptime.series.*` text is written, so a producer and a +/// reader cannot spell one frame two ways. +pub fn series_frame_schema( + schema: &SeriesSchema, + labels: FrameLabelMode, + decode: Option<&LabelDecodeSpec>, +) -> Result { if schema.fields.is_empty() { - return invalid_series("raw series schema must have fields"); + return invalid_series("series frame schema must have fields"); + } + let content = match &schema.eval_grid { + Some(_) => FrameContent::Evaluated, + None => FrameContent::Raw, + }; + match (labels, decode) { + (FrameLabelMode::Key, None) => { + return Err(err( + "key-encoded frames must carry a label decode description", + )); + } + (FrameLabelMode::Explicit, Some(_)) => { + return Err(err( + "explicit-label frames carry no label decode description", + )); + } + (FrameLabelMode::Key, Some(_)) | (FrameLabelMode::Explicit, None) => {} + } + if content == FrameContent::Evaluated && labels == FrameLabelMode::Key { + return Err(err( + "an evaluated frame names its series by label, not by a storage key", + )); } let list = |ty, nullable| DataType::List(Arc::new(Field::new("item", ty, nullable))); - let time = DataType::Struct( - vec![Field::new( - "timestamps", - list(DataType::Int64, false), - false, - )] - .into(), - ); + let time = match content { + FrameContent::Raw => DataType::Struct( + vec![Field::new( + "timestamps", + list(DataType::Int64, false), + false, + )] + .into(), + ), + FrameContent::Evaluated => DataType::Struct( + vec![Field::new( + "spans", + list( + DataType::Struct( + vec![ + Field::new("start", DataType::UInt64, false), + Field::new("len", DataType::UInt64, false), + ] + .into(), + ), + false, + ), + false, + )] + .into(), + ), + }; let hist = DataType::Struct( vec![ Field::new("indices", list(DataType::UInt32, true), false), @@ -298,12 +451,19 @@ pub fn raw_series_frame_schema(schema: &SeriesSchema, mode: RawLabelMode) -> Res .map(|field| Field::new(field.name.as_ref(), payload.clone(), false)) .collect(); let mut columns = vec![ - Field::new("read_key", DataType::Binary, false), + // A key frame *is* its key, so the column is required. An explicit frame names its series + // by label and keeps the column for the runtime chunk's own contract: a raw chunk still + // carries the key it was read under, an evaluated one carries none. + Field::new( + "read_key", + DataType::Binary, + labels == FrameLabelMode::Explicit, + ), Field::new("is_last", DataType::Boolean, false), Field::new("time", time, false), Field::new("fields", DataType::Struct(fields), false), ]; - if mode == RawLabelMode::Explicit { + if labels == FrameLabelMode::Explicit { columns.push(Field::new("name", DataType::Utf8, true)); columns.push(Field::new( "labels", @@ -320,189 +480,308 @@ pub fn raw_series_frame_schema(schema: &SeriesSchema, mode: RawLabelMode) -> Res false, )); } - let metadata = [ - (LAYOUT.to_string(), "v1".to_string()), - (CONTENT.to_string(), "raw".to_string()), - ( - LABELS.to_string(), - match mode { - RawLabelMode::Key => "key", - RawLabelMode::Explicit => "explicit", - } - .to_string(), - ), - (TIME.to_string(), "timestamps".to_string()), - ] - .into_iter() - .collect(); - Ok(Arc::new(Schema::new_with_metadata(columns, metadata))) -} - -/// Builds the typed schema for a raw frame together with its label decode description. -/// -/// The description is written only for [`RawLabelMode::Key`], where labels stay encoded in -/// `read_key`; explicit-label frames carry complete labels and must not declare one. Both -/// directions are enforced here so a frame cannot claim a label layout it does not describe. -pub fn raw_series_frame_schema_with_label_decode( - schema: &SeriesSchema, - mode: RawLabelMode, - decode: Option<&LabelDecodeSpec>, -) -> Result { - match (mode, decode) { - (RawLabelMode::Key, None) => { - return Err(err( - "key-encoded raw frames must carry a label decode description", - )); - } - (RawLabelMode::Explicit, Some(_)) => { - return Err(err("explicit raw frames carry no label decode description")); - } - (RawLabelMode::Key, Some(_)) | (RawLabelMode::Explicit, None) => {} + let mut metadata = vec![ + (LAYOUT.to_string(), FRAME_LAYOUT_VERSION.to_string()), + (CONTENT.to_string(), content.as_str().to_string()), + (LABELS.to_string(), labels.as_str().to_string()), + (TIME.to_string(), content.time_text().to_string()), + (ORDERING.to_string(), "contiguous".to_string()), + ]; + if let Some(grid) = &schema.eval_grid { + metadata.push((GRID.to_string(), grid_text(grid.spec())?)); + } + if let Some(decode) = decode { + // Writing is faithful: the description records what the producer declares, and nothing is + // rewritten or dropped. The one exception is the order of `logical_tags`, which carries no + // meaning to a decoder and which this text nevertheless has to fix so two producers of the + // same labels write the same bytes; a copy is canonicalized and the caller's value is + // untouched. + // + // What is not accepted is a description this revision would refuse to read. A frame is worth + // writing only if it can be decoded, so the same check runs here: a duplicate key column, a + // logical tag naming no listed column, or a description of an unknown version is refused at + // the producer rather than when the frame reaches its consumer, where the producer is no + // longer in a position to fix it. + let mut canonical = decode.clone(); + canonical.canonicalize(); + canonical.validate()?; + metadata.push(( + LABEL_DECODE.to_string(), + serde_json::to_string(&canonical) + .map_err(|e| err(format!("cannot encode label decode description: {e}")))?, + )); } - let layout = raw_series_frame_schema(schema, mode)?; - let Some(decode) = decode else { - return Ok(layout); - }; - // Writing is faithful: the description records what the producer declares, and nothing is - // rewritten or dropped. The one exception is the order of `logical_tags`, which carries no - // meaning to a decoder and which this text nevertheless has to fix so two producers of the same - // labels write the same bytes; a copy is canonicalized and the caller's value is untouched. A - // description this revision cannot honour is rejected when it is read, which is the only - // direction a consumer can rely on. - let mut canonical = decode.clone(); - canonical.canonicalize(); - let encoded = serde_json::to_string(&canonical) - .map_err(|e| err(format!("cannot encode label decode description: {e}")))?; - let mut metadata = layout.metadata().clone(); - metadata.insert(LABEL_DECODE.to_string(), encoded); Ok(Arc::new(Schema::new_with_metadata( - layout.fields().clone(), - metadata, + columns, + metadata.into_iter().collect(), ))) } -/// Recovers the label decode description a raw frame schema declares. +/// Builds the typed schema for a raw frame, refusing a series schema with an evaluation grid. +/// +/// A key frame is unreadable without its description, so this constructor serves explicit-label +/// frames only; a caller that holds a binding declares through [`series_frame_schema`]. +pub fn raw_series_frame_schema(schema: &SeriesSchema, labels: FrameLabelMode) -> Result { + if schema.eval_grid.is_some() { + return invalid_series("raw series schema cannot have an evaluation grid"); + } + match labels { + FrameLabelMode::Key => Err(err( + "raw key frames must declare a label decode description", + )), + FrameLabelMode::Explicit => series_frame_schema(schema, labels, None), + } +} + +/// The canonical `greptime.series.grid` text of one grid. +/// +/// Field order is the struct's declaration order and the text is compact, so two producers of the +/// same grid write the same bytes; nothing else in the tree spells this text. +fn grid_text(spec: GridSpec) -> Result { + serde_json::to_string(&spec).map_err(|e| err(format!("cannot encode evaluation grid: {e}"))) +} + +/// Parses one `greptime.series.grid` text back into a validated, deferrable grid. +/// +/// Validation is [`GridSpec::try_new`]'s plus the unknown-field refusal serde performs, so a frame +/// cannot declare a grid this revision would compute differently. The timestamps themselves stay +/// unmaterialized until a consumer asks for them. +fn parse_grid(text: &str) -> Result> { + let spec: GridSpec = serde_json::from_str(text) + .map_err(|e| err(format!("cannot decode evaluation grid: {e}")))?; + Ok(Arc::new(EvalGrid::try_deferred(spec)?)) +} + +/// A frame schema's reserved declarations, parsed and cross-checked. +/// +/// The three facts a receiver needs are not independent - the content decides the time mode, the +/// label mode decides whether a description must be present, and only an evaluated frame may name a +/// grid - so they are parsed together and every contradictory combination is refused instead of one +/// side of it being picked. +#[derive(Clone, Debug)] +pub struct SeriesFrameDeclaration { + series: SeriesSchema, + labels: FrameLabelMode, + label_decode: Option, +} + +fn validate_ordering(schema: &Schema, content: FrameContent) -> Result<()> { + match schema.metadata().get(ORDERING).map(String::as_str) { + Some("contiguous") => Ok(()), + Some("read_key_then_time") if content == FrameContent::Raw => Ok(()), + Some("read_key_then_time") => Err(err( + "an evaluated frame cannot claim key-then-time ordering", + )), + None => Err(err(format!("frame declares no {ORDERING} value"))), + Some(_) => Err(err("unknown series ordering")), + } +} + +impl SeriesFrameDeclaration { + /// Parses and cross-checks one frame schema. + pub fn parse(schema: &Schema) -> Result { + for key in schema.metadata().keys() { + if key.starts_with("greptime.series.") && !RESERVED_KEYS.contains(&key.as_str()) { + return Err(err(format!( + "frame declares unknown reserved metadata {key}" + ))); + } + } + if schema.metadata().get(LAYOUT).map(String::as_str) != Some(FRAME_LAYOUT_VERSION) { + return Err(err(format!("frame declares no known {LAYOUT} value"))); + } + let content = schema + .metadata() + .get(CONTENT) + .and_then(|text| FrameContent::parse(text)) + .ok_or_else(|| err(format!("frame declares no known {CONTENT} value")))?; + validate_ordering(schema, content)?; + let labels = schema + .metadata() + .get(LABELS) + .and_then(|text| FrameLabelMode::parse(text)) + .ok_or_else(|| err(format!("frame declares no known {LABELS} value")))?; + // Time mode is read as the content's own text: a frame whose time column is addressed the + // other way contradicts itself, and there is no third time mode to fall back to. + if schema.metadata().get(TIME).map(String::as_str) != Some(content.time_text()) { + return Err(err(format!( + "frame declares no {TIME} value this content can carry" + ))); + } + if content == FrameContent::Evaluated && labels == FrameLabelMode::Key { + return Err(err( + "an evaluated frame names its series by label, not by a storage key", + )); + } + let grid = match (content, schema.metadata().get(GRID)) { + (FrameContent::Raw, None) => None, + (FrameContent::Raw, Some(_)) => { + return Err(err("a raw frame cannot declare an evaluation grid")); + } + (FrameContent::Evaluated, None) => { + return Err(err("an evaluated frame must declare its evaluation grid")); + } + (FrameContent::Evaluated, Some(text)) => Some(parse_grid(text)?), + }; + let label_decode = match schema.metadata().get(LABEL_DECODE) { + None => None, + Some(text) => { + let decode: LabelDecodeSpec = serde_json::from_str(text) + .map_err(|e| err(format!("cannot decode label decode description: {e}")))?; + decode.validate()?; + Some(decode) + } + }; + match (labels, &label_decode) { + (FrameLabelMode::Key, None) => { + return Err(err( + "key-encoded frames must carry a label decode description", + )); + } + (FrameLabelMode::Explicit, Some(_)) => { + return Err(err( + "explicit-label frames carry no label decode description", + )); + } + (FrameLabelMode::Key, Some(_)) | (FrameLabelMode::Explicit, None) => {} + } + let fields = schema + .fields() + .iter() + .find(|field| field.name() == "fields") + .ok_or_else(|| err("frame declares no fields column"))?; + let DataType::Struct(members) = fields.data_type() else { + return Err(err("frame fields column is not a struct")); + }; + if members.is_empty() { + return Err(err("series frame schema must have fields")); + } + let declared_time = schema + .fields() + .iter() + .find(|field| field.name() == "time") + .ok_or_else(|| err("frame declares no time column"))?; + let DataType::Struct(time_members) = declared_time.data_type() else { + return Err(err("frame time column is not a struct")); + }; + if time_members.len() != 1 || time_members[0].name() != content.time_member() { + return Err(err( + "frame time column does not match its declared time mode", + )); + } + Ok(Self { + series: SeriesSchema { + fields: members + .iter() + .map(|member| FieldSpec { + name: Arc::from(member.name().as_str()), + }) + .collect(), + eval_grid: grid, + }, + labels, + label_decode, + }) + } + + /// The logical fields and evaluation grid the frame declares. + pub fn series(&self) -> &SeriesSchema { + &self.series + } + + /// What the frame's rows hold. + pub fn content(&self) -> FrameContent { + match self.series.eval_grid { + Some(_) => FrameContent::Evaluated, + None => FrameContent::Raw, + } + } + + /// How the frame names its series. + pub fn labels(&self) -> FrameLabelMode { + self.labels + } + + /// The label decode description the frame declares, present exactly for key identity. + pub fn label_decode(&self) -> Option<&LabelDecodeSpec> { + self.label_decode.as_ref() + } +} + +/// Recovers the label decode description a frame schema declares. /// /// Returns `None` for an explicit-label frame, which must not declare one. A key-encoded frame must /// declare one, so a missing description is an error rather than a reason to reuse a caller-side -/// binding. A present description is parsed strictly: an unknown version, a missing or unknown -/// field at this structure's own levels, a type mismatch, or a description whose logical tags do -/// not name listed key columns is an error, never a silent default. Only the description's own -/// levels reject unknown fields; [`LabelDecodeColumn::physical_type`] is a -/// [`ConcreteDataType`] and accepts whatever that type's own serde accepts. -pub fn raw_series_frame_label_decode(schema: &Schema) -> Result> { - let mode = match schema.metadata().get(LABELS).map(String::as_str) { - Some("key") => RawLabelMode::Key, - Some("explicit") => RawLabelMode::Explicit, - _ => return Err(err("raw frame schema declares no known label mode")), - }; - let decode = match schema.metadata().get(LABEL_DECODE) { - None => None, - Some(encoded) => { - let decode: LabelDecodeSpec = serde_json::from_str(encoded) - .map_err(|e| err(format!("malformed label decode description: {e}")))?; - decode.validate()?; - Some(decode) - } - }; - match (mode, &decode) { - (RawLabelMode::Key, None) => Err(err( - "key-encoded raw frame declares no label decode description", - )), - (RawLabelMode::Explicit, Some(_)) => Err(err( - "explicit raw frame declares a label decode description", - )), - (RawLabelMode::Key, Some(_)) | (RawLabelMode::Explicit, None) => Ok(decode), +/// binding; [`series_frame_schema`] and [`SeriesFrameDeclaration::parse`] enforce that both ways. +pub fn series_frame_label_decode(schema: &Schema) -> Result> { + Ok(SeriesFrameDeclaration::parse(schema)?.label_decode) +} + +/// The two declarations the encoder and decoder dispatch on. +/// +/// A frame's time mode follows from its content and its identity is the label mode, so those two +/// are all that has to travel next to the column layout. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct FrameShape { + content: FrameContent, + labels: FrameLabelMode, +} + +impl FrameShape { + fn of(schema: &SeriesSchema, labels: FrameLabelMode) -> Self { + let content = match schema.eval_grid { + Some(_) => FrameContent::Evaluated, + None => FrameContent::Raw, + }; + Self { content, labels } } } -/// Reconstructs the series schema a raw frame declares. +/// Bounded multi-row builder for finite series frame batches. /// -/// A raw frame names its payload fields by name only: every member of the `fields` struct carries -/// the same payload type ([`raw_series_frame_schema`]), and [`SeriesSchema`] holds nothing else per -/// field. The reconstruction is therefore exact rather than a projection — the declared names in -/// declared order, and no evaluation grid, which a raw frame cannot carry. -/// -/// The reserved layout, content, and time metadata are checked here together with the absence of an -/// evaluation grid, so a frame this revision cannot read is rejected instead of being silently -/// interpreted as the current layout. -pub fn raw_series_frame_series_schema(schema: &Schema) -> Result { - for (key, expected) in [(LAYOUT, "v1"), (CONTENT, "raw"), (TIME, "timestamps")] { - if schema.metadata().get(key).map(String::as_str) != Some(expected) { - return Err(err(format!("raw frame declares no known {key} value"))); - } - } - if schema - .metadata() - .keys() - .any(|key| key == GRID || key.starts_with("greptime.series.grid.")) - { - return Err(err("raw series schema cannot have an evaluation grid")); - } - let fields = schema - .fields() - .iter() - .find(|field| field.name() == "fields") - .ok_or_else(|| err("raw frame declares no fields column"))?; - let DataType::Struct(members) = fields.data_type() else { - return Err(err("raw frame fields column is not a struct")); - }; - if members.is_empty() { - return Err(err("raw series schema must have fields")); - } - Ok(SeriesSchema { - fields: members - .iter() - .map(|member| FieldSpec { - name: Arc::from(member.name().as_str()), - }) - .collect(), - eval_grid: None, - }) -} - -/// Bounded multi-row builder for finite raw frame batches. -/// -/// Rows accumulate until the row limit or the estimated byte limit is reached; the next -/// [`RawSeriesChunkCursor::append_to`] call then emits them as one batch. `max_bytes` is stored as -/// the effective group cap `min(requested, i32::MAX)`, and each row carries a measure that -/// dominates every cumulative Arrow offset domain, so any admitted group can be encoded. The -/// measure is an internal work estimate, not an exact IPC size and not a resident-memory cap: one -/// individually representable row that exceeds the requested cap is still accepted and emitted -/// alone, because that slice cannot be split into smaller legal rows. -pub struct RawSeriesFrameBuilder { +/// Rows accumulate until the row limit or the estimated byte limit is reached; the next cursor call +/// then emits them as one batch. `max_bytes` is stored as the effective group cap +/// `min(requested, i32::MAX)`, and each row carries a measure that dominates every cumulative Arrow +/// offset domain, so any admitted group can be encoded. The measure is an internal work estimate, not +/// an exact IPC size and not a resident-memory cap: one individually representable row that exceeds +/// the requested cap is still accepted and emitted alone, because that slice cannot be split into +/// smaller legal rows. +pub struct SeriesFrameBuilder { schema: SeriesSchema, frame_schema: SchemaRef, - mode: RawLabelMode, + shape: FrameShape, max_rows: usize, max_bytes: usize, - rows: Vec, + rows: Vec, bytes: usize, } -impl RawSeriesFrameBuilder { +impl SeriesFrameBuilder { /// Default maximum rows in one packed batch when no explicit bound is needed. pub const DEFAULT_MAX_ROWS: usize = 256; /// Default maximum estimated encoded bytes in one packed batch when no explicit bound is /// needed. pub const DEFAULT_MAX_BYTES: usize = 1024 * 1024; - /// Creates an empty builder for the given schema, label mode, and nonzero bounds. + /// Creates an empty builder for the given schema, identity declaration, and nonzero bounds. /// - /// The frame schema is built once here and reused for every emitted batch. + /// The frame schema is built once here and reused for every emitted batch. It declares + /// everything the rows carry except the stream's producer ordering, which only the streaming + /// layer knows and adds when it declares the batch. pub fn try_new( schema: &SeriesSchema, - mode: RawLabelMode, + labels: FrameLabelMode, + decode: Option<&LabelDecodeSpec>, max_rows: usize, max_bytes: usize, ) -> Result { if max_rows == 0 || max_bytes == 0 { - return invalid_series("raw series frame limits must be nonzero"); + return invalid_series("series frame limits must be nonzero"); } Ok(Self { schema: schema.clone(), - frame_schema: raw_series_frame_schema(schema, mode)?, - mode, + frame_schema: series_frame_schema(schema, labels, decode)?, + shape: FrameShape::of(schema, labels), max_rows, max_bytes: max_bytes.min(i32::MAX as usize), rows: Vec::new(), @@ -520,7 +799,7 @@ impl RawSeriesFrameBuilder { if self.rows.is_empty() { return Ok(None); } - let batch = encode_rows(&self.frame_schema, self.mode, &self.rows)?; + let batch = encode_rows(&self.frame_schema, self.shape, &self.rows)?; self.rows.clear(); self.bytes = 0; Ok(Some(batch)) @@ -536,7 +815,7 @@ impl RawSeriesFrameBuilder { /// An empty builder accepts any individually measured row; a group total that would overflow /// or exceed the cap flushes the buffered rows before this row is consumed, never dropping or /// splitting the row. - fn accepts(&self, row: &RawFrameRow) -> bool { + fn accepts(&self, row: &FrameRow) -> bool { self.rows.is_empty() || (self.bytes < self.max_bytes && self.rows.len() < self.max_rows @@ -547,29 +826,98 @@ impl RawSeriesFrameBuilder { } /// Buffers one individually measured row. - fn push(&mut self, row: RawFrameRow) -> Result<()> { + fn push(&mut self, row: FrameRow) -> Result<()> { self.bytes = add_checked(self.bytes, row.bytes)?; self.rows.push(row); Ok(()) } + + /// Checks that a cursor walks the same fields, identity, and time of what this builder encodes. + /// + /// The grid is compared by its specification rather than by whether one is present: a cursor + /// holding positions on one grid and a builder encoding them under another would write the same + /// numbers meaning different instants, and nothing downstream could tell. Two grids with equal + /// bounds are the same grid, so they compare equal however they were allocated. + fn matches(&self, schema: &SeriesSchema, shape: FrameShape) -> Result<()> { + let same_fields = schema.fields.len() == self.schema.fields.len() + && schema + .fields + .iter() + .zip(self.schema.fields.iter()) + .all(|(left, right)| left.name == right.name); + let same_grid = match (&schema.eval_grid, &self.schema.eval_grid) { + (None, None) => true, + (Some(left), Some(right)) => left.spec() == right.spec(), + _ => false, + }; + if shape != self.shape || !same_grid || !same_fields { + return invalid_series("series frame cursor and builder schema mismatch"); + } + Ok(()) + } } -/// One validated chunk walked segment by segment into a bounded builder. -pub struct RawSeriesChunkCursor { +/// One validated chunk walked into a bounded builder, one row at a time. +/// +/// The two sample shapes differ only in how a row's positions are addressed and where its identity +/// comes from, so one cursor serves both: a raw chunk is walked segment by segment over the +/// timestamps it carries, an evaluated chunk run by run over the grid positions it holds. Each row is +/// measured once and retried after a flush, so a flush error never consumes input. +pub struct SeriesFrameCursor { schema: SeriesSchema, - mode: RawLabelMode, + shape: FrameShape, + walk: ChunkWalk, + pending: Option, +} + +/// Where a cursor is in the chunk it walks. +enum ChunkWalk { + /// A raw chunk: one timestamp per row of samples. + Raw(RawWalk), + /// An evaluated chunk: absolute grid positions, carried as runs. + Evaluated(EvaluatedWalk), +} + +/// The walk state of one raw chunk. +/// +/// The chunk's fields are walked segment by segment, so a row never spans two segments of one field +/// and no per-row temporary vectors are allocated. +struct RawWalk { chunk: RawSeriesChunk, + /// The dense segment each field is being read from. + segments: Vec, + /// How much of each field's current segment has been consumed. + offsets: Vec, + /// How many timestamps have been consumed. + pos: usize, +} + +/// The walk state of one evaluated chunk. +/// +/// The runs are collected once, so a row is one intersection of a run with a sample segment and the +/// walk never re-derives the runs per row. A chunk that carries no position keeps a single `closing` +/// flag: it still owes one row, and that row is what tells its consumer the chain ended. +struct EvaluatedWalk { + series: Series, + is_last: bool, + /// The runs of carried rows, in absolute grid order and never empty. + runs: Vec, + /// The run being read. + run: usize, + /// How much of the current run has been consumed. + offset: usize, + /// The dense segment each field is being read from. segments: Vec, offsets: Vec, - pos: usize, - pending: Option, + /// Whether the one row a chunk without positions still owes is yet to be emitted. + closing: bool, } -impl RawSeriesChunkCursor { - /// Validates one public chunk's shape and label mode once, taking ownership of it. - pub fn try_new( +impl SeriesFrameCursor { + /// Validates one public raw chunk's shape and label mode once, taking ownership of it. + pub fn try_new_raw( schema: &SeriesSchema, - mode: RawLabelMode, + mode: FrameLabelMode, chunk: RawSeriesChunk, ) -> Result { let RawSeriesChunk { @@ -581,18 +929,64 @@ impl RawSeriesChunkCursor { } = chunk; let chunk = RawSeriesChunk::try_new(schema, primary_key, metric, time, fields, is_last)?; match (mode, &chunk.metric) { - (RawLabelMode::Key, None) | (RawLabelMode::Explicit, Some(_)) => {} + (FrameLabelMode::Key, None) | (FrameLabelMode::Explicit, Some(_)) => {} _ => return invalid_series("raw chunk label mode does not match frame mode"), } let segments = vec![0usize; chunk.fields.len()]; let offsets = vec![0usize; chunk.fields.len()]; Ok(Self { schema: schema.clone(), - mode, - chunk, - segments, - offsets, - pos: 0, + shape: FrameShape::of(schema, mode), + walk: ChunkWalk::Raw(RawWalk { + chunk, + segments, + offsets, + pos: 0, + }), + pending: None, + }) + } + + /// Validates one evaluated chunk against the frame's schema, grid, and finality rules. + /// + /// A chunk that carries no position is legal only when it closes its series: it is how a + /// consumer tells "nothing more here" from "no more input", and a chain of empty chunks that + /// close nothing would never make progress. It still occupies one frame row, because dropping it + /// would drop the finality it exists to carry. + pub fn try_new_evaluated(schema: &SeriesSchema, series: Series, is_last: bool) -> Result { + if schema.eval_grid.is_none() { + return invalid_series("an evaluated chunk needs a schema with an evaluation grid"); + } + let series = Series::try_new( + schema, + series.metric.clone(), + series.time.clone(), + series.fields.clone(), + )?; + if series.time.grid().is_none() { + return invalid_series("an evaluated chunk must be addressed on a grid"); + } + if series.time.is_empty() && !is_last { + return invalid_series("an evaluated chunk without positions must close its series"); + } + // The runs are the only runs this chunk carries, so collecting them once is what keeps the + // walk from re-deriving them per row; a maximal run is merged by the axis itself. + let runs: Vec = series.time.runs().collect(); + let segments = vec![0usize; series.fields.len()]; + let offsets = vec![0usize; series.fields.len()]; + Ok(Self { + schema: schema.clone(), + shape: FrameShape::of(schema, FrameLabelMode::Explicit), + walk: ChunkWalk::Evaluated(EvaluatedWalk { + closing: runs.is_empty(), + series, + is_last, + runs, + run: 0, + offset: 0, + segments, + offsets, + }), pending: None, }) } @@ -602,11 +996,8 @@ impl RawSeriesChunkCursor { /// `None` means the cursor is exhausted; calling again returns `None` without touching the /// builder. The single pending row is measured once by [`Self::next_row`] and retried after the /// buffered rows are emitted. - pub fn append_to( - &mut self, - builder: &mut RawSeriesFrameBuilder, - ) -> Result> { - self.compatible(builder)?; + pub fn append_to(&mut self, builder: &mut SeriesFrameBuilder) -> Result> { + builder.matches(&self.schema, self.shape)?; loop { let row = match self.pending.take() { Some(row) => row, @@ -623,7 +1014,7 @@ impl RawSeriesChunkCursor { self.pending = Some(row); let batch = builder .flush()? - .ok_or_else(|| err("raw frame builder rejected a row without buffered rows"))?; + .ok_or_else(|| err("frame builder rejected a row without buffered rows"))?; return Ok(Some(batch)); } builder.push(row)?; @@ -635,68 +1026,157 @@ impl RawSeriesChunkCursor { } } - /// The cursor and builder must describe the same fields and label mode. - fn compatible(&self, builder: &RawSeriesFrameBuilder) -> Result<()> { - let same_fields = self.schema.fields.len() == builder.schema.fields.len() - && self - .schema - .fields - .iter() - .zip(builder.schema.fields.iter()) - .all(|(left, right)| left.name == right.name); - if self.mode != builder.mode - || self.schema.eval_grid.is_some() != builder.schema.eval_grid.is_some() - || !same_fields - { - return invalid_series("raw frame cursor and builder schema mismatch"); + /// Builds and measures the next row, or `None` when the chunk is exhausted. + fn next_row(&mut self) -> Result> { + match &mut self.walk { + ChunkWalk::Raw(walk) => walk.next_row(), + ChunkWalk::Evaluated(walk) => walk.next_row(), } - Ok(()) } +} - /// Builds and measures the next union-of-segment row, or `None` when the chunk is exhausted. +impl RawWalk { + /// Builds the next row, or `None` when the chunk is exhausted. /// - /// The chunk fields are walked three times so no per-row temporary vectors are allocated; the - /// offsets advance only after the row measure succeeded. - fn next_row(&mut self) -> Result> { - if self.pos >= self.chunk.time.len() { + /// The chunk fields are walked twice: once to find how many positions every field can cover, and + /// once to slice them. A row does collect one slice per field, so the saving here is not + /// allocation-free rows but no row spanning two segments of one field, which is what keeps the + /// slice arithmetic per row a single offset. The offsets advance only after the row measure + /// succeeded. + fn next_row(&mut self) -> Result> { + let Self { + chunk, + segments, + offsets, + pos, + } = self; + if *pos >= chunk.time.len() { return Ok(None); } - let time_axis = self.chunk.time.as_buffer(); - let mut take = time_axis.len() - self.pos; - for (index, field) in self.chunk.fields.iter().enumerate() { - let part = segment(field, self.segments[index])?; - take = take.min(part.len().saturating_sub(self.offsets[index])); + let time_axis = chunk.time.as_buffer(); + let mut take = time_axis.len() - *pos; + for (index, field) in chunk.fields.iter().enumerate() { + let part = segment(field, segments[index])?; + take = take.min(part.len().saturating_sub(offsets[index])); } if take == 0 { return invalid_series("empty sample segment"); } - let mut samples = Vec::with_capacity(self.chunk.fields.len()); - for (index, field) in self.chunk.fields.iter().enumerate() { - let part = segment(field, self.segments[index])?; - samples.push(part.slice(self.offsets[index], take)?); + let mut samples = Vec::with_capacity(chunk.fields.len()); + for (index, field) in chunk.fields.iter().enumerate() { + let part = segment(field, segments[index])?; + samples.push(part.slice(offsets[index], take)?); } - let mut row = RawFrameRow { - primary_key: self.chunk.primary_key.clone(), - metric: self.chunk.metric.clone(), - is_last: self.chunk.is_last && self.pos + take == self.chunk.time.len(), - times: time_axis.slice(self.pos, take), + let mut row = FrameRow { + key: Some(chunk.primary_key.clone()), + metric: chunk.metric.clone(), + is_last: chunk.is_last && *pos + take == chunk.time.len(), + time: RowTime::Timestamps(time_axis.slice(*pos, take)), samples, bytes: 0, }; row.bytes = measure_row(&row)?; - self.pos += take; - for (index, field) in self.chunk.fields.iter().enumerate() { - let length = segment(field, self.segments[index])?.len(); - self.offsets[index] += take; - if self.offsets[index] == length { - self.segments[index] += 1; - self.offsets[index] = 0; + *pos += take; + for (index, field) in chunk.fields.iter().enumerate() { + let length = segment(field, segments[index])?.len(); + offsets[index] += take; + if offsets[index] == length { + segments[index] += 1; + offsets[index] = 0; } } Ok(Some(row)) } } +impl EvaluatedWalk { + /// Builds the next row, or `None` when the chunk is exhausted. + /// + /// A row covers one `(run of carried positions, sample segment)` intersection, which is why its + /// positions are always one span: the holes between runs are not rows, so nothing is ever filled + /// in for a grid position the chunk does not hold. The `is_last` a producer declared is carried + /// by the chunk's final row only, so a consumer that sees it knows the chain ended exactly there. + fn next_row(&mut self) -> Result> { + let Self { + series, + is_last, + runs, + run, + offset, + segments, + offsets, + closing, + } = self; + if *closing { + *closing = false; + // A chunk that carries no position still owes its consumer one row: the row that says the + // chain ends here. It holds no grid position and no sample, and each field is kept as the + // producer built it - an empty float stays float, an empty histogram stays histogram - so a + // decoder sees an empty closing chunk of the same payload kind rather than a missing one or + // one silently converted. + let mut samples = Vec::with_capacity(series.fields.len()); + for field in series.fields.iter() { + samples.push(field.clone()); + } + let mut row = FrameRow { + key: None, + metric: Some(series.metric.clone()), + is_last: *is_last, + time: RowTime::Spans(Vec::new()), + samples, + bytes: 0, + }; + row.bytes = measure_row(&row)?; + return Ok(Some(row)); + } + let Some(current) = runs.get(*run) else { + return Ok(None); + }; + let remaining = current.len - *offset; + let mut take = remaining; + for (index, field) in series.fields.iter().enumerate() { + let part = segment(field, segments[index])?; + take = take.min(part.len().saturating_sub(offsets[index])); + } + if take == 0 { + return invalid_series("empty sample segment"); + } + let row_offset = current.row + *offset; + let mut samples = Vec::with_capacity(series.fields.len()); + for (index, field) in series.fields.iter().enumerate() { + let part = segment(field, segments[index])?; + samples.push(part.slice(offsets[index], take)?); + } + let mut row = FrameRow { + key: None, + metric: Some(series.metric.clone()), + is_last: *is_last && *run + 1 == runs.len() && take == remaining, + time: RowTime::Spans(vec![GridSpan::new(current.position + *offset, take)]), + samples, + bytes: 0, + }; + row.bytes = measure_row(&row)?; + *offset += take; + if *offset == current.len { + *run += 1; + *offset = 0; + } + for (index, field) in series.fields.iter().enumerate() { + let length = segment(field, segments[index])?.len(); + offsets[index] += take; + if offsets[index] == length { + segments[index] += 1; + offsets[index] = 0; + } + } + debug_assert!( + row_offset + take <= series.time.len(), + "an evaluated row exceeds the axis it walks" + ); + Ok(Some(row)) + } +} + /// The next dense segment of one field, or an error when the walk left the segment list. fn segment(field: &Samples, index: usize) -> Result<&Samples> { field @@ -708,19 +1188,21 @@ fn segment(field: &Samples, index: usize) -> Result<&Samples> { /// Packs one raw chunk, retaining the `Vec` compatibility signature. /// /// The chunk packs into bounded multi-row batches; callers that need streaming behavior should -/// drive [`RawSeriesChunkCursor`] and [`RawSeriesFrameBuilder`] directly instead of collecting. +/// drive [`SeriesFrameCursor`] and [`SeriesFrameBuilder`] directly instead of collecting. pub fn pack_raw_series_chunk( schema: &SeriesSchema, - mode: RawLabelMode, + mode: FrameLabelMode, + decode: Option<&LabelDecodeSpec>, chunk: &RawSeriesChunk, ) -> Result> { - let mut builder = RawSeriesFrameBuilder::try_new( + let mut builder = SeriesFrameBuilder::try_new( schema, mode, - RawSeriesFrameBuilder::DEFAULT_MAX_ROWS, - RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, + decode, + SeriesFrameBuilder::DEFAULT_MAX_ROWS, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, )?; - let mut cursor = RawSeriesChunkCursor::try_new(schema, mode, chunk.clone())?; + let mut cursor = SeriesFrameCursor::try_new_raw(schema, mode, chunk.clone())?; let mut batches = Vec::new(); while let Some(batch) = cursor.append_to(&mut builder)? { batches.push(batch); @@ -731,17 +1213,53 @@ pub fn pack_raw_series_chunk( Ok(batches) } -/// One pending output row: a complete key with one dense sample slice per field and its checked -/// encoded-size measure. -struct RawFrameRow { - primary_key: Buffer, +/// One pending output row: one chunk's identity, finality, time slice, and one dense sample slice per +/// field, with its checked encoded-size measure. +/// +/// This is the codec's own row, not a third runtime chunk type: `RawSeriesChunk` and `Series` are +/// what a producer and a consumer see, and this is what the encoder writes between them. +struct FrameRow { + /// The encoded storage key. A raw row always carries the key it was read under, a key-encoded + /// frame names its series only by it, and an evaluated row has none. + key: Option, + /// Complete labels, present exactly for explicit identity. metric: Option, + /// Whether this row closes the chunk chain it belongs to. is_last: bool, - times: ScalarBuffer, + /// The positions this row's samples are addressed at. + time: RowTime, samples: Vec, bytes: usize, } +/// How one row addresses the time of its samples. +enum RowTime { + /// The timestamps of this row's samples, strictly increasing within the row. + Timestamps(ScalarBuffer), + /// Runs of absolute grid positions, ordered, non-overlapping, and never zero length. + Spans(Vec), +} + +impl RowTime { + /// How many grid positions or timestamps this row's samples cover. + /// + /// The sum is checked because this is the first place a decoded row's positions are counted, and + /// a frame can name spans whose lengths overflow `usize` long before any span is validated + /// against the grid it claims to sit on. Adding them up unchecked would panic on a hostile frame + /// in a debug build and wrap in a release build, both before the grid check that should have + /// refused it. + fn len(&self) -> Result { + match self { + Self::Timestamps(timestamps) => Ok(timestamps.len()), + Self::Spans(spans) => spans.iter().try_fold(0usize, |total, span| { + total + .checked_add(span.len()) + .ok_or_else(|| err("grid span length overflow")) + }), + } + } +} + /// Per-row encoded-size measure, computed once per row when the cursor produces it. /// /// The measure dominates every cumulative offset domain the encoder writes: binary/UTF-8 child @@ -749,11 +1267,19 @@ struct RawFrameRow { /// A group total of `min(max_bytes, i32::MAX)` therefore guarantees that all Arrow offsets and /// lengths stay representable, without tracking a separate offset counter. This is a bound used /// for batching work, not the exact encoded size. -fn measure_row(row: &RawFrameRow) -> Result { - checked_offset(row.primary_key.len())?; - checked_offset(row.times.len())?; - let mut bytes = add_checked(ROW_CHARGE, row.primary_key.len())?; - bytes = add_checked(bytes, mul_checked(row.times.len(), 8)?)?; +fn measure_row(row: &FrameRow) -> Result { + let key_len = row.key.as_ref().map(Buffer::len).unwrap_or(0); + checked_offset(key_len)?; + checked_offset(row.time.len()?)?; + let mut bytes = add_checked(ROW_CHARGE, key_len)?; + bytes = match &row.time { + RowTime::Timestamps(timestamps) => add_checked(bytes, mul_checked(timestamps.len(), 8)?)?, + // Two `u64` per span, plus the list's own offsets and validity. + RowTime::Spans(spans) => add_checked( + add_checked(bytes, mul_checked(spans.len(), SPAN_CHARGE)?)?, + mul_checked(spans.len(), 2)?, + )?, + }; for samples in &row.samples { bytes = add_checked(bytes, FIELD_CHARGE)?; match samples @@ -846,11 +1372,7 @@ fn histogram_charge(values: &HistogramSamples) -> Result { /// complete table, including unreferenced entries; many segments referencing one large table can /// therefore repeat that table in the encoded batch. A single constituent child array is reused /// by clone (sharing its buffers); multiple constituents are concatenated, which copies. -fn encode_rows( - schema: &SchemaRef, - mode: RawLabelMode, - rows: &[RawFrameRow], -) -> Result { +fn encode_rows(schema: &SchemaRef, shape: FrameShape, rows: &[FrameRow]) -> Result { let count = rows.len(); let DataType::Struct(time_fields) = schema.field(2).data_type() else { return invalid_series("time schema mismatch"); @@ -861,17 +1383,72 @@ fn encode_rows( let DataType::Struct(field_types) = schema.field(3).data_type() else { return invalid_series("fields schema mismatch"); }; - let mut time_lengths = Vec::with_capacity(count); - let mut time_parts: Vec = Vec::with_capacity(count); - for row in rows { - time_lengths.push(row.times.len()); - time_parts.push(Arc::new(Int64Array::new(row.times.clone(), None))); - } - let time_child = materialize_parts(&time_parts, || { - Ok(Arc::new(Int64Array::new_null(0)) as ArrayRef) - })?; - let times = list_array(&time_lengths, time_child, time_item.as_ref().clone(), None)?; - let time = struct_array(time_fields.clone(), vec![Arc::new(times)])?; + let time = match shape.content { + FrameContent::Raw => { + let mut time_lengths = Vec::with_capacity(count); + let mut time_parts: Vec = Vec::with_capacity(count); + for row in rows { + let RowTime::Timestamps(timestamps) = &row.time else { + return invalid_series("a raw frame row carries grid positions"); + }; + time_lengths.push(timestamps.len()); + time_parts.push(Arc::new(Int64Array::new(timestamps.clone(), None))); + } + let time_child = materialize_parts(&time_parts, || { + Ok(Arc::new(Int64Array::new_null(0)) as ArrayRef) + })?; + let times = list_array(&time_lengths, time_child, time_item.as_ref().clone(), None)?; + struct_array(time_fields.clone(), vec![Arc::new(times)])? + } + FrameContent::Evaluated => { + let DataType::Struct(span_fields) = time_item.data_type() else { + return invalid_series("grid span schema mismatch"); + }; + let mut span_lengths = Vec::with_capacity(count); + let mut starts = Vec::new(); + let mut lengths = Vec::new(); + for row in rows { + let RowTime::Spans(spans) = &row.time else { + return invalid_series("an evaluated frame row carries timestamps"); + }; + span_lengths.push(spans.len()); + for span in spans { + starts.push(u64::try_from(span.start()).map_err(|_| { + err("a grid position does not fit the frame's unsigned span") + })?); + lengths.push(u64::try_from(span.len()).map_err(|_| { + err("a grid span length does not fit the frame's unsigned span") + })?); + } + } + // A batch with no span at all still needs a typed child, so the empty case is spelled + // out instead of being left to whatever `concat` does with nothing. + let span_struct = if starts.is_empty() { + struct_array( + span_fields.clone(), + vec![ + Arc::new(UInt64Array::new_null(0)), + Arc::new(UInt64Array::new_null(0)), + ], + )? + } else { + struct_array( + span_fields.clone(), + vec![ + Arc::new(UInt64Array::from(starts)), + Arc::new(UInt64Array::from(lengths)), + ], + )? + }; + let spans = list_array( + &span_lengths, + Arc::new(span_struct), + time_item.as_ref().clone(), + None, + )?; + struct_array(time_fields.clone(), vec![Arc::new(spans)])? + } + }; let mut empty_table: Option = None; let mut field_arrays = Vec::with_capacity(field_types.len()); @@ -990,14 +1567,14 @@ fn encode_rows( let fields = struct_array(field_types.clone(), field_arrays)?; let mut columns: Vec = vec![ - Arc::new(binary_array(rows)?), + Arc::new(binary_array(rows, shape)?), Arc::new(BooleanArray::from( rows.iter().map(|row| row.is_last).collect::>(), )), Arc::new(time), Arc::new(fields), ]; - if mode == RawLabelMode::Explicit { + if shape.labels == FrameLabelMode::Explicit { columns.push(Arc::new( rows.iter() .map(|row| row.metric.as_ref().and_then(SeriesLabels::name)) @@ -1045,28 +1622,55 @@ fn encode_rows( )) } -/// Row-level binary keys; keys are copied exactly like the previous single-row encoder. -fn binary_array(rows: &[RawFrameRow]) -> Result { +/// Row-level binary keys. +/// +/// A raw row always carries the key its chunk was read under, so the column is complete and +/// non-null for a key frame and for a raw explicit frame alike; only an evaluated row has none, and +/// it is a null entry rather than an empty key. A null and `b""` stay distinct, because the second is +/// a key a producer could hold and the first is the absence of one. +fn binary_array(rows: &[FrameRow], shape: FrameShape) -> Result { let mut offsets = Vec::with_capacity(rows.len() + 1); let mut total = 0usize; + let mut valid = Vec::with_capacity(rows.len()); offsets.push(0); for row in rows { - total = add_checked(total, row.primary_key.len())?; + match (&row.key, shape) { + (Some(key), _) => { + total = add_checked(total, key.len())?; + valid.push(true); + } + (None, shape) if shape.content == FrameContent::Raw => { + // `RawSeriesChunk` still carries the key it was read under, and an explicit frame + // only adds the decoded labels beside it. Dropping the key would lose the identity + // a consumer of the raw chunk needs. + return invalid_series("a raw frame row carries no key"); + } + (None, _) => valid.push(false), + } offsets.push(checked_offset(total)?); } let mut values = Vec::with_capacity(total); for row in rows { - values.extend_from_slice(row.primary_key.as_slice()); + if let Some(key) = &row.key { + values.extend_from_slice(key.as_slice()); + } } + let nulls = if valid.iter().all(|valid| *valid) { + None + } else { + validity(&valid) + }; arrow(BinaryArray::try_new( OffsetBuffer::new(ScalarBuffer::from(offsets)), Buffer::from_vec(values), - None, + nulls, )) } /// Per-row charge for key/time/nullable overhead shared by every row. const ROW_CHARGE: usize = 64; +/// Per-span charge for the two 64-bit coordinates one grid span carries. +const SPAN_CHARGE: usize = 16; /// Per-field charge for the payload struct, nested list and validity overhead. const FIELD_CHARGE: usize = 64; /// Conservative per-table-row charge: the fixed fourteen-column histogram schema has seven @@ -1149,172 +1753,299 @@ fn mul_checked(len: usize, width: usize) -> Result { } /// Reuses one prepared frame schema while decoding all batches of a stream. -pub struct RawSeriesFrameDecoder { +/// +/// The decoder is built from the schema a consumer expects and then checks every batch against it: +/// the reserved declarations are compared as text, the column layout as Arrow fields, and every row +/// on its own. A batch that disagrees anywhere is an error rather than a partial decode, because a +/// receiver has no way to tell which half of a contradictory frame is the true one. +pub struct SeriesFrameDecoder { schema: SeriesSchema, - mode: RawLabelMode, + shape: FrameShape, expected: SchemaRef, } -impl RawSeriesFrameDecoder { - /// Builds the expected frame schema once from the series schema and label mode. - pub fn try_new(schema: &SeriesSchema, mode: RawLabelMode) -> Result { +impl SeriesFrameDecoder { + /// Builds the expected frame schema once from the series schema and identity declaration. + pub fn try_new( + schema: &SeriesSchema, + labels: FrameLabelMode, + decode: Option<&LabelDecodeSpec>, + ) -> Result { Ok(Self { schema: schema.clone(), - mode, - expected: raw_series_frame_schema(schema, mode)?, + shape: FrameShape::of(schema, labels), + expected: series_frame_schema(schema, labels, decode)?, }) } + /// Decodes all rows of one batch as raw chunks. + pub fn decode_raw(&self, batch: &RecordBatch) -> Result> { + if self.shape.content != FrameContent::Raw { + return invalid_series("a raw frame decoder cannot read an evaluated frame"); + } + let rows = self.decode_rows(batch)?; + let mut out = Vec::with_capacity(rows.len()); + for row in rows { + let RowTime::Timestamps(timestamps) = row.time else { + return invalid_series("a raw frame row carries grid positions"); + }; + let key = row + .key + .ok_or_else(|| err("a raw frame row carries no key"))?; + let time = RawTimestamps::try_new(timestamps)?; + let fields = SeriesFields::try_from_vec(row.samples)?; + out.push(RawSeriesChunk::try_new( + &self.schema, + key, + row.metric, + time, + fields, + row.is_last, + )?); + } + Ok(out) + } + + /// Decodes all rows of one batch as evaluated series and their finality. + /// + /// The pair is what a compute consumer needs and nothing more: the values on their absolute grid + /// positions, and whether this chunk closed the chain. A row that carries no position is kept, so + /// a closing chunk with no value still reaches the consumer that has to see it. + pub fn decode_evaluated(&self, batch: &RecordBatch) -> Result> { + if self.shape.content != FrameContent::Evaluated { + return invalid_series("an evaluated frame decoder cannot read a raw frame"); + } + let grid = self + .schema + .eval_grid + .clone() + .ok_or_else(|| err("an evaluated frame needs an evaluation grid"))?; + let rows = self.decode_rows(batch)?; + let mut out = Vec::with_capacity(rows.len()); + for row in rows { + let RowTime::Spans(spans) = row.time else { + return invalid_series("an evaluated frame row carries timestamps"); + }; + let metric = row + .metric + .ok_or_else(|| err("an evaluated frame row carries no labels"))?; + let time = TimeAxis::from_grid_spans(grid.clone(), spans)?; + // A row that covers no position is the closing chunk's shape and nothing else. The + // streaming validator refuses one that closes nothing, but this decoder is public and a + // caller driving it directly must not be handed a chunk it could never have produced + // either: an empty chunk that opens a chain carries no value and no finality, so no + // consumer could act on it. + if time.is_empty() && !row.is_last { + return invalid_series( + "an evaluated chunk without positions must close its series", + ); + } + let fields = SeriesFields::try_from_vec(row.samples)?; + out.push(( + Series::try_new(&self.schema, metric, time, fields)?, + row.is_last, + )); + } + Ok(out) + } + /// Decodes all rows of one batch, checking its reserved metadata, declared fields, and rows. - pub fn decode(&self, batch: &RecordBatch) -> Result> { - decode_raw_series_frame(&self.schema, self.mode, &self.expected, batch) + fn decode_rows(&self, batch: &RecordBatch) -> Result> { + let actual = batch.schema(); + validate_ordering(actual.as_ref(), self.shape.content)?; + for key in [LAYOUT, CONTENT, LABELS, TIME, GRID, LABEL_DECODE] { + if actual.metadata().get(key) != self.expected.metadata().get(key) { + return invalid_series("frame metadata mismatch"); + } + } + // The same reservation rule the declaration parser applies, because a frame's protocol + // legality must not depend on which entry point reads it: a reserved key nobody checks would + // travel as dead metadata here while being refused a layer up. + for key in actual.metadata().keys() { + if key.starts_with("greptime.series.") && !RESERVED_KEYS.contains(&key.as_str()) { + return invalid_series("frame declares unknown reserved metadata"); + } + } + if actual.fields() != self.expected.fields() { + return invalid_series("frame schema mismatch"); + } + let cols = batch.columns(); + let keys = down::(&cols[0], "read_key")?; + let lasts = down::(&cols[1], "is_last")?; + let time_struct = down::(&cols[2], "time")?; + let fs = down::(&cols[3], "fields")?; + let (names, labels) = if self.shape.labels == FrameLabelMode::Explicit { + ( + Some(down::(&cols[4], "name")?), + Some(down::(&cols[5], "labels")?), + ) + } else { + (None, None) + }; + let mut out = Vec::with_capacity(batch.num_rows()); + for row in 0..batch.num_rows() { + if lasts.is_null(row) || time_struct.is_null(row) || fs.is_null(row) { + return invalid_series("null required frame parent"); + } + match self.shape.labels { + FrameLabelMode::Key => { + if keys.is_null(row) { + return invalid_series("null required frame parent"); + } + } + // An explicit row names its series by label; the key column is the raw chunk's own + // contract, and an evaluated row has none at all. + FrameLabelMode::Explicit => { + if self.shape.content == FrameContent::Raw && keys.is_null(row) { + return invalid_series("a raw explicit frame row carries no key"); + } + if self.shape.content == FrameContent::Evaluated && !keys.is_null(row) { + return invalid_series("an evaluated frame row carries a storage key"); + } + } + } + let time = match self.shape.content { + FrameContent::Raw => { + let times = down::(time_struct.column(0), "timestamps")?; + if times.is_null(row) { + return invalid_series("null required raw frame parent"); + } + let values = times.value(row); + let tv = down::(&values, "timestamp values")?; + if tv.null_count() != 0 { + return invalid_series("null timestamp"); + } + RowTime::Timestamps(tv.values().clone()) + } + FrameContent::Evaluated => { + let typed = down::(time_struct.column(0), "spans")?; + if typed.is_null(row) { + return invalid_series("null required evaluated frame parent"); + } + let values = typed.value(row); + let span_struct = down::(&values, "spans")?; + if span_struct.null_count() != 0 { + return invalid_series("null grid span"); + } + let starts = down::(span_struct.column(0), "span starts")?; + let lengths = down::(span_struct.column(1), "span lengths")?; + if starts.null_count() != 0 || lengths.null_count() != 0 { + return invalid_series("null grid span bound"); + } + let mut spans = Vec::with_capacity(starts.len()); + for index in 0..starts.len() { + let start = usize::try_from(starts.value(index)) + .map_err(|_| err("grid span start exceeds the supported range"))?; + let len = usize::try_from(lengths.value(index)) + .map_err(|_| err("grid span length exceeds the supported range"))?; + if len == 0 { + return invalid_series("grid spans must not be empty"); + } + spans.push(GridSpan::new(start, len)); + } + RowTime::Spans(spans) + } + }; + let positions = time.len()?; + let metric = match (names, labels) { + (None, None) => None, + (Some(ns), Some(ls)) => { + if ls.is_null(row) { + return invalid_series("null labels list"); + } + let label_values = ls.value(row); + let label_struct = down::(&label_values, "labels")?; + if label_struct.null_count() != 0 { + return invalid_series("null label struct element"); + } + let ln = down::(label_struct.column(0), "label names")?; + let lv = down::(label_struct.column(1), "label values")?; + if ln.null_count() != 0 || lv.null_count() != 0 { + return invalid_series("null label element"); + } + let tags = (0..ln.len()) + .map(|i| Label { + name: Arc::from(ln.value(i)), + value: Arc::from(lv.value(i)), + }) + .collect(); + Some(SeriesLabels::try_new( + (!ns.is_null(row)).then(|| Arc::from(ns.value(row))), + tags, + )?) + } + _ => return invalid_series("incomplete explicit label columns"), + }; + let mut decoded = Vec::with_capacity(fs.num_columns()); + for col in fs.columns() { + let payload = down::(col, "field payload")?; + if payload.is_null(row) { + return invalid_series("null field payload"); + } + let floats_list = down::(payload.column(0), "float values")?; + let hist = down::(payload.column(1), "histogram payload")?; + let has_float = !floats_list.is_null(row); + let has_hist = !hist.is_null(row); + if !has_float && !has_hist { + return invalid_series("field has neither float nor histogram payload"); + } + let floats = if has_float { + down::(&floats_list.value(row), "float values")?.clone() + } else { + Float64Array::new_null(0) + }; + if has_float && floats.len() != positions { + return invalid_series("float payload length mismatch"); + } + let samples = if !has_hist { + Samples::from_float(floats) + } else { + let inds = down::(hist.column(0), "histogram indices")?; + let tables = down::(hist.column(1), "histogram table")?; + if inds.is_null(row) || tables.is_null(row) { + return invalid_series("null histogram indices or table"); + } + let indices = + down::(&inds.value(row), "histogram indices")?.clone(); + if indices.len() != positions { + return invalid_series("histogram index length mismatch"); + } + let table = down::(&tables.value(row), "histogram table")?.clone(); + let batch = Arc::new(HistogramBatch::try_from_arrow(table)?); + if has_float { + Samples::try_from_mixed(floats, HistogramSamples::try_new(indices, batch)?)? + } else { + Samples::try_from_histogram(indices, batch)? + } + }; + decoded.push(samples); + } + let key = if keys.is_null(row) { + None + } else { + Some(binary_slice(keys, row)?) + }; + out.push(FrameRow { + key, + metric, + is_last: lasts.value(row), + time, + samples: decoded, + bytes: 0, + }); + } + Ok(out) } } -/// Decodes all rows of a raw frame; any malformed row makes the entire call fail. +/// Decodes all rows of one raw frame; any malformed row makes the entire call fail. pub fn unpack_raw_series_frame( schema: &SeriesSchema, - mode: RawLabelMode, + mode: FrameLabelMode, + decode: Option<&LabelDecodeSpec>, batch: &RecordBatch, ) -> Result> { - RawSeriesFrameDecoder::try_new(schema, mode)?.decode(batch) -} - -fn decode_raw_series_frame( - schema: &SeriesSchema, - mode: RawLabelMode, - expected: &SchemaRef, - batch: &RecordBatch, -) -> Result> { - let actual = batch.schema(); - for key in [LAYOUT, CONTENT, LABELS, TIME] { - if actual.metadata().get(key) != expected.metadata().get(key) { - return invalid_series("raw frame metadata mismatch"); - } - } - if actual - .metadata() - .keys() - .any(|key| key == GRID || key.starts_with("greptime.series.grid.")) - || actual.fields() != expected.fields() - { - return invalid_series("raw frame schema mismatch"); - } - let cols = batch.columns(); - let keys = down::(&cols[0], "read_key")?; - let lasts = down::(&cols[1], "is_last")?; - let time_struct = down::(&cols[2], "time")?; - let times = down::(time_struct.column(0), "timestamps")?; - let fs = down::(&cols[3], "fields")?; - let (names, labels) = if mode == RawLabelMode::Explicit { - ( - Some(down::(&cols[4], "name")?), - Some(down::(&cols[5], "labels")?), - ) - } else { - (None, None) - }; - let mut out = Vec::with_capacity(batch.num_rows()); - for row in 0..batch.num_rows() { - if keys.is_null(row) - || lasts.is_null(row) - || time_struct.is_null(row) - || times.is_null(row) - || fs.is_null(row) - { - return invalid_series("null required raw frame parent"); - } - let timestamp_values = times.value(row); - let tv = down::(×tamp_values, "timestamp values")?; - if tv.null_count() != 0 { - return invalid_series("null timestamp"); - } - let time = tv.values().clone(); - let metric = match (names, labels) { - (None, None) => None, - (Some(ns), Some(ls)) => { - if ls.is_null(row) { - return invalid_series("null labels list"); - } - let label_values = ls.value(row); - let label_struct = down::(&label_values, "labels")?; - if label_struct.null_count() != 0 { - return invalid_series("null label struct element"); - } - let ln = down::(label_struct.column(0), "label names")?; - let lv = down::(label_struct.column(1), "label values")?; - if ln.null_count() != 0 || lv.null_count() != 0 { - return invalid_series("null label element"); - } - let tags = (0..ln.len()) - .map(|i| Label { - name: Arc::from(ln.value(i)), - value: Arc::from(lv.value(i)), - }) - .collect(); - Some(SeriesLabels::try_new( - (!ns.is_null(row)).then(|| Arc::from(ns.value(row))), - tags, - )?) - } - _ => return invalid_series("incomplete explicit label columns"), - }; - let mut decoded = Vec::with_capacity(fs.num_columns()); - for col in fs.columns() { - let payload = down::(col, "field payload")?; - if payload.is_null(row) { - return invalid_series("null field payload"); - } - let floats_list = down::(payload.column(0), "float values")?; - let hist = down::(payload.column(1), "histogram payload")?; - let has_float = !floats_list.is_null(row); - let has_hist = !hist.is_null(row); - if !has_float && !has_hist { - return invalid_series("field has neither float nor histogram payload"); - } - let floats = if has_float { - down::(&floats_list.value(row), "float values")?.clone() - } else { - Float64Array::new_null(0) - }; - if has_float && floats.len() != time.len() { - return invalid_series("float payload length mismatch"); - } - let samples = if !has_hist { - Samples::from_float(floats) - } else { - let inds = down::(hist.column(0), "histogram indices")?; - let tables = down::(hist.column(1), "histogram table")?; - if inds.is_null(row) || tables.is_null(row) { - return invalid_series("null histogram indices or table"); - } - let indices = down::(&inds.value(row), "histogram indices")?.clone(); - if indices.len() != time.len() { - return invalid_series("histogram index length mismatch"); - } - let table = down::(&tables.value(row), "histogram table")?.clone(); - let batch = Arc::new(HistogramBatch::try_from_arrow(table)?); - if has_float { - Samples::try_from_mixed(floats, HistogramSamples::try_new(indices, batch)?)? - } else { - Samples::try_from_histogram(indices, batch)? - } - }; - decoded.push(samples); - } - let fields = SeriesFields::try_from_vec(decoded)?; - let key = binary_slice(keys, row)?; - let raw_time = RawTimestamps::try_new(time)?; - out.push(RawSeriesChunk::try_new( - schema, - key, - metric, - raw_time, - fields, - lasts.value(row), - )?); - } - Ok(out) + SeriesFrameDecoder::try_new(schema, mode, decode)?.decode_raw(batch) } fn binary_slice(array: &BinaryArray, row: usize) -> Result { diff --git a/src/datatypes/src/series/frame/tests.rs b/src/datatypes/src/series/frame/tests.rs index 5356ebb84dc..aa748a528f4 100644 --- a/src/datatypes/src/series/frame/tests.rs +++ b/src/datatypes/src/series/frame/tests.rs @@ -115,7 +115,7 @@ fn histogram_batch_with_sum(sum: f64) -> Arc DataType { - raw_series_frame_schema(schema, RawLabelMode::Key) + series_frame_schema(schema, FrameLabelMode::Key, Some(&dense_description())) .unwrap() .field(3) .data_type() @@ -253,13 +253,16 @@ fn assert_float_bits(actual: &Samples, expected: &[Option]) { /// single-chunk wrapper: every batch the walk produces, then the final flush. fn pack_chunk_with_budget( schema: &SeriesSchema, - mode: RawLabelMode, + mode: FrameLabelMode, chunk: &RawSeriesChunk, max_rows: usize, max_bytes: usize, ) -> crate::error::Result> { - let mut builder = RawSeriesFrameBuilder::try_new(schema, mode, max_rows, max_bytes)?; - let mut cursor = RawSeriesChunkCursor::try_new(schema, mode, chunk.clone())?; + // Only a key frame declares a label decode description; explicit frames carry their labels. + let decode = (mode == FrameLabelMode::Key).then(dense_description); + let mut builder = + SeriesFrameBuilder::try_new(schema, mode, decode.as_ref(), max_rows, max_bytes)?; + let mut cursor = SeriesFrameCursor::try_new_raw(schema, mode, chunk.clone())?; let mut batches = Vec::new(); while let Some(batch) = cursor.append_to(&mut builder)? { batches.push(batch); @@ -273,12 +276,12 @@ fn pack_chunk_with_budget( /// Advances one owned chunk through an existing builder. A partially filled buffer stays buffered /// instead of becoming a batch of its own, which is what makes cross-chunk grouping observable. fn append_chunk( - builder: &mut RawSeriesFrameBuilder, + builder: &mut SeriesFrameBuilder, schema: &SeriesSchema, - mode: RawLabelMode, + mode: FrameLabelMode, chunk: &RawSeriesChunk, ) -> crate::error::Result> { - let mut cursor = RawSeriesChunkCursor::try_new(schema, mode, chunk.clone())?; + let mut cursor = SeriesFrameCursor::try_new_raw(schema, mode, chunk.clone())?; let mut batches = Vec::new(); while let Some(batch) = cursor.append_to(builder)? { batches.push(batch); @@ -289,13 +292,14 @@ fn append_chunk( /// Decodes every batch with one reused decoder, so repeated calls exercise the cached schema. fn decode_batches( schema: &SeriesSchema, - mode: RawLabelMode, + mode: FrameLabelMode, batches: &[RecordBatch], ) -> Vec { - let decoder = RawSeriesFrameDecoder::try_new(schema, mode).unwrap(); + let decode = (mode == FrameLabelMode::Key).then(dense_description); + let decoder = SeriesFrameDecoder::try_new(schema, mode, decode.as_ref()).unwrap(); batches .iter() - .flat_map(|batch| decoder.decode(batch).unwrap()) + .flat_map(|batch| decoder.decode_raw(batch).unwrap()) .collect() } @@ -376,16 +380,22 @@ fn sliced_histogram_frames_densify_without_duplicating_shared_table() { // paths below stay observable; the default budget coalesces the same rows later on. let frames = pack_chunk_with_budget( &schema, - RawLabelMode::Key, + FrameLabelMode::Key, &chunk, 1, - RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, ) .unwrap(); assert_eq!(frames.len(), 64); let mut decoded_chunks = Vec::with_capacity(frames.len()); for frame in &frames { - let decoded = unpack_raw_series_frame(&schema, RawLabelMode::Key, frame).unwrap(); + let decoded = unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + frame, + ) + .unwrap(); assert_eq!(decoded.len(), 1); assert_eq!(decoded[0].time.len(), 1); decoded_chunks.push(decoded.into_iter().next().unwrap()); @@ -419,10 +429,16 @@ fn sliced_histogram_frames_densify_without_duplicating_shared_table() { // The default budget groups the same 64 rows into one multirow batch; every row still decodes // to its own slice of the shared table with bit-exact values. - let coalesced = pack_raw_series_chunk(&schema, RawLabelMode::Key, &chunk).unwrap(); + let coalesced = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &chunk, + ) + .unwrap(); assert_eq!(coalesced.len(), 1); assert_eq!(coalesced[0].num_rows(), 64); - let coalesced_rows = decode_batches(&schema, RawLabelMode::Key, &coalesced); + let coalesced_rows = decode_batches(&schema, FrameLabelMode::Key, &coalesced); assert_eq!(coalesced_rows.len(), 64); for (position, row) in coalesced_rows.iter().enumerate() { assert_eq!(row.primary_key.as_slice(), &[0x42]); @@ -478,8 +494,9 @@ fn raw_frame_field_names_bind_payloads_to_their_schema() { ]) .unwrap(); - for mode in [RawLabelMode::Key, RawLabelMode::Explicit] { - let metric = (mode == RawLabelMode::Explicit).then(|| metric(Some("host"), &[])); + for mode in [FrameLabelMode::Key, FrameLabelMode::Explicit] { + let decode = (mode == FrameLabelMode::Key).then(dense_description); + let metric = (mode == FrameLabelMode::Explicit).then(|| metric(Some("host"), &[])); let chunk = RawSeriesChunk::try_new( &schema, Buffer::from(vec![0xaa_u8]), @@ -489,7 +506,7 @@ fn raw_frame_field_names_bind_payloads_to_their_schema() { true, ) .unwrap(); - let batch = &pack_raw_series_chunk(&schema, mode, &chunk).unwrap()[0]; + let batch = &pack_raw_series_chunk(&schema, mode, decode.as_ref(), &chunk).unwrap()[0]; let payloads = batch .column(3) .as_any() @@ -504,7 +521,7 @@ fn raw_frame_field_names_bind_payloads_to_their_schema() { ["cpu_usage", "cpu_idle"] ); - let decoded = unpack_raw_series_frame(&schema, mode, batch).unwrap(); + let decoded = unpack_raw_series_frame(&schema, mode, decode.as_ref(), batch).unwrap(); assert_eq!(decoded[0].primary_key.as_slice(), &[0xaa]); assert!(decoded[0].is_last); let times = decoded[0].time.as_buffer(); @@ -513,19 +530,35 @@ fn raw_frame_field_names_bind_payloads_to_their_schema() { assert_float_bits(decoded[0].fields.get(1).unwrap(), &[Some(10.0), Some(20.0)]); assert!( - unpack_raw_series_frame(&test_schema(&["cpu_idle", "cpu_usage"]), mode, batch).is_err() + unpack_raw_series_frame( + &test_schema(&["cpu_idle", "cpu_usage"]), + mode, + decode.as_ref(), + batch + ) + .is_err() ); assert!( - unpack_raw_series_frame(&test_schema(&["cpu_usage", "cpu_busy"]), mode, batch).is_err() + unpack_raw_series_frame( + &test_schema(&["cpu_usage", "cpu_busy"]), + mode, + decode.as_ref(), + batch + ) + .is_err() ); } } +/// A key frame declares the label decode description that makes its `read_key` readable, so its +/// metadata carries that text; an explicit frame declares none, and its `read_key` column is +/// nullable because an evaluated explicit row carries no storage key at all. #[test] fn schema_and_key_explicit_round_trips_preserve_identity_and_bits() { let schema = test_schema(&["value"]); - let key_schema = raw_series_frame_schema(&schema, RawLabelMode::Key).unwrap(); - let explicit_schema = raw_series_frame_schema(&schema, RawLabelMode::Explicit).unwrap(); + let key_schema = + series_frame_schema(&schema, FrameLabelMode::Key, Some(&dense_description())).unwrap(); + let explicit_schema = raw_series_frame_schema(&schema, FrameLabelMode::Explicit).unwrap(); let payload = DataType::Struct( vec![ Field::new( @@ -560,10 +593,17 @@ fn schema_and_key_explicit_round_trips_preserve_identity_and_bits() { .into(), ); let mut expected_key_metadata = std::collections::HashMap::new(); - expected_key_metadata.insert("greptime.series.layout".to_string(), "v1".to_string()); + expected_key_metadata.insert("greptime.series.layout".to_string(), "v2".to_string()); expected_key_metadata.insert("greptime.series.content".to_string(), "raw".to_string()); expected_key_metadata.insert("greptime.series.labels".to_string(), "key".to_string()); expected_key_metadata.insert("greptime.series.time".to_string(), "timestamps".to_string()); + expected_key_metadata.insert(ORDERING.to_string(), "contiguous".to_string()); + // A key frame also declares the description that decodes its labels. Its exact text is pinned by + // the metadata fixture, so it is spelled here by the same serializer the writer uses. + expected_key_metadata.insert( + LABEL_DECODE.to_string(), + serde_json::to_string(&dense_description()).unwrap(), + ); let expected_key = Schema::new_with_metadata( vec![ Field::new("read_key", DataType::Binary, false), @@ -589,7 +629,10 @@ fn schema_and_key_explicit_round_trips_preserve_identity_and_bits() { expected_key_metadata, ); assert_eq!(key_schema.as_ref(), &expected_key); + // An explicit frame still has the key column, but only a raw explicit row carries a key: an + // evaluated one has none, so the column is nullable here while a key frame's is required. let mut explicit_fields = expected_key.fields().iter().cloned().collect::>(); + explicit_fields[0] = Field::new("read_key", DataType::Binary, true).into(); explicit_fields.push(Field::new("name", DataType::Utf8, true).into()); explicit_fields.push( Field::new( @@ -611,6 +654,8 @@ fn schema_and_key_explicit_round_trips_preserve_identity_and_bits() { ); let mut explicit_metadata = expected_key.metadata().clone(); explicit_metadata.insert("greptime.series.labels".to_string(), "explicit".to_string()); + // An explicit frame carries complete labels, so it declares no decode description. + explicit_metadata.remove(LABEL_DECODE); let expected_explicit = Schema::new_with_metadata(explicit_fields, explicit_metadata); assert_eq!(explicit_schema.as_ref(), &expected_explicit); @@ -625,9 +670,21 @@ fn schema_and_key_explicit_round_trips_preserve_identity_and_bits() { values.to_vec(), true, ); - let packed = pack_raw_series_chunk(&schema, RawLabelMode::Key, &native).unwrap(); + let packed = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &native, + ) + .unwrap(); assert_eq!(packed.len(), 1); - let decoded = unpack_raw_series_frame(&schema, RawLabelMode::Key, &packed[0]).unwrap(); + let decoded = unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &packed[0], + ) + .unwrap(); assert_eq!(decoded.len(), 1); assert_eq!(decoded[0].primary_key.as_slice(), &[0, 0xff, 0, 0x80]); assert!(decoded[0].metric.is_none()); @@ -648,20 +705,45 @@ fn schema_and_key_explicit_round_trips_preserve_identity_and_bits() { vec![Some(1.25)], false, ); - let packed = pack_raw_series_chunk(&schema, RawLabelMode::Explicit, &explicit).unwrap(); - let decoded = unpack_raw_series_frame(&schema, RawLabelMode::Explicit, &packed[0]).unwrap(); + let packed = pack_raw_series_chunk(&schema, FrameLabelMode::Explicit, None, &explicit).unwrap(); + let decoded = + unpack_raw_series_frame(&schema, FrameLabelMode::Explicit, None, &packed[0]).unwrap(); assert_eq!(decoded[0].primary_key.as_slice(), &[0x80, 0, 0xff]); assert!(!decoded[0].is_last); let labels = decoded[0].metric.as_ref().unwrap(); assert_eq!(labels.name(), None); assert_eq!(labels.get("rack"), Some("")); assert_eq!(labels.get("zone"), Some("west")); - assert!(pack_raw_series_chunk(&schema, RawLabelMode::Key, &explicit).is_err()); - assert!(pack_raw_series_chunk(&schema, RawLabelMode::Explicit, &native).is_err()); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &packed[0]).is_err()); + assert!( + pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &explicit + ) + .is_err() + ); + assert!(pack_raw_series_chunk(&schema, FrameLabelMode::Explicit, None, &native).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &packed[0] + ) + .is_err() + ); - let key_packed = pack_raw_series_chunk(&schema, RawLabelMode::Key, &native).unwrap(); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Explicit, &key_packed[0]).is_err()); + let key_packed = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &native, + ) + .unwrap(); + assert!( + unpack_raw_series_frame(&schema, FrameLabelMode::Explicit, None, &key_packed[0]).is_err() + ); for name in [None, Some(""), Some("cpu")] { let named = float_chunk( &schema, @@ -671,9 +753,10 @@ fn schema_and_key_explicit_round_trips_preserve_identity_and_bits() { vec![Some(2.0)], false, ); - let encoded = pack_raw_series_chunk(&schema, RawLabelMode::Explicit, &named).unwrap(); + let encoded = + pack_raw_series_chunk(&schema, FrameLabelMode::Explicit, None, &named).unwrap(); let restored = - unpack_raw_series_frame(&schema, RawLabelMode::Explicit, &encoded[0]).unwrap(); + unpack_raw_series_frame(&schema, FrameLabelMode::Explicit, None, &encoded[0]).unwrap(); assert_eq!(restored[0].metric.as_ref().unwrap().name(), name); assert_eq!(restored[0].metric.as_ref().unwrap().get("empty"), Some("")); } @@ -714,7 +797,13 @@ fn segmented_mixed_fields_emit_every_aligned_position_and_marker() { .unwrap(); // Variable field boundaries now group every aligned interval into one multirow batch instead // of one batch per boundary, so decode the actual rows that batch carries. - let frames = pack_raw_series_chunk(&schema, RawLabelMode::Key, &chunk).unwrap(); + let frames = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &chunk, + ) + .unwrap(); assert_eq!(frames.len(), 1); assert_eq!(frames[0].num_rows(), 4); let mut observed_times = Vec::new(); @@ -722,7 +811,13 @@ fn segmented_mixed_fields_emit_every_aligned_position_and_marker() { let mut observed_second = Vec::new(); let mut observed_keys = Vec::new(); let mut observed_markers = Vec::new(); - let rows = unpack_raw_series_frame(&schema, RawLabelMode::Key, &frames[0]).unwrap(); + let rows = unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &frames[0], + ) + .unwrap(); assert_eq!(rows.len(), 4); for (row_index, row) in rows.iter().enumerate() { assert_eq!(row.is_last, row_index == rows.len() - 1); @@ -795,12 +890,22 @@ fn segmented_mixed_fields_emit_every_aligned_position_and_marker() { true, ) .unwrap(); - let aligned_frames = - pack_raw_series_chunk(&aligned_schema, RawLabelMode::Key, &aligned_chunk).unwrap(); + let aligned_frames = pack_raw_series_chunk( + &aligned_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &aligned_chunk, + ) + .unwrap(); assert_eq!(aligned_frames.len(), 1); assert_eq!(aligned_frames[0].num_rows(), 3); - let aligned_rows = - unpack_raw_series_frame(&aligned_schema, RawLabelMode::Key, &aligned_frames[0]).unwrap(); + let aligned_rows = unpack_raw_series_frame( + &aligned_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &aligned_frames[0], + ) + .unwrap(); assert_eq!(aligned_rows.len(), 3); let mut interval_lengths = Vec::new(); let mut aligned_times = Vec::new(); @@ -896,12 +1001,23 @@ fn segmented_mixed_fields_emit_every_aligned_position_and_marker() { ) .unwrap(); distinct_rows.push( - pack_raw_series_chunk(&alternate_schema, RawLabelMode::Key, &alternate) - .unwrap() - .remove(0), + pack_raw_series_chunk( + &alternate_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &alternate, + ) + .unwrap() + .remove(0), ); let multirow = concat_batches(&frames[0].schema(), &distinct_rows).unwrap(); - let decoded = unpack_raw_series_frame(&schema, RawLabelMode::Key, &multirow).unwrap(); + let decoded = unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &multirow, + ) + .unwrap(); assert_eq!(decoded.len(), 5); assert_eq!( decoded.iter().map(|row| row.is_last).collect::>(), @@ -929,7 +1045,13 @@ fn segmented_mixed_fields_emit_every_aligned_position_and_marker() { Some(&[0xed][..]), ])); let sliced = arrow::record_batch::RecordBatch::try_new(sliced.schema(), altered).unwrap(); - let sliced_rows = unpack_raw_series_frame(&schema, RawLabelMode::Key, &sliced).unwrap(); + let sliced_rows = unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &sliced, + ) + .unwrap(); assert_eq!( sliced_rows .iter() @@ -975,10 +1097,24 @@ fn segmented_mixed_fields_emit_every_aligned_position_and_marker() { false, ) .unwrap(); - let not_last_frames = pack_raw_series_chunk(&schema, RawLabelMode::Key, ¬_last).unwrap(); + let not_last_frames = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + ¬_last, + ) + .unwrap(); let markers = not_last_frames .iter() - .flat_map(|frame| unpack_raw_series_frame(&schema, RawLabelMode::Key, frame).unwrap()) + .flat_map(|frame| { + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + frame, + ) + .unwrap() + }) .map(|row| row.is_last) .collect::>(); assert_eq!(markers, [false; 4]); @@ -1036,8 +1172,20 @@ fn histogram_payload_and_present_all_null_representation_survive() { false, ) .unwrap(); - let frame = pack_raw_series_chunk(&schema, RawLabelMode::Key, &chunk).unwrap(); - let decoded = unpack_raw_series_frame(&schema, RawLabelMode::Key, &frame[0]).unwrap(); + let frame = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &chunk, + ) + .unwrap(); + let decoded = unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &frame[0], + ) + .unwrap(); let DenseSamplesRef::Histogram(histograms) = decoded[0].fields.get(0).unwrap().dense().unwrap() else { panic!("histogram representation was normalized"); @@ -1088,10 +1236,20 @@ fn histogram_payload_and_present_all_null_representation_survive() { false, ) .unwrap(); - let mixed_frame = - pack_raw_series_chunk(&mixed_schema, RawLabelMode::Key, &mixed_chunk).unwrap(); - let mixed_decoded = - unpack_raw_series_frame(&mixed_schema, RawLabelMode::Key, &mixed_frame[0]).unwrap(); + let mixed_frame = pack_raw_series_chunk( + &mixed_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &mixed_chunk, + ) + .unwrap(); + let mixed_decoded = unpack_raw_series_frame( + &mixed_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &mixed_frame[0], + ) + .unwrap(); let DenseSamplesRef::Mixed(mixed) = mixed_decoded[0].fields.get(0).unwrap().dense().unwrap() else { panic!("all-null mixed representation was normalized"); @@ -1103,6 +1261,134 @@ fn histogram_payload_and_present_all_null_representation_survive() { ); } +#[test] +fn direct_decoder_validates_ordering_for_raw_and_evaluated_batches() { + let raw_schema = test_schema(&["value"]); + let raw_frame_schema = + series_frame_schema(&raw_schema, FrameLabelMode::Key, Some(&dense_description())).unwrap(); + let raw_chunk = float_chunk(&raw_schema, &[1], None, vec![1], vec![Some(2.0)], true); + let raw_batch = pack_raw_series_chunk( + &raw_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &raw_chunk, + ) + .unwrap() + .remove(0); + let raw_decoder = + SeriesFrameDecoder::try_new(&raw_schema, FrameLabelMode::Key, Some(&dense_description())) + .unwrap(); + + let evaluated_schema = SeriesSchema { + fields: raw_schema.fields.clone(), + eval_grid: Some(Arc::new( + crate::series::time::EvalGrid::try_new( + crate::series::time::GridSpec::try_new(0, 10, 3).unwrap(), + ) + .unwrap(), + )), + }; + let evaluated_frame_schema = + series_frame_schema(&evaluated_schema, FrameLabelMode::Explicit, None).unwrap(); + let series = Series::try_new( + &evaluated_schema, + metric(Some("cpu"), &[]), + TimeAxis::from_grid_spans( + evaluated_schema.eval_grid.clone().unwrap(), + vec![crate::series::time::GridSpan::new(1, 1)], + ) + .unwrap(), + SeriesFields::one(Samples::from_float(Float64Array::from(vec![Some(3.0)]))), + ) + .unwrap(); + let mut builder = + SeriesFrameBuilder::try_new(&evaluated_schema, FrameLabelMode::Explicit, None, 8, 1024) + .unwrap(); + let mut cursor = SeriesFrameCursor::try_new_evaluated(&evaluated_schema, series, true).unwrap(); + while cursor.append_to(&mut builder).unwrap().is_some() {} + let evaluated_batch = builder.flush().unwrap().unwrap(); + let evaluated_decoder = + SeriesFrameDecoder::try_new(&evaluated_schema, FrameLabelMode::Explicit, None).unwrap(); + + let raw_empty = arrow::record_batch::RecordBatch::new_empty(raw_frame_schema); + let evaluated_empty = arrow::record_batch::RecordBatch::new_empty(evaluated_frame_schema); + for (content, empty, nonempty) in [ + ("raw", &raw_empty, &raw_batch), + ("evaluated", &evaluated_empty, &evaluated_batch), + ] { + for (ordering, accepted) in [ + (None, false), + (Some("future_value"), false), + (Some("interleaved"), false), + (Some("contiguous"), true), + (Some("read_key_then_time"), content == "raw"), + ] { + for batch in [empty, nonempty] { + let mut metadata = batch.schema().metadata().clone(); + match ordering { + Some(value) => { + metadata.insert(ORDERING.to_string(), value.to_string()); + } + None => { + metadata.remove(ORDERING); + } + } + let schema = Arc::new(Schema::new_with_metadata( + batch.schema().fields().clone(), + metadata, + )); + let batch = arrow::record_batch::RecordBatch::try_new( + schema.clone(), + batch.columns().to_vec(), + ) + .unwrap(); + let declaration = SeriesFrameDeclaration::parse(schema.as_ref()); + if accepted { + assert!(declaration.is_ok(), "{content} ordering {ordering:?}"); + if content == "raw" { + let rows = raw_decoder.decode_raw(&batch).unwrap(); + assert_eq!(rows.len(), batch.num_rows()); + if !rows.is_empty() { + assert!(rows[0].is_last); + assert!(matches!( + rows[0].fields.get(0).unwrap().sample(0), + SampleRef::Float(value) if value == 2.0 + )); + } + } else { + let rows = evaluated_decoder.decode_evaluated(&batch).unwrap(); + assert_eq!(rows.len(), batch.num_rows()); + if !rows.is_empty() { + assert!(rows[0].1); + assert!(matches!( + rows[0].0.fields.get(0).unwrap().sample(0), + SampleRef::Float(value) if value == 3.0 + )); + } + } + } else { + assert!( + declaration.unwrap_err().to_string().contains("ordering"), + "{content} ordering {ordering:?}" + ); + let result = if content == "raw" { + raw_decoder.decode_raw(&batch).map(|_| ()) + } else { + evaluated_decoder.decode_evaluated(&batch).map(|_| ()) + }; + assert!( + result.unwrap_err().to_string().contains("ordering"), + "{content} ordering {ordering:?}" + ); + } + } + } + } +} + +/// The encoder's own declarations are refused when they contradict the frame, and the schema with +/// no fields is refused; an evaluation grid, by contrast, now describes a legal evaluated frame +/// instead of making the schema unusable here. #[test] fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() { let schema = test_schema(&["value"]); @@ -1115,13 +1401,41 @@ fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() { .unwrap(), )), }; - assert!(raw_series_frame_schema(&grid_schema, RawLabelMode::Key).is_err()); - let empty_schema = raw_series_frame_schema(&schema, RawLabelMode::Key).unwrap(); + // A schema with an evaluation grid describes evaluated samples, which is a legal frame in this + // family: the raw-only constructor still refuses it, while the one constructor accepts it as an + // evaluated frame that declares its grid and addresses time by grid spans. + assert!(raw_series_frame_schema(&grid_schema, FrameLabelMode::Explicit).is_err()); + let evaluated = series_frame_schema(&grid_schema, FrameLabelMode::Explicit, None).unwrap(); + assert_eq!( + evaluated.metadata().get(CONTENT).map(String::as_str), + Some("evaluated") + ); + assert_eq!( + evaluated.metadata().get(TIME).map(String::as_str), + Some("grid_spans") + ); + assert!(evaluated.metadata().get(GRID).is_some()); + // An evaluated frame names its series by label, so a key identity stays refused. + assert!( + series_frame_schema( + &grid_schema, + FrameLabelMode::Key, + Some(&dense_description()) + ) + .is_err() + ); + let empty_schema = + series_frame_schema(&schema, FrameLabelMode::Key, Some(&dense_description())).unwrap(); let empty = arrow::record_batch::RecordBatch::new_empty(empty_schema.clone()); assert!( - unpack_raw_series_frame(&schema, RawLabelMode::Key, &empty) - .unwrap() - .is_empty() + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &empty + ) + .unwrap() + .is_empty() ); let wrong = arrow::record_batch::RecordBatch::new_empty(Arc::new(Schema::new(vec![Field::new( @@ -1129,12 +1443,22 @@ fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() { DataType::Utf8, false, )]))); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &wrong).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &wrong + ) + .is_err() + ); let no_fields = SeriesSchema { fields: Arc::from([]), eval_grid: None, }; - assert!(raw_series_frame_schema(&no_fields, RawLabelMode::Key).is_err()); + assert!( + series_frame_schema(&no_fields, FrameLabelMode::Key, Some(&dense_description())).is_err() + ); let mut metadata = empty_schema.metadata().clone(); metadata.insert( @@ -1146,14 +1470,35 @@ fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() { metadata, )); let empty_bad = arrow::record_batch::RecordBatch::new_empty(contaminated); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &empty_bad).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &empty_bad + ) + .is_err() + ); let chunk = float_chunk(&schema, &[1], None, vec![1], vec![Some(2.0)], false); - assert!(pack_raw_series_chunk(&grid_schema, RawLabelMode::Key, &chunk).is_err()); + assert!( + pack_raw_series_chunk( + &grid_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &chunk + ) + .is_err() + ); - let valid = pack_raw_series_chunk(&schema, RawLabelMode::Key, &chunk) - .unwrap() - .remove(0); + let valid = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &chunk, + ) + .unwrap() + .remove(0); let mut extra_metadata = valid.schema().metadata().clone(); extra_metadata.insert("application.note".to_string(), "ignored".to_string()); let unrelated = arrow::record_batch::RecordBatch::try_new( @@ -1165,9 +1510,14 @@ fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() { ) .unwrap(); assert_eq!( - unpack_raw_series_frame(&schema, RawLabelMode::Key, &unrelated) - .unwrap() - .len(), + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &unrelated + ) + .unwrap() + .len(), 1 ); @@ -1181,10 +1531,23 @@ fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() { valid.columns().to_vec(), ) .unwrap(); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &wrong).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &wrong + ) + .is_err() + ); + // A raw frame must not declare an evaluation grid: `greptime.series.grid` belongs to an + // evaluated frame only. let mut grid_metadata = valid.schema().metadata().clone(); - grid_metadata.insert("greptime.series.grid.start".to_string(), "0".to_string()); + grid_metadata.insert( + "greptime.series.grid".to_string(), + "{\"start_ms\":0,\"step_ms\":1,\"len\":3}".to_string(), + ); let contaminated = arrow::record_batch::RecordBatch::try_new( Arc::new(Schema::new_with_metadata( valid.schema().fields().clone(), @@ -1193,13 +1556,32 @@ fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() { valid.columns().to_vec(), ) .unwrap(); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &contaminated).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &contaminated + ) + .is_err() + ); + + // A reserved key this revision does not know is refused when the declarations are parsed, + // rather than travelling along as dead metadata or being silently ignored. + let mut unknown_key_metadata = valid.schema().metadata().clone(); + unknown_key_metadata.insert("greptime.series.grid.start".to_string(), "0".to_string()); + let unregistered = + Schema::new_with_metadata(valid.schema().fields().clone(), unknown_key_metadata); + assert!(series_frame_label_decode(&unregistered).is_err()); + assert!(SeriesFrameDeclaration::parse(&unregistered).is_err()); for metadata_case in ["wrong_layout", "wrong_labels", "wrong_time"] { let mut metadata = valid.schema().metadata().clone(); match metadata_case { "wrong_layout" => { - metadata.insert("greptime.series.layout".to_string(), "v2".to_string()); + // `v2` is the only layout this revision reads, so the previously unreadable `v1` + // is what a wrong layout is spelled as now. + metadata.insert("greptime.series.layout".to_string(), "v1".to_string()); } "wrong_labels" => { metadata.insert("greptime.series.labels".to_string(), "explicit".to_string()); @@ -1216,7 +1598,15 @@ fn rejects_invalid_schema_and_metadata_and_accepts_valid_empty() { valid.columns().to_vec(), ) .unwrap(); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &malformed).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &malformed + ) + .is_err() + ); } } @@ -1227,12 +1617,25 @@ fn rejects_invalid_timestamps_and_empty_raw_payloads() { let schema = test_schema(&["value"]); let chunk = float_chunk(&schema, &[1], None, vec![1], vec![Some(2.0)], false); - let valid = pack_raw_series_chunk(&schema, RawLabelMode::Key, &chunk) - .unwrap() - .remove(0); + let valid = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &chunk, + ) + .unwrap() + .remove(0); for bad_timestamps in [vec![], vec![1, 1], vec![2, 1]] { let malformed = with_timestamp_values(&valid, bad_timestamps); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &malformed).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &malformed + ) + .is_err() + ); } let mut empty_columns = valid.columns().to_vec(); let empty_time = with_timestamp_values(&valid, vec![]); @@ -1267,7 +1670,15 @@ fn rejects_invalid_timestamps_and_empty_raw_payloads() { empty_columns[3] = Arc::new(empty_fields); let empty_time_and_payload = arrow::record_batch::RecordBatch::try_new(valid.schema(), empty_columns).unwrap(); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &empty_time_and_payload).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &empty_time_and_payload + ) + .is_err() + ); let two_samples = float_chunk( &schema, @@ -1277,12 +1688,25 @@ fn rejects_invalid_timestamps_and_empty_raw_payloads() { vec![Some(2.0), Some(3.0)], false, ); - let valid_two_samples = pack_raw_series_chunk(&schema, RawLabelMode::Key, &two_samples) - .unwrap() - .remove(0); + let valid_two_samples = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &two_samples, + ) + .unwrap() + .remove(0); for bad_timestamps in [vec![1, 1], vec![2, 1]] { let malformed = with_timestamp_values(&valid_two_samples, bad_timestamps); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &malformed).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &malformed + ) + .is_err() + ); } let mut empty_payload_columns = valid.columns().to_vec(); @@ -1322,7 +1746,15 @@ fn rejects_invalid_timestamps_and_empty_raw_payloads() { empty_payload_columns[3] = Arc::new(empty_fields); let empty_payload = arrow::record_batch::RecordBatch::try_new(valid.schema(), empty_payload_columns).unwrap(); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &empty_payload).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &empty_payload + ) + .is_err() + ); let mut columns = valid.columns().to_vec(); columns[3] = Arc::new(StructArray::new_null( @@ -1348,7 +1780,15 @@ fn rejects_invalid_timestamps_and_empty_raw_payloads() { let invalid_later_row = arrow::record_batch::RecordBatch::try_new(valid.schema(), bad).unwrap(); let later_invalid = concat_batches(&valid.schema(), &[valid.clone(), invalid_later_row]).unwrap(); - assert!(unpack_raw_series_frame(&schema, RawLabelMode::Key, &later_invalid).is_err()); + assert!( + unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &later_invalid + ) + .is_err() + ); } #[test] @@ -1362,9 +1802,14 @@ fn rejects_invalid_explicit_labels() { vec![Some(1.0)], false, ); - let explicit = pack_raw_series_chunk(&explicit_schema, RawLabelMode::Explicit, &explicit_chunk) - .unwrap() - .remove(0); + let explicit = pack_raw_series_chunk( + &explicit_schema, + FrameLabelMode::Explicit, + None, + &explicit_chunk, + ) + .unwrap() + .remove(0); let name = explicit .column(4) .as_any() @@ -1427,7 +1872,10 @@ fn rejects_invalid_explicit_labels() { duplicate_labels[5] = Arc::new(duplicate_list); let duplicate = arrow::record_batch::RecordBatch::try_new(explicit.schema(), duplicate_labels).unwrap(); - assert!(unpack_raw_series_frame(&explicit_schema, RawLabelMode::Explicit, &duplicate).is_err()); + assert!( + unpack_raw_series_frame(&explicit_schema, FrameLabelMode::Explicit, None, &duplicate) + .is_err() + ); let mut reserved_labels = explicit.columns().to_vec(); let reserved_items = StructArray::try_new( @@ -1458,7 +1906,10 @@ fn rejects_invalid_explicit_labels() { reserved_labels[5] = Arc::new(reserved_list); let reserved = arrow::record_batch::RecordBatch::try_new(explicit.schema(), reserved_labels).unwrap(); - assert!(unpack_raw_series_frame(&explicit_schema, RawLabelMode::Explicit, &reserved).is_err()); + assert!( + unpack_raw_series_frame(&explicit_schema, FrameLabelMode::Explicit, None, &reserved) + .is_err() + ); } #[test] @@ -1466,7 +1917,8 @@ fn rejects_malformed_histogram_and_payload_encodings() { let hist_schema = test_schema(&["hist"]); let valid_hist = pack_raw_series_chunk( &hist_schema, - RawLabelMode::Key, + FrameLabelMode::Key, + Some(&dense_description()), &RawSeriesChunk::try_new( &hist_schema, Buffer::from(vec![1_u8]), @@ -1553,8 +2005,13 @@ fn rejects_malformed_histogram_and_payload_encodings() { let absent_hist = arrow::record_batch::RecordBatch::try_new(valid_hist.schema(), absent_hist_columns) .unwrap(); - let decoded_absent_hist = - unpack_raw_series_frame(&hist_schema, RawLabelMode::Key, &absent_hist).unwrap(); + let decoded_absent_hist = unpack_raw_series_frame( + &hist_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &absent_hist, + ) + .unwrap(); assert!( matches!(decoded_absent_hist[0].fields.get(0).unwrap().sample(0), SampleRef::Float(v) if v == 1.25) ); @@ -1606,7 +2063,15 @@ fn rejects_malformed_histogram_and_payload_encodings() { let absent_both = arrow::record_batch::RecordBatch::try_new(valid_hist.schema(), absent_both_columns) .unwrap(); - assert!(unpack_raw_series_frame(&hist_schema, RawLabelMode::Key, &absent_both).is_err()); + assert!( + unpack_raw_series_frame( + &hist_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &absent_both + ) + .is_err() + ); let DataType::List(index_item) = null_hist.fields()[0].data_type() else { panic!("expected histogram indices list"); @@ -1636,7 +2101,15 @@ fn rejects_malformed_histogram_and_payload_encodings() { bad_parent_columns[3] = Arc::new(bad_fields); let present_bad_parent = arrow::record_batch::RecordBatch::try_new(valid_hist.schema(), bad_parent_columns).unwrap(); - assert!(unpack_raw_series_frame(&hist_schema, RawLabelMode::Key, &present_bad_parent).is_err()); + assert!( + unpack_raw_series_frame( + &hist_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &present_bad_parent + ) + .is_err() + ); let valid_float = ListArray::try_new( match payload.fields()[0].data_type() { @@ -1677,7 +2150,15 @@ fn rejects_malformed_histogram_and_payload_encodings() { overlap_columns[3] = Arc::new(overlap_fields); let overlap = arrow::record_batch::RecordBatch::try_new(valid_hist.schema(), overlap_columns).unwrap(); - assert!(unpack_raw_series_frame(&hist_schema, RawLabelMode::Key, &overlap).is_err()); + assert!( + unpack_raw_series_frame( + &hist_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &overlap + ) + .is_err() + ); let short_indices = ListArray::try_new( index_item.clone(), @@ -1704,7 +2185,15 @@ fn rejects_malformed_histogram_and_payload_encodings() { short_columns[3] = Arc::new(short_fields); let short_indices = arrow::record_batch::RecordBatch::try_new(valid_hist.schema(), short_columns).unwrap(); - assert!(unpack_raw_series_frame(&hist_schema, RawLabelMode::Key, &short_indices).is_err()); + assert!( + unpack_raw_series_frame( + &hist_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &short_indices + ) + .is_err() + ); let malformed_histogram = || { let mut columns = valid_hist.columns().to_vec(); @@ -1779,7 +2268,13 @@ fn rejects_malformed_histogram_and_payload_encodings() { arrow::record_batch::RecordBatch::try_new(valid_hist.schema(), columns).unwrap() }; assert!( - unpack_raw_series_frame(&hist_schema, RawLabelMode::Key, &malformed_histogram()).is_err() + unpack_raw_series_frame( + &hist_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &malformed_histogram() + ) + .is_err() ); let mut hidden_float = valid_hist.columns().to_vec(); @@ -1820,8 +2315,13 @@ fn rejects_malformed_histogram_and_payload_encodings() { hidden_float[3] = Arc::new(fields); let absent_float = arrow::record_batch::RecordBatch::try_new(valid_hist.schema(), hidden_float).unwrap(); - let decoded_absent_float = - unpack_raw_series_frame(&hist_schema, RawLabelMode::Key, &absent_float).unwrap(); + let decoded_absent_float = unpack_raw_series_frame( + &hist_schema, + FrameLabelMode::Key, + Some(&dense_description()), + &absent_float, + ) + .unwrap(); assert_histogram( match decoded_absent_float[0].fields.get(0).unwrap().sample(0) { SampleRef::Histogram(histogram) => histogram, @@ -1849,8 +2349,20 @@ fn decoded_key_and_payload_slices_keep_buffers_alive() { true, ) .unwrap(); - let frames = pack_raw_series_chunk(&schema, RawLabelMode::Key, &chunk).unwrap(); - let decoded = unpack_raw_series_frame(&schema, RawLabelMode::Key, &frames[0]).unwrap(); + let frames = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &chunk, + ) + .unwrap(); + let decoded = unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &frames[0], + ) + .unwrap(); (owner, chunk, frames, decoded) }; let batch = frames[0].clone(); @@ -2090,10 +2602,10 @@ fn sliced_canonical_mixed_histogram_round_trip_keeps_buffers_alive() { // observable: the default budget could coalesce several rows and copy their payloads. let frames = pack_chunk_with_budget( &schema, - RawLabelMode::Key, + FrameLabelMode::Key, &chunk, 1, - RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, ) .unwrap(); assert_eq!(frames.len(), 1); @@ -2200,7 +2712,13 @@ fn sliced_canonical_mixed_histogram_round_trip_keeps_buffers_alive() { .as_ptr() ); } - let decoded = unpack_raw_series_frame(&schema, RawLabelMode::Key, packed).unwrap(); + let decoded = unpack_raw_series_frame( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + packed, + ) + .unwrap(); let DenseSamplesRef::Mixed(decoded_mixed) = decoded[0].fields.get(0).unwrap().dense().unwrap() else { panic!("mixed representation was normalized"); @@ -2369,7 +2887,15 @@ fn malformed_pack_input_is_revalidated_before_encoding() { fields: SeriesFields::one(Samples::from_float(Float64Array::from(vec![Some(3.0)]))), ..chunk }; - assert!(pack_raw_series_chunk(&schema, RawLabelMode::Key, &invalid).is_err()); + assert!( + pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &invalid + ) + .is_err() + ); } #[test] @@ -2476,21 +3002,23 @@ fn cross_chunk_rows_coalesce_into_one_multirow_batch() { vec![Some(3.0), Some(4.0), None], true, ); - assert_eq!(RawSeriesFrameBuilder::DEFAULT_MAX_ROWS, 256); - assert_eq!(RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, 1 << 20); - let expected = raw_series_frame_schema(&schema, RawLabelMode::Key).unwrap(); + assert_eq!(SeriesFrameBuilder::DEFAULT_MAX_ROWS, 256); + assert_eq!(SeriesFrameBuilder::DEFAULT_MAX_BYTES, 1 << 20); + let expected = + series_frame_schema(&schema, FrameLabelMode::Key, Some(&dense_description())).unwrap(); - let mut builder = RawSeriesFrameBuilder::try_new( + let mut builder = SeriesFrameBuilder::try_new( &schema, - RawLabelMode::Key, - RawSeriesFrameBuilder::DEFAULT_MAX_ROWS, - RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, + FrameLabelMode::Key, + Some(&dense_description()), + SeriesFrameBuilder::DEFAULT_MAX_ROWS, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, ) .unwrap(); assert_eq!(builder.schema(), &expected); - let mut batches = append_chunk(&mut builder, &schema, RawLabelMode::Key, &first).unwrap(); - batches.extend(append_chunk(&mut builder, &schema, RawLabelMode::Key, &second).unwrap()); + let mut batches = append_chunk(&mut builder, &schema, FrameLabelMode::Key, &first).unwrap(); + batches.extend(append_chunk(&mut builder, &schema, FrameLabelMode::Key, &second).unwrap()); if let Some(batch) = builder.flush().unwrap() { batches.push(batch); } @@ -2510,7 +3038,7 @@ fn cross_chunk_rows_coalesce_into_one_multirow_batch() { ["read_key", "is_last", "time", "fields"] ); for (key, value) in [ - ("greptime.series.layout", "v1"), + ("greptime.series.layout", "v2"), ("greptime.series.content", "raw"), ("greptime.series.labels", "key"), ("greptime.series.time", "timestamps"), @@ -2578,7 +3106,7 @@ fn cross_chunk_rows_coalesce_into_one_multirow_batch() { assert_eq!(float_values.value(2).to_bits(), 3.0_f64.to_bits()); assert_eq!(float_values.value(3).to_bits(), 4.0_f64.to_bits()); - let decoded = decode_batches(&schema, RawLabelMode::Key, &batches); + let decoded = decode_batches(&schema, FrameLabelMode::Key, &batches); assert_eq!(decoded.len(), 2); assert_eq!(decoded[0].primary_key.as_slice(), &[0xaa]); assert!(!decoded[0].is_last); @@ -2614,17 +3142,18 @@ fn raw_frame_builder_emits_at_the_row_budget_without_another_cursor() { ]; for max_rows in [2usize, 3] { - let mut builder = RawSeriesFrameBuilder::try_new( + let mut builder = SeriesFrameBuilder::try_new( &schema, - RawLabelMode::Key, + FrameLabelMode::Key, + Some(&dense_description()), max_rows, - RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, ) .unwrap(); let mut emitted_at = None; let mut batches = Vec::new(); for (index, chunk) in chunks.iter().enumerate() { - let produced = append_chunk(&mut builder, &schema, RawLabelMode::Key, chunk).unwrap(); + let produced = append_chunk(&mut builder, &schema, FrameLabelMode::Key, chunk).unwrap(); assert!(produced.len() <= 1, "one cursor emits at most one batch"); if let Some(batch) = produced.first() { assert!( @@ -2660,7 +3189,7 @@ fn raw_frame_builder_emits_at_the_row_budget_without_another_cursor() { .collect::>() ); - let decoded = decode_batches(&schema, RawLabelMode::Key, &batches); + let decoded = decode_batches(&schema, FrameLabelMode::Key, &batches); assert_eq!(decoded.len(), 3); assert_eq!( decoded @@ -2681,30 +3210,42 @@ fn raw_frame_builder_rejects_zero_budgets_and_bounds_offsets() { let schema = test_schema(&["value"]); // Zero in either budget is not a legal configuration; the literals take the parameter's type. assert!( - RawSeriesFrameBuilder::try_new( + SeriesFrameBuilder::try_new( &schema, - RawLabelMode::Key, + FrameLabelMode::Key, + Some(&dense_description()), 0, - RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, ) .is_err() ); assert!( - RawSeriesFrameBuilder::try_new( + SeriesFrameBuilder::try_new( &schema, - RawLabelMode::Key, - RawSeriesFrameBuilder::DEFAULT_MAX_ROWS, + FrameLabelMode::Key, + Some(&dense_description()), + SeriesFrameBuilder::DEFAULT_MAX_ROWS, 0, ) .is_err() ); - assert!(RawSeriesFrameBuilder::try_new(&schema, RawLabelMode::Key, 0, 0).is_err()); assert!( - RawSeriesFrameBuilder::try_new( + SeriesFrameBuilder::try_new( &schema, - RawLabelMode::Key, + FrameLabelMode::Key, + Some(&dense_description()), + 0, + 0 + ) + .is_err() + ); + assert!( + SeriesFrameBuilder::try_new( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), 1, - RawSeriesFrameBuilder::DEFAULT_MAX_BYTES, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, ) .is_ok() ); @@ -2740,9 +3281,9 @@ fn byte_budget_emits_legal_oversized_rows_without_dropping_them() { // batch instead of being dropped or merged. let batches = pack_chunk_with_budget( &schema, - RawLabelMode::Key, + FrameLabelMode::Key, &chunk, - RawSeriesFrameBuilder::DEFAULT_MAX_ROWS, + SeriesFrameBuilder::DEFAULT_MAX_ROWS, 1, ) .unwrap(); @@ -2754,7 +3295,7 @@ fn byte_budget_emits_legal_oversized_rows_without_dropping_them() { "an oversized row is emitted alone, never dropped" ); - let decoded = decode_batches(&schema, RawLabelMode::Key, &batches); + let decoded = decode_batches(&schema, FrameLabelMode::Key, &batches); assert_eq!(decoded.len(), 3); for (position, row) in decoded.iter().enumerate() { assert_eq!(row.primary_key.as_slice(), &[0x9a]); @@ -2776,24 +3317,25 @@ fn byte_budget_flushes_before_exceeding_and_retries_the_pending_row() { let mut measured = Vec::with_capacity(chunks.len()); for chunk in &chunks { let mut cursor = - RawSeriesChunkCursor::try_new(&schema, RawLabelMode::Key, chunk.clone()).unwrap(); + SeriesFrameCursor::try_new_raw(&schema, FrameLabelMode::Key, chunk.clone()).unwrap(); measured.push(cursor.next_row().unwrap().unwrap().bytes); } let budget = measured[0] + measured[1] - 1; - let mut builder = RawSeriesFrameBuilder::try_new( + let mut builder = SeriesFrameBuilder::try_new( &schema, - RawLabelMode::Key, - RawSeriesFrameBuilder::DEFAULT_MAX_ROWS, + FrameLabelMode::Key, + Some(&dense_description()), + SeriesFrameBuilder::DEFAULT_MAX_ROWS, budget, ) .unwrap(); let mut first = - RawSeriesChunkCursor::try_new(&schema, RawLabelMode::Key, chunks[0].clone()).unwrap(); + SeriesFrameCursor::try_new_raw(&schema, FrameLabelMode::Key, chunks[0].clone()).unwrap(); assert!(first.append_to(&mut builder).unwrap().is_none()); let mut second = - RawSeriesChunkCursor::try_new(&schema, RawLabelMode::Key, chunks[1].clone()).unwrap(); + SeriesFrameCursor::try_new_raw(&schema, FrameLabelMode::Key, chunks[1].clone()).unwrap(); let emitted = second.append_to(&mut builder).unwrap().unwrap(); assert_eq!(emitted.num_rows(), 1, "the first row is flushed alone"); // The second row stays pending across the flush and is retried, never dropped. @@ -2801,7 +3343,7 @@ fn byte_budget_flushes_before_exceeding_and_retries_the_pending_row() { let remainder = builder.flush().unwrap().unwrap(); assert_eq!(remainder.num_rows(), 1); - let decoded = decode_batches(&schema, RawLabelMode::Key, &[emitted, remainder]); + let decoded = decode_batches(&schema, FrameLabelMode::Key, &[emitted, remainder]); assert_eq!(decoded.len(), 2); assert_eq!(decoded[0].primary_key.as_slice(), &[1]); assert_eq!(decoded[1].primary_key.as_slice(), &[2]); @@ -2816,10 +3358,11 @@ fn byte_budget_flushes_before_exceeding_and_retries_the_pending_row() { // A requested byte bound is clamped to the representable cumulative offset range, and that // range is rejected before any child array is allocated. - let clamped = RawSeriesFrameBuilder::try_new( + let clamped = SeriesFrameBuilder::try_new( &schema, - RawLabelMode::Key, - RawSeriesFrameBuilder::DEFAULT_MAX_ROWS, + FrameLabelMode::Key, + Some(&dense_description()), + SeriesFrameBuilder::DEFAULT_MAX_ROWS, usize::MAX, ) .unwrap(); @@ -2848,7 +3391,13 @@ fn multirow_histogram_rows_keep_row_local_indices_and_tables() { .unwrap(); // The public wrapper groups two independent rows into one multirow batch. - let packed = pack_raw_series_chunk(&schema, RawLabelMode::Key, &chunk).unwrap(); + let packed = pack_raw_series_chunk( + &schema, + FrameLabelMode::Key, + Some(&dense_description()), + &chunk, + ) + .unwrap(); assert_eq!(packed.len(), 1); assert_eq!(packed[0].num_rows(), 2); @@ -2880,7 +3429,11 @@ fn multirow_histogram_rows_keep_row_local_indices_and_tables() { .unwrap(); assert_eq!(concatenated.len(), 4); - let decoded = decode_batches(&schema, RawLabelMode::Key, std::slice::from_ref(&packed[0])); + let decoded = decode_batches( + &schema, + FrameLabelMode::Key, + std::slice::from_ref(&packed[0]), + ); assert_row_local_histogram_rows(&decoded); // A real Arrow IPC round trip keeps the schema, the whole tables, the null index entries and @@ -2907,17 +3460,19 @@ fn multirow_histogram_rows_keep_row_local_indices_and_tables() { .value(1), 0 ); - let restored_rows = decode_batches(&schema, RawLabelMode::Key, &restored); + let restored_rows = decode_batches(&schema, FrameLabelMode::Key, &restored); assert_row_local_histogram_rows(&restored_rows); } #[test] fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { let schema = test_schema(&["value"]); - let expected = raw_series_frame_schema(&schema, RawLabelMode::Key).unwrap(); + let expected = + series_frame_schema(&schema, FrameLabelMode::Key, Some(&dense_description())).unwrap(); let first = pack_raw_series_chunk( &schema, - RawLabelMode::Key, + FrameLabelMode::Key, + Some(&dense_description()), &float_chunk( &schema, &[1], @@ -2931,16 +3486,26 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { .remove(0); let second = pack_raw_series_chunk( &schema, - RawLabelMode::Key, + FrameLabelMode::Key, + Some(&dense_description()), &float_chunk(&schema, &[2], None, vec![3], vec![Some(3.0)], true), ) .unwrap() .remove(0); - let key_decoder = RawSeriesFrameDecoder::try_new(&schema, RawLabelMode::Key).unwrap(); - let decoded = key_decoder.decode(&first).unwrap(); + let key_decoder = + SeriesFrameDecoder::try_new(&schema, FrameLabelMode::Key, Some(&dense_description())) + .unwrap(); + let decoded = key_decoder.decode_raw(&first).unwrap(); assert_eq!(decoded.len(), 1); assert_eq!(decoded[0].primary_key.as_slice(), &[1]); + assert_eq!( + match decoded[0].fields.get(0).unwrap().dense().unwrap() { + crate::series::samples::DenseSamplesRef::Float(values) => values.value(0), + _ => panic!("expected float samples"), + }, + 1.0 + ); assert_eq!( (0..decoded[0].time.len()) .map(|position| decoded[0].time.timestamp(position).unwrap()) @@ -2948,7 +3513,7 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { [1, 2] ); // The same decoder accepts a later valid batch before the malformed ones below. - let decoded = key_decoder.decode(&second).unwrap(); + let decoded = key_decoder.decode_raw(&second).unwrap(); assert_eq!(decoded.len(), 1); assert_eq!(decoded[0].primary_key.as_slice(), &[2]); assert!(decoded[0].is_last); @@ -2964,11 +3529,12 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { second.columns().to_vec(), ) .unwrap(); - assert_eq!(key_decoder.decode(&unrelated).unwrap().len(), 1); + assert_eq!(key_decoder.decode_raw(&unrelated).unwrap().len(), 1); // Reserved metadata and evolved fields are rejected, including by a zero-row batch. for (key, value) in [ - ("greptime.series.layout", "v2"), + // `v2` is the layout this decoder expects, so the unreadable `v1` is the wrong one. + ("greptime.series.layout", "v1"), ("greptime.series.content", "evaluated"), ("greptime.series.labels", "explicit"), ("greptime.series.time", "grid"), @@ -2984,28 +3550,34 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { ) .unwrap(); assert!( - key_decoder.decode(&contaminated).is_err(), + key_decoder.decode_raw(&contaminated).is_err(), "{key} must be rejected" ); } let evolved_schema = test_schema(&["other"]); let evolved = pack_raw_series_chunk( &evolved_schema, - RawLabelMode::Key, + FrameLabelMode::Key, + Some(&dense_description()), &float_chunk(&evolved_schema, &[3], None, vec![4], vec![Some(4.0)], false), ) .unwrap() .remove(0); - assert!(key_decoder.decode(&evolved).is_err()); + assert!(key_decoder.decode_raw(&evolved).is_err()); let empty_evolved = arrow::record_batch::RecordBatch::new_empty( - raw_series_frame_schema(&evolved_schema, RawLabelMode::Key).unwrap(), + series_frame_schema( + &evolved_schema, + FrameLabelMode::Key, + Some(&dense_description()), + ) + .unwrap(), ); - assert!(key_decoder.decode(&empty_evolved).is_err()); + assert!(key_decoder.decode_raw(&empty_evolved).is_err()); // Rejections leave the cached schema usable, and a valid zero-row batch stays valid. - assert_eq!(key_decoder.decode(&first).unwrap().len(), 1); + assert_eq!(key_decoder.decode_raw(&first).unwrap().len(), 1); assert!( key_decoder - .decode(&arrow::record_batch::RecordBatch::new_empty( + .decode_raw(&arrow::record_batch::RecordBatch::new_empty( expected.clone() )) .unwrap() @@ -3013,10 +3585,12 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { ); // Explicit mode reuses one decoder for a present but nameless metric and a named one. - let explicit_decoder = RawSeriesFrameDecoder::try_new(&schema, RawLabelMode::Explicit).unwrap(); + let explicit_decoder = + SeriesFrameDecoder::try_new(&schema, FrameLabelMode::Explicit, None).unwrap(); let nameless = pack_raw_series_chunk( &schema, - RawLabelMode::Explicit, + FrameLabelMode::Explicit, + None, &float_chunk( &schema, &[7], @@ -3030,7 +3604,8 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { .remove(0); let named = pack_raw_series_chunk( &schema, - RawLabelMode::Explicit, + FrameLabelMode::Explicit, + None, &float_chunk( &schema, &[8], @@ -3042,12 +3617,12 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { ) .unwrap() .remove(0); - let decoded = explicit_decoder.decode(&nameless).unwrap(); + let decoded = explicit_decoder.decode_raw(&nameless).unwrap(); let labels = decoded[0].metric.as_ref().unwrap(); assert_eq!(labels.name(), None); assert_eq!(labels.get("zone"), Some("west")); assert_eq!( - explicit_decoder.decode(&named).unwrap()[0] + explicit_decoder.decode_raw(&named).unwrap()[0] .metric .as_ref() .unwrap() @@ -3055,8 +3630,8 @@ fn decoder_reuses_cached_schema_and_rejects_schema_evolution() { Some("cpu") ); // Label modes are not interchangeable. - assert!(explicit_decoder.decode(&first).is_err()); - assert!(key_decoder.decode(&nameless).is_err()); + assert!(explicit_decoder.decode_raw(&first).is_err()); + assert!(key_decoder.decode_raw(&nameless).is_err()); } /// A dense description carrying the ordered key columns and the logical tags the decoder reads. @@ -3087,22 +3662,26 @@ fn dense_description() -> LabelDecodeSpec { #[test] fn label_decode_description_roundtrips_through_schema_metadata() { let description = dense_description(); - let key = raw_series_frame_schema_with_label_decode( + let key = series_frame_schema( &test_schema(&["value"]), - RawLabelMode::Key, + FrameLabelMode::Key, Some(&description), ) .unwrap(); assert_eq!( - raw_series_frame_label_decode(&key).unwrap(), + series_frame_label_decode(&key).unwrap(), Some(description.clone()) ); - // The description is metadata-only: the column contract is untouched. - assert_eq!( - key.fields(), - raw_series_frame_schema(&test_schema(&["value"]), RawLabelMode::Key) - .unwrap() - .fields() + // The description is metadata-only: the column contract is untouched, so only the + // `greptime.series.label_decode` text differs from a frame declaring another description. + let mut other = dense_description(); + other.encoding = LabelDecodeEncoding::Sparse; + let other = + series_frame_schema(&test_schema(&["value"]), FrameLabelMode::Key, Some(&other)).unwrap(); + assert_eq!(key.fields(), other.fields()); + assert_ne!( + key.metadata().get(LABEL_DECODE), + other.metadata().get(LABEL_DECODE) ); assert_eq!(key.metadata().get(LABELS).map(String::as_str), Some("key")); } @@ -3110,20 +3689,19 @@ fn label_decode_description_roundtrips_through_schema_metadata() { #[test] fn explicit_frames_declare_no_label_decode_description() { let schema = test_schema(&["value"]); - let explicit = - raw_series_frame_schema_with_label_decode(&schema, RawLabelMode::Explicit, None).unwrap(); + let explicit = series_frame_schema(&schema, FrameLabelMode::Explicit, None).unwrap(); assert!(explicit.metadata().get(LABEL_DECODE).is_none()); - assert_eq!(raw_series_frame_label_decode(&explicit).unwrap(), None); + assert_eq!(series_frame_label_decode(&explicit).unwrap(), None); // A key frame without a description, or an explicit frame with one, is rejected instead of // being written as a frame the consumer cannot decode. assert!( - raw_series_frame_schema_with_label_decode(&schema, RawLabelMode::Key, None).is_err(), + series_frame_schema(&schema, FrameLabelMode::Key, None).is_err(), "key-encoded frames must describe their labels" ); assert!( - raw_series_frame_schema_with_label_decode( + series_frame_schema( &schema, - RawLabelMode::Explicit, + FrameLabelMode::Explicit, Some(&dense_description()) ) .is_err(), @@ -3133,25 +3711,37 @@ fn explicit_frames_declare_no_label_decode_description() { /// The raw frame's own columns with exactly the supplied metadata map. fn schema_with_metadata(metadata: Vec<(String, String)>) -> Schema { - let layout = raw_series_frame_schema(&test_schema(&["value"]), RawLabelMode::Key).unwrap(); + let layout = series_frame_schema( + &test_schema(&["value"]), + FrameLabelMode::Key, + Some(&dense_description()), + ) + .unwrap(); Schema::new_with_metadata(layout.fields().clone(), metadata.into_iter().collect()) } -/// The four metadata entries every raw frame declares, with the given label mode. +/// The five metadata entries every raw v2 frame declares, with the given label mode. fn base_metadata(labels: &str) -> Vec<(String, String)> { [ - ("greptime.series.layout", "v1"), + ("greptime.series.layout", "v2"), ("greptime.series.content", "raw"), ("greptime.series.labels", labels), ("greptime.series.time", "timestamps"), + ("greptime.series.ordering", "contiguous"), ] .into_iter() .map(|(key, value)| (key.to_string(), value.to_string())) .collect() } +/// A description the reader would refuse cannot be written into a frame. +/// +/// The three shapes below are each a self-contradictory binding: a duplicate key column, a logical +/// tag naming a column the primary key does not list, and a tag listed twice. Writing one would +/// produce a frame no consumer could decode, so the producer is where it is refused. A frame that +/// arrives already carrying one is a different case, and the decoder rejects those on read. #[test] -fn label_decode_description_rejects_inconsistent_bindings() { +fn label_decode_description_rejects_inconsistent_bindings_on_write() { let schema = test_schema(&["value"]); let mut cases = Vec::new(); @@ -3169,20 +3759,13 @@ fn label_decode_description_rejects_inconsistent_bindings() { cases.push(("duplicate logical tag", duplicate_tag)); for (label, description) in cases { - // Writing is faithful: the description records what the producer declares. Reading is - // where a description this revision cannot honour is rejected, so a producer cannot turn a - // bad binding into a frame a consumer would decode under silently. - let frame = raw_series_frame_schema_with_label_decode( - &schema, - RawLabelMode::Key, - Some(&description), - ) - .unwrap_or_else(|error| panic!("{label}: {error}")); - // The writer records what the producer declares, so the rejection has to come from the - // reader; one assertion covers it, since the frame it reads is the text just written. + // Both directions reject: a frame is worth writing only if it can be read back, so an + // inconsistent description is refused at the producer instead of being written faithfully + // and refused only once it reaches a consumer who cannot fix it. The case that must still + // reach the wire is a hand-built frame, which the fixture below covers. assert!( - raw_series_frame_label_decode(&frame).is_err(), - "{label}: an inconsistent description is rejected when read" + series_frame_schema(&schema, FrameLabelMode::Key, Some(&description)).is_err(), + "{label}: an inconsistent description is rejected when written" ); } } @@ -3198,7 +3781,7 @@ fn label_decode_rejects_a_frame_without_a_label_mode() { .filter(|(key, _)| key != LABELS) .collect(); assert!( - raw_series_frame_label_decode(&schema_with_metadata(without_mode)).is_err(), + series_frame_label_decode(&schema_with_metadata(without_mode)).is_err(), "a frame without a label mode has no label layout to describe" ); } @@ -3216,25 +3799,26 @@ fn raw_series_label_decode_metadata_fixture() { } let cases: Vec = serde_json::from_str(include_str!( - "../../../../../tests/compatibility/wire/raw_series_label_decode/v1_metadata.json" + "../../../../../tests/compatibility/wire/raw_series_label_decode/v2_metadata.json" )) .unwrap(); for case in cases { - let layout = raw_series_frame_schema(&test_schema(&["value"]), RawLabelMode::Key).unwrap(); - let metadata = case - .metadata - .iter() - .map(|(key, value)| (key.clone(), value.clone())) - .collect(); - let frame_schema = Schema::new_with_metadata(layout.fields().clone(), metadata); + let layout = series_frame_schema( + &test_schema(&["value"]), + FrameLabelMode::Key, + Some(&dense_description()), + ) + .unwrap(); + let frame_schema = + Schema::new_with_metadata(layout.fields().clone(), case.metadata.clone()); match case.encoding.as_deref() { None => assert!( - raw_series_frame_label_decode(&frame_schema).is_err(), + series_frame_label_decode(&frame_schema).is_err(), "{}: must be rejected", case.name ), Some(expected) => { - let decode = raw_series_frame_label_decode(&frame_schema) + let decode = series_frame_label_decode(&frame_schema) .unwrap_or_else(|error| panic!("{}: {error}", case.name)) .unwrap_or_else(|| panic!("{}: no description", case.name)); let encoding = match decode.encoding { @@ -3279,9 +3863,9 @@ fn raw_series_label_decode_metadata_fixture() { if expected == "dense" { // The fixture text is the encoder's own output, byte for byte: re-encoding the // description it decodes to must reproduce it exactly. - let roundtrip = raw_series_frame_schema_with_label_decode( + let roundtrip = series_frame_schema( &test_schema(&["value"]), - RawLabelMode::Key, + FrameLabelMode::Key, Some(&decode), ) .unwrap(); @@ -3309,9 +3893,9 @@ fn label_decode_description_carries_a_timestamp_typed_tag_column() { column_id: 2, name: "observed".to_string(), }); - let frame = raw_series_frame_schema_with_label_decode( + let frame = series_frame_schema( &test_schema(&["value"]), - RawLabelMode::Key, + FrameLabelMode::Key, Some(&description), ) .unwrap(); @@ -3321,7 +3905,7 @@ fn label_decode_description_carries_a_timestamp_typed_tag_column() { text.contains("\"physical_type\":{\"Timestamp\":{\"Millisecond\":null}}"), "the encoder wrote a timestamp type: {text}" ); - let recovered = raw_series_frame_label_decode(&frame).unwrap().unwrap(); + let recovered = series_frame_label_decode(&frame).unwrap().unwrap(); assert_eq!(recovered, description); assert_eq!( recovered.primary_key[1].physical_type, @@ -3354,9 +3938,9 @@ fn label_decode_text_is_independent_of_the_callers_tag_order() { descending.primary_key.reverse(); let write = |description: &LabelDecodeSpec| { - let frame = raw_series_frame_schema_with_label_decode( + let frame = series_frame_schema( &test_schema(&["value"]), - RawLabelMode::Key, + FrameLabelMode::Key, Some(description), ) .unwrap(); @@ -3394,10 +3978,10 @@ fn label_decode_text_is_independent_of_the_callers_tag_order() { write(&descending), "the writer reordered the encoding columns" ); - let recovered = raw_series_frame_label_decode( - &raw_series_frame_schema_with_label_decode( + let recovered = series_frame_label_decode( + &series_frame_schema( &test_schema(&["value"]), - RawLabelMode::Key, + FrameLabelMode::Key, Some(&descending), ) .unwrap(), diff --git a/src/datatypes/src/series/time.rs b/src/datatypes/src/series/time.rs index 966ee2f6e15..1568ce7d24c 100644 --- a/src/datatypes/src/series/time.rs +++ b/src/datatypes/src/series/time.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use arrow::buffer::ScalarBuffer; use once_cell::sync::OnceCell; +use serde::{Deserialize, Serialize}; use snafu::OptionExt; use crate::error::{InvalidSeriesSnafu, Result}; @@ -25,7 +26,12 @@ use crate::error::{InvalidSeriesSnafu, Result}; mod tests; /// Value description of a non-empty, fixed-step evaluation grid. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +/// +/// The serde form is the wire form a series frame writes under `greptime.series.grid`: field names +/// in declaration order, no optional members. Two nodes compare that text byte for byte, so the +/// only place that spells it is a series frame's own grid metadata entry. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] pub struct GridSpec { /// First evaluation timestamp in milliseconds. pub start_ms: i64, diff --git a/src/mito-codec/src/labels/tests.rs b/src/mito-codec/src/labels/tests.rs index 88651bd1c35..2d84176d530 100644 --- a/src/mito-codec/src/labels/tests.rs +++ b/src/mito-codec/src/labels/tests.rs @@ -629,13 +629,13 @@ fn frame_label_decode_description_decodes_like_the_bound_source() { ); let description = common_query::series_frame::label_decode_spec(&bound) .expect("a primary-key binding describes its key layout"); - let frame_schema = datatypes::series::frame::raw_series_frame_schema_with_label_decode( + let frame_schema = datatypes::series::frame::series_frame_schema( &frame_series_schema(), - datatypes::series::frame::RawLabelMode::Key, + datatypes::series::frame::FrameLabelMode::Key, Some(&description), ) .unwrap(); - let recovered = datatypes::series::frame::raw_series_frame_label_decode(&frame_schema) + let recovered = datatypes::series::frame::series_frame_label_decode(&frame_schema) .unwrap() .expect("the frame declares its decode description"); assert_eq!(recovered, description); diff --git a/src/mito2/src/engine/scan_test.rs b/src/mito2/src/engine/scan_test.rs index 55f8cf80408..bfb316959a6 100644 --- a/src/mito2/src/engine/scan_test.rs +++ b/src/mito2/src/engine/scan_test.rs @@ -5093,7 +5093,7 @@ async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() { assert_eq!(raw_visible, visible); // Exercise the production table physical consumer over a fresh real Mito scanner. - use common_query::series_frame::{raw_series_frame_ordering, unpack_raw_series}; + use common_query::series_frame::{series_frame_ordering, unpack_raw_series}; use datafusion::execution::TaskContext; use datafusion::physical_plan::ExecutionPlan; use table::table::scan::RegionScanExec; @@ -5127,7 +5127,7 @@ async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() { .unwrap(); // Unpack under the frame's own declaration. A legacy producer declares `KeyThenTime`, and // only that declaration makes the decoder check that keys increase. - let frame_ordering = raw_series_frame_ordering(native_frames.schema().as_ref()).unwrap(); + let frame_ordering = series_frame_ordering(native_frames.schema().as_ref()).unwrap(); assert_eq!( SeriesOrdering::KeyThenTime, frame_ordering, @@ -5233,9 +5233,7 @@ async fn test_mito_handle_query_raw_series_candidate_projection_and_mvcc() { /// `promql::source::merge::tests::rejects_unsupported_ordering_and_empty_input_without_polling`. #[tokio::test] async fn test_two_phase_native_raw_stream_declares_contiguous_frames() { - use common_query::series_frame::{ - pack_raw_series, raw_series_frame_ordering, unpack_raw_series, - }; + use common_query::series_frame::{pack_raw_series, series_frame_ordering, unpack_raw_series}; use datatypes::series::SeriesOrdering; use datatypes::series::samples::SeriesSchema; use store_api::region_engine::{BoundTag, QueryScanContext, RawLabelSource}; @@ -5374,7 +5372,7 @@ async fn test_two_phase_native_raw_stream_declares_contiguous_frames() { chunks, ))) .unwrap(); - let ordering = raw_series_frame_ordering(packed.schema().as_ref()).unwrap(); + let ordering = series_frame_ordering(packed.schema().as_ref()).unwrap(); assert_eq!(SeriesOrdering::Contiguous, ordering); let validated = unpack_raw_series(packed, Some(schema)) .unwrap() @@ -6560,7 +6558,7 @@ async fn test_mito_raw_series_int64_predicate_uses_original_precision() { assert_eq!(vec![(2, 20.0)], output_values); // Validate the typed integer gate through RegionScanExec's actual physical consumer. - use common_query::series_frame::{raw_series_frame_ordering, unpack_raw_series}; + use common_query::series_frame::{series_frame_ordering, unpack_raw_series}; use datafusion::execution::TaskContext; use datafusion::physical_plan::ExecutionPlan; use datatypes::series::SeriesOrdering; @@ -6597,7 +6595,7 @@ async fn test_mito_raw_series_int64_predicate_uses_original_precision() { .execute(0, Arc::new(TaskContext::default())) .unwrap(); // Unpack under the frame's own declaration so the decoder checks the legacy key order. - let ordering = raw_series_frame_ordering(frames.schema().as_ref()).unwrap(); + let ordering = series_frame_ordering(frames.schema().as_ref()).unwrap(); assert_eq!( SeriesOrdering::KeyThenTime, ordering, @@ -6655,7 +6653,7 @@ async fn test_mito_raw_series_int64_predicate_uses_original_precision() { let empty_frames = empty_plan .execute(0, Arc::new(TaskContext::default())) .unwrap(); - let empty_ordering = raw_series_frame_ordering(empty_frames.schema().as_ref()).unwrap(); + let empty_ordering = series_frame_ordering(empty_frames.schema().as_ref()).unwrap(); assert_eq!( SeriesOrdering::KeyThenTime, empty_ordering, diff --git a/src/promql/src/extension_plan/fused_series/tests.rs b/src/promql/src/extension_plan/fused_series/tests.rs index 30bd1983dcf..00df6f05aeb 100644 --- a/src/promql/src/extension_plan/fused_series/tests.rs +++ b/src/promql/src/extension_plan/fused_series/tests.rs @@ -63,7 +63,11 @@ use datafusion::physical_plan::{ RecordBatchStream, SendableRecordBatchStream, collect, displayable, }; use datafusion::prelude::SessionContext; -use datatypes::series::frame::{RawLabelMode, pack_raw_series_chunk, raw_series_frame_schema}; +use datatypes::prelude::ConcreteDataType; +use datatypes::series::frame::{ + FrameLabelMode, LABEL_DECODE_VERSION, LabelDecodeColumn, LabelDecodeEncoding, LabelDecodeSpec, + LabelDecodeTag, pack_raw_series_chunk, raw_series_frame_schema, series_frame_schema, +}; use datatypes::series::raw::{RawSeriesChunk, RawTimestamps}; use datatypes::series::samples::{FieldSpec, Samples, Series, SeriesFields, SeriesSchema}; use datatypes::series::time::{EvalGrid, GridSpan, GridSpec, TimeAxis}; @@ -488,7 +492,7 @@ fn binding(tree: &NativeTree) -> FusedSeriesBinding { /// The schema of a frame carrying series of `schema` with their labels explicit. fn frame_schema_of(schema: &SeriesSchema) -> SchemaRef { - raw_series_frame_schema(schema, RawLabelMode::Explicit).expect("explicit labels are a frame") + raw_series_frame_schema(schema, FrameLabelMode::Explicit).expect("explicit labels are a frame") } /// The schema of a frame carrying these series, which is what the node declares as its input. @@ -496,6 +500,30 @@ fn frame_schema() -> SchemaRef { frame_schema_of(&raw_schema()) } +/// The description a frame with its labels encoded in a dense [`HOST`] key declares. +fn host_label_decode() -> LabelDecodeSpec { + LabelDecodeSpec { + version: LABEL_DECODE_VERSION, + metric_name: METRIC.to_string(), + encoding: LabelDecodeEncoding::Dense, + primary_key: vec![LabelDecodeColumn { + column_id: 1, + name: HOST.to_string(), + physical_type: ConcreteDataType::string_datatype(), + }], + logical_tags: vec![LabelDecodeTag { + column_id: 1, + name: HOST.to_string(), + }], + } +} + +/// The schema of a frame carrying series of `schema` whose labels are encoded in their key. +fn keyed_frame_schema_of(schema: &SeriesSchema) -> SchemaRef { + series_frame_schema(schema, FrameLabelMode::Key, Some(&host_label_decode())) + .expect("key labels are a frame") +} + /// The columns this node produces: the field, the tags and the grid timestamps. fn output_schema() -> SchemaRef { Arc::new(Schema::new(vec![ @@ -975,7 +1003,7 @@ fn metric(labels: &[(&str, &str)], name: Option<&str>) -> SeriesLabels { /// One frame batch carrying `chunk` of `schema`, packed by the scan's own encoder. fn frame_of(schema: &SeriesSchema, chunk: &RawSeriesChunk) -> RecordBatch { - let mut batches = pack_raw_series_chunk(schema, RawLabelMode::Explicit, chunk) + let mut batches = pack_raw_series_chunk(schema, FrameLabelMode::Explicit, None, chunk) .expect("a chunk of this schema packs into a frame"); assert_eq!(1, batches.len(), "one small chunk is one frame batch"); batches.remove(0) @@ -2270,8 +2298,7 @@ async fn a_pending_boundary_loses_no_rows() { /// before the first batch, not when the missing labels reach the output. #[test] fn a_frame_whose_labels_are_encoded_is_rejected_when_the_plan_is_built() { - let keyed = - raw_series_frame_schema(&raw_schema(), RawLabelMode::Key).expect("key labels are a frame"); + let keyed = keyed_frame_schema_of(&raw_schema()); let error = plan(rate_tree(), memory_exec_with(keyed, Vec::new())) .expect_err("the frames carry no labels to read tags from"); assert!( @@ -2283,18 +2310,19 @@ fn a_frame_whose_labels_are_encoded_is_rejected_when_the_plan_is_built() { /// A frame that carries an evaluation grid is refused when the plan is built. /// /// A frame holds raw series: its rows are samples, not grid positions, and the grid every output -/// row is addressed on is the tree's own. A schema that declares a grid is one this node would -/// read positions out of, so it is refused rather than decoded. The source below is one partition, -/// which is the shape the node accepts, so what refuses the plan here is the frame's own -/// declaration and not the node's partition count: the refusal has to name the grid. +/// row is addressed on is the tree's own. A frame that declares a grid is one whose rows this node +/// would read as positions on it, so the shared decoder refuses it rather than reading samples out +/// of it. The source below is one partition, which is the shape the node accepts, so what refuses +/// the plan here is the frame's own declaration and not the node's partition count: the refusal has +/// to name the frame. #[test] fn a_frame_with_an_evaluation_grid_is_rejected_when_the_plan_is_built() { - let mut metadata = frame_schema().metadata().clone(); - metadata.insert("greptime.series.grid".to_string(), "v1".to_string()); - let gridded = Arc::new(Schema::new_with_metadata( - frame_schema().fields().clone(), - metadata, - )); + let gridded_schema = Arc::new(SeriesSchema { + fields: raw_schema().fields.clone(), + eval_grid: Some(grid()), + }); + let gridded = series_frame_schema(&gridded_schema, FrameLabelMode::Explicit, None) + .expect("a grid is a frame"); // One empty partition: every check but the frame's passes. let source = memory_exec_with(gridded, Vec::new()); assert_eq!( @@ -2305,7 +2333,7 @@ fn a_frame_with_an_evaluation_grid_is_rejected_when_the_plan_is_built() { let error = plan(rate_tree(), source).expect_err("the frame declares a grid"); let message = error_chain(&error); assert!( - message.contains("raw series schema cannot have an evaluation grid"), + message.contains("expected a raw frame, found an evaluated one"), "refused for another reason: {message}" ); assert!( @@ -2326,7 +2354,9 @@ fn a_frame_of_another_series_schema_is_rejected_when_the_plan_is_built() { let error = plan(rate_tree(), memory_exec_with(other, Vec::new())) .expect_err("the tree's input reads one field"); assert!( - error.to_string().contains("raw frame"), + error + .to_string() + .contains("frame fields do not match the expected series schema"), "refused for another reason: {error}" ); } diff --git a/src/promql/src/source/merge/tests.rs b/src/promql/src/source/merge/tests.rs index ff8a8ef9ad9..b7fcc0e0c69 100644 --- a/src/promql/src/source/merge/tests.rs +++ b/src/promql/src/source/merge/tests.rs @@ -24,7 +24,7 @@ use api::v1::SemanticType; use common_error::ext::{BoxedError, ErrorExt, PlainError}; use common_error::status_code::StatusCode; use common_query::series_frame::{ - label_decode_spec, pack_raw_series, raw_series_frame_ordering, unpack_raw_series, + label_decode_spec, pack_raw_series, series_frame_ordering, unpack_raw_series, }; use datatypes::arrow::array::Float64Array; use datatypes::arrow::buffer::{Buffer, ScalarBuffer}; @@ -458,7 +458,7 @@ async fn merges_two_unpacked_frames_through_the_frame_codec() { ); let schema = Arc::clone(stream.schema()); let packed = pack_raw_series(stream).unwrap(); - let ordering = raw_series_frame_ordering(packed.schema().as_ref()).unwrap(); + let ordering = series_frame_ordering(packed.schema().as_ref()).unwrap(); assert_eq!(SeriesOrdering::KeyThenTime, ordering); unpack_raw_series(packed, Some(schema)).unwrap() }; @@ -1889,7 +1889,7 @@ async fn merge_rejects_inputs_that_describe_different_decode_layouts() { /// that its chunks are undecoded key chunks, which is what a consumer decodes from. #[tokio::test] async fn merge_without_a_bound_identity_still_advertises_the_decode_layout() { - use common_query::series_frame::raw_series_frame_ordering; + use common_query::series_frame::series_frame_ordering; use common_query::series_frame::{pack_raw_series, unpack_raw_series}; use store_api::region_engine::NO_BOUND_LABEL_SOURCE; @@ -1937,11 +1937,11 @@ async fn merge_without_a_bound_identity_still_advertises_the_decode_layout() { // Packing advertises that layout under the merge's ordering, so the frame states both. let framed = pack_raw_series(merged).unwrap(); assert_eq!( - datatypes::series::frame::raw_series_frame_label_decode(framed.schema().as_ref()).unwrap(), + datatypes::series::frame::series_frame_label_decode(framed.schema().as_ref()).unwrap(), Some(carried.clone()) ); assert_eq!( - raw_series_frame_ordering(framed.schema().as_ref()).unwrap(), + series_frame_ordering(framed.schema().as_ref()).unwrap(), SeriesOrdering::KeyThenTime ); @@ -1970,7 +1970,7 @@ async fn merge_without_a_bound_identity_still_advertises_the_decode_layout() { #[tokio::test] async fn merged_frame_declares_a_label_decode_description_that_decodes_its_keys() { use common_query::series_frame::{pack_raw_series, unpack_raw_series}; - use datatypes::series::frame::raw_series_frame_label_decode; + use datatypes::series::frame::series_frame_label_decode; use mito_codec::labels::{LabelProjection, PrimaryKeyLabelDecoder}; let key = read_key("rack-a", "sensor-1"); @@ -1994,12 +1994,12 @@ async fn merged_frame_declares_a_label_decode_description_that_decodes_its_keys( let ordering = merged.ordering(); let framed = pack_raw_series(merged).unwrap(); assert_eq!( - common_query::series_frame::raw_series_frame_ordering(framed.schema().as_ref()).unwrap(), + common_query::series_frame::series_frame_ordering(framed.schema().as_ref()).unwrap(), ordering ); let declared = framed.schema(); assert_eq!( - raw_series_frame_label_decode(declared.as_ref()).unwrap(), + series_frame_label_decode(declared.as_ref()).unwrap(), Some(expected.clone()) ); diff --git a/src/promql/src/source/merge/tests/payload.rs b/src/promql/src/source/merge/tests/payload.rs index a180a01784a..459fad9ef57 100644 --- a/src/promql/src/source/merge/tests/payload.rs +++ b/src/promql/src/source/merge/tests/payload.rs @@ -19,8 +19,8 @@ use std::task::{Context, Poll}; use common_error::ext::BoxedError; use common_query::series_frame::{ - ensure_frame_matches_label_source, label_decode_spec, pack_raw_series, - raw_series_frame_ordering, unpack_raw_series, + ensure_frame_matches_label_source, label_decode_spec, pack_raw_series, series_frame_ordering, + unpack_raw_series, }; use datatypes::arrow::array::{Array, Float64Array, ListArray, UInt32Array}; use datatypes::arrow::buffer::{Buffer, ScalarBuffer}; @@ -263,7 +263,7 @@ async fn merges_typed_multifield_payload_through_frame_and_label_roundtrip() { // declares, since the merge only accepts inputs that declare one layout. ensure_frame_matches_label_source(framed.schema().as_ref(), &label_source(1)).unwrap(); assert_eq!( - raw_series_frame_ordering(framed.schema().as_ref()).unwrap(), + series_frame_ordering(framed.schema().as_ref()).unwrap(), frame_ordering, "the packed frame declares the merge's ordering" ); diff --git a/src/promql/src/source/tests.rs b/src/promql/src/source/tests.rs index 10c955dd3cf..acfd6d4107e 100644 --- a/src/promql/src/source/tests.rs +++ b/src/promql/src/source/tests.rs @@ -1073,8 +1073,7 @@ async fn materializes_labels_from_a_frame_with_no_supplied_binding() { ); let framed = pack_raw_series(raw).unwrap(); assert_eq!( - datatypes::series::frame::raw_series_frame_label_decode(framed.schema().as_ref()) - .unwrap(), + datatypes::series::frame::series_frame_label_decode(framed.schema().as_ref()).unwrap(), Some(description.clone()), "{encoding:?}: the frame schema is the producer's only label statement" ); diff --git a/src/query/src/dist_plan/merge_scan.rs b/src/query/src/dist_plan/merge_scan.rs index a9107b49375..78fb9792490 100644 --- a/src/query/src/dist_plan/merge_scan.rs +++ b/src/query/src/dist_plan/merge_scan.rs @@ -1423,7 +1423,7 @@ mod tests { }; use datatypes::prelude::{ConcreteDataType, VectorRef}; use datatypes::schema::{ColumnSchema, Schema}; - use datatypes::series::frame::{self, RawLabelMode}; + use datatypes::series::frame::{self, FrameLabelMode}; use datatypes::series::raw::{RawSeriesChunk, RawTimestamps}; use datatypes::series::samples::{ DenseSamplesRef, FieldSpec, SampleRef, Samples, SeriesFields, SeriesSchema, @@ -3361,7 +3361,8 @@ mod tests { async fn merge_scan_remote_schema_identity_rejects_empty_raw_frame_metadata_mismatch() { let region_id = RegionId::new(1024, 1); let raw = raw_test_series_schema(); - let frame = frame::raw_series_frame_schema(&raw, RawLabelMode::Key).unwrap(); + let decode = raw_test_label_decode(); + let frame = frame::series_frame_schema(&raw, FrameLabelMode::Key, Some(&decode)).unwrap(); let metadata = frame.metadata().clone(); let mut cases = Vec::new(); for (key, value) in [ @@ -3428,7 +3429,7 @@ mod tests { label_decode_spec, raw_series_frame_schema_with_ordering, }; use datatypes::series::SeriesOrdering; - use datatypes::series::frame::raw_series_frame_label_decode; + use datatypes::series::frame::series_frame_label_decode; let raw = raw_test_series_schema(); // Two bindings that are equal but not the same allocation: the expectation and the @@ -3503,7 +3504,7 @@ mod tests { spaced_metadata.insert(key.to_string(), spaced_text); let spaced = ArrowSchema::new_with_metadata(expected.fields().clone(), spaced_metadata); assert_eq!( - raw_series_frame_label_decode(&spaced) + series_frame_label_decode(&spaced) .unwrap() .map(|decode| decode.version), Some(1), @@ -3512,6 +3513,27 @@ mod tests { assert!(validate_remote_schema(&expected, &spaced, "test").is_err()); } + /// The key layout a key frame over [`raw_test_series_schema`] declares. + /// + /// A key-encoded frame cannot be declared without the description that decodes its key, and the + /// merge-scan tests never decode a key with it, so it only has to be self-consistent. + fn raw_test_label_decode() -> frame::LabelDecodeSpec { + frame::LabelDecodeSpec { + version: frame::LABEL_DECODE_VERSION, + metric_name: "cpu_usage".to_string(), + encoding: frame::LabelDecodeEncoding::Dense, + primary_key: vec![frame::LabelDecodeColumn { + column_id: 1, + name: "host".to_string(), + physical_type: ConcreteDataType::string_datatype(), + }], + logical_tags: vec![frame::LabelDecodeTag { + column_id: 1, + name: "host".to_string(), + }], + } + } + fn raw_test_series_schema() -> SeriesSchema { SeriesSchema { fields: vec![ @@ -3531,8 +3553,9 @@ mod tests { async fn merge_scan_remote_schema_identity_rejects_later_raw_frame_inner_metadata_mismatch() { let region_id = RegionId::new(1024, 1); let raw = raw_test_series_schema(); - let mode = RawLabelMode::Key; - let frame = frame::raw_series_frame_schema(&raw, mode).unwrap(); + let mode = FrameLabelMode::Key; + let decode = raw_test_label_decode(); + let frame = frame::series_frame_schema(&raw, mode, Some(&decode)).unwrap(); let mut changed_metadata = frame.metadata().clone(); changed_metadata.insert("greptime.series.marker".to_string(), "changed".to_string()); let changed_schema = Arc::new(ArrowSchema::new_with_metadata( @@ -3554,13 +3577,22 @@ mod tests { ) .unwrap() }; - let first = frame::pack_raw_series_chunk(&raw, mode, &chunk([1.0, 2.0], [10, 20], false)) - .unwrap() - .remove(0); - let mut second = - frame::pack_raw_series_chunk(&raw, mode, &chunk([3.0, 4.0], [30, 40], true)) - .unwrap() - .remove(0); + let first = frame::pack_raw_series_chunk( + &raw, + mode, + Some(&decode), + &chunk([1.0, 2.0], [10, 20], false), + ) + .unwrap() + .remove(0); + let mut second = frame::pack_raw_series_chunk( + &raw, + mode, + Some(&decode), + &chunk([3.0, 4.0], [30, 40], true), + ) + .unwrap() + .remove(0); second = DfRecordBatch::try_new(changed_schema, second.columns().to_vec()).unwrap(); let advertised = Arc::new(Schema::try_from(frame.clone()).unwrap()); let first = RecordBatch::from_df_record_batch(advertised.clone(), first); @@ -3578,7 +3610,7 @@ mod tests { ); let mut stream = exec.execute(0, Arc::new(TaskContext::default())).unwrap(); let emitted = stream.next().await.unwrap().unwrap(); - let decoded = frame::unpack_raw_series_frame(&raw, mode, &emitted).unwrap(); + let decoded = frame::unpack_raw_series_frame(&raw, mode, Some(&decode), &emitted).unwrap(); assert_eq!(decoded[0].time.timestamp(0), Some(10)); assert!(matches!( decoded[0].fields.get(0).unwrap().sample(0), @@ -3593,7 +3625,9 @@ mod tests { async fn merge_scan_remote_schema_identity_rejects_raw_frame_field_metadata_mismatch() { let region_id = RegionId::new(1024, 1); let raw = raw_test_series_schema(); - let expected = frame::raw_series_frame_schema(&raw, RawLabelMode::Key).unwrap(); + let expected = + frame::series_frame_schema(&raw, FrameLabelMode::Key, Some(&raw_test_label_decode())) + .unwrap(); let frame_metadata = expected.metadata().clone(); let fields = expected.fields().to_vec(); let mut field_cases = Vec::new(); @@ -3678,7 +3712,9 @@ mod tests { async fn merge_scan_remote_schema_identity_preserves_valid_raw_frame_payload() { let region_id = RegionId::new(1024, 1); let raw = raw_test_series_schema(); - let expected_frame = frame::raw_series_frame_schema(&raw, RawLabelMode::Key).unwrap(); + let decode = raw_test_label_decode(); + let expected_frame = + frame::series_frame_schema(&raw, FrameLabelMode::Key, Some(&decode)).unwrap(); let mut expected_metadata = expected_frame.metadata().clone(); expected_metadata.insert("application.note".to_string(), "expected".to_string()); let expected = Arc::new(ArrowSchema::new_with_metadata( @@ -3702,7 +3738,8 @@ mod tests { true, ) .unwrap(); - let packed = frame::pack_raw_series_chunk(&raw, RawLabelMode::Key, &chunk).unwrap(); + let packed = + frame::pack_raw_series_chunk(&raw, FrameLabelMode::Key, Some(&decode), &chunk).unwrap(); let inner = packed.into_iter().next().unwrap(); let original_columns = inner.columns().to_vec(); let batch = RecordBatch::from_df_record_batch(remote_schema.clone(), inner); @@ -3722,7 +3759,8 @@ mod tests { for (original, returned) in original_columns.iter().zip(result[0].columns()) { assert!(Arc::ptr_eq(original, returned)); } - let decoded = frame::unpack_raw_series_frame(&raw, RawLabelMode::Key, &result[0]); + let decoded = + frame::unpack_raw_series_frame(&raw, FrameLabelMode::Key, Some(&decode), &result[0]); let decoded = decoded.unwrap(); assert_eq!(decoded.len(), 1); assert_eq!(decoded[0].primary_key.as_slice(), &[1, 2, 3]); diff --git a/src/table/src/table/scan/series_tests.rs b/src/table/src/table/scan/series_tests.rs index f2120924877..6269b9cceeb 100644 --- a/src/table/src/table/scan/series_tests.rs +++ b/src/table/src/table/scan/series_tests.rs @@ -20,7 +20,7 @@ use std::task::{Context, Poll}; use api::v1::SemanticType; use common_error::ext::BoxedError; use common_query::request::RawSeriesRequest; -use common_query::series_frame::raw_series_frame_ordering; +use common_query::series_frame::series_frame_ordering; use common_recordbatch::{ QueryMemoryTracker, RecordBatch, RecordBatches, SendableRecordBatchStream, }; @@ -36,7 +36,7 @@ use datatypes::data_type::ConcreteDataType; use datatypes::error::InvalidSeriesSnafu; use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; use datatypes::series::SeriesOrdering; -use datatypes::series::frame::{LabelDecodeSpec, RawLabelMode}; +use datatypes::series::frame::{FrameLabelMode, LabelDecodeSpec}; use datatypes::series::raw::{RawSeriesChunk, RawTimestamps}; use datatypes::series::samples::{FieldSpec, Samples, SeriesFields, SeriesSchema}; use datatypes::vectors::{Int32Vector, TimestampMillisecondVector, VectorRef}; @@ -311,8 +311,37 @@ fn named_fixture() -> ( Arc>, store_api::metadata::RegionMetadataRef, ) { - let schema = Arc::new(Schema::new(vec![ - ColumnSchema::new("host", ConcreteDataType::string_datatype(), false), + named_fixture_with_tag_count(false) +} + +fn named_fixture_with_two_tags() -> ( + Arc, + Arc>, + store_api::metadata::RegionMetadataRef, +) { + named_fixture_with_tag_count(true) +} + +fn named_fixture_with_tag_count( + two_tags: bool, +) -> ( + Arc, + Arc>, + store_api::metadata::RegionMetadataRef, +) { + let mut columns = vec![ColumnSchema::new( + "host", + ConcreteDataType::string_datatype(), + false, + )]; + if two_tags { + columns.push(ColumnSchema::new( + "rack", + ConcreteDataType::string_datatype(), + false, + )); + } + columns.extend([ ColumnSchema::new("aux", ConcreteDataType::float64_datatype(), false), ColumnSchema::new("value", ConcreteDataType::float64_datatype(), false), ColumnSchema::new( @@ -320,25 +349,34 @@ fn named_fixture() -> ( ConcreteDataType::timestamp_millisecond_datatype(), false, ), - ])); - let batch = RecordBatch::new( - schema.clone(), - vec![ - Arc::new(datatypes::vectors::StringVector::from_slice(&["node"])) as VectorRef, - Arc::new(datatypes::vectors::Float64Vector::from_slice([2.0])) as VectorRef, - Arc::new(datatypes::vectors::Float64Vector::from_slice([1.0])) as VectorRef, - Arc::new(TimestampMillisecondVector::from_slice([42])) as VectorRef, - ], - ) - .unwrap(); + ]); + let schema = Arc::new(Schema::new(columns)); + let mut vectors: Vec = + vec![Arc::new(datatypes::vectors::StringVector::from_slice(&["node"])) as VectorRef]; + if two_tags { + vectors.push(Arc::new(datatypes::vectors::StringVector::from_slice(&["r1"])) as VectorRef); + } + vectors.extend([ + Arc::new(datatypes::vectors::Float64Vector::from_slice([2.0])) as VectorRef, + Arc::new(datatypes::vectors::Float64Vector::from_slice([1.0])) as VectorRef, + Arc::new(TimestampMillisecondVector::from_slice([42])) as VectorRef, + ]); + let batch = RecordBatch::new(schema.clone(), vectors).unwrap(); let batches = RecordBatches::try_new(schema, vec![batch]).unwrap(); let mut builder = RegionMetadataBuilder::new(RegionId::new(1, 2)); - builder - .push_column_metadata(ColumnMetadata { - column_schema: ColumnSchema::new("host", ConcreteDataType::string_datatype(), false), + builder.push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new("host", ConcreteDataType::string_datatype(), false), + semantic_type: SemanticType::Tag, + column_id: 7, + }); + if two_tags { + builder.push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new("rack", ConcreteDataType::string_datatype(), false), semantic_type: SemanticType::Tag, - column_id: 7, - }) + column_id: 8, + }); + } + builder .push_column_metadata(ColumnMetadata { column_schema: ColumnSchema::new("aux", ConcreteDataType::float64_datatype(), false), semantic_type: SemanticType::Field, @@ -357,8 +395,8 @@ fn named_fixture() -> ( ), semantic_type: SemanticType::Timestamp, column_id: 13, - }) - .primary_key(vec![7]); + }); + builder.primary_key(if two_tags { vec![7, 8] } else { vec![7] }); let metadata = Arc::new(builder.build().unwrap()); let raw_schema = Arc::new(SeriesSchema { fields: vec![ @@ -673,7 +711,8 @@ async fn native_raw_frame_executes_and_decodes_bound_data() { assert_eq!(frames[0].schema(), plan.schema()); let decoded = datatypes::series::frame::unpack_raw_series_frame( &schema, - RawLabelMode::Explicit, + FrameLabelMode::Explicit, + None, &frames[0], ) .unwrap(); @@ -765,10 +804,7 @@ async fn planned_ordering_is_preserved_for_empty_and_nonempty_frames() { .unwrap() .unwrap(); let planned_schema = plan.schema(); - assert_eq!( - ordering, - raw_series_frame_ordering(&planned_schema).unwrap() - ); + assert_eq!(ordering, series_frame_ordering(&planned_schema).unwrap()); { let state = state.lock().unwrap(); assert_eq!(0, state.raw_calls); @@ -783,7 +819,7 @@ async fn planned_ordering_is_preserved_for_empty_and_nonempty_frames() { ); assert_eq!( ordering, - raw_series_frame_ordering(stream.schema().as_ref()).unwrap() + series_frame_ordering(stream.schema().as_ref()).unwrap() ); let frames = stream.try_collect::>().await.unwrap(); assert_eq!(usize::from(nonempty), frames.len()); @@ -791,7 +827,7 @@ async fn planned_ordering_is_preserved_for_empty_and_nonempty_frames() { assert_eq!(planned_schema.as_ref(), frame.schema().as_ref()); assert_eq!( ordering, - raw_series_frame_ordering(frame.schema().as_ref()).unwrap() + series_frame_ordering(frame.schema().as_ref()).unwrap() ); } let state = state.lock().unwrap(); @@ -839,7 +875,7 @@ async fn planned_and_stream_ordering_mismatches_fail_without_legacy_fallback() { .unwrap(); assert_eq!( planned, - raw_series_frame_ordering(plan.schema().as_ref()).unwrap() + series_frame_ordering(plan.schema().as_ref()).unwrap() ); state.lock().unwrap().stream_ordering = Some(actual); let error = plan @@ -1032,18 +1068,17 @@ async fn malformed_returned_stream_binding_fails_before_poll_even_when_empty() { async fn primary_key_binding_mismatch_is_rejected_before_poll() { let logical_tags: Arc<[BoundTag]> = vec![ BoundTag { - column_id: 1, - name: Arc::from("zone"), + column_id: 7, + name: Arc::from("host"), }, BoundTag { - column_id: 2, + column_id: 8, name: Arc::from("rack"), }, ] .into(); for case in 0..6 { - let (scan, state) = fixture(Some(series_schema()), vec![]); - let metadata = scan.scanner.lock().unwrap().metadata(); + let (scan, state, metadata) = named_fixture_with_two_tags(); let bound = RawLabelSource::PrimaryKey { metric_name: Arc::from("cpu"), physical_metadata: metadata.clone(), @@ -1080,18 +1115,17 @@ async fn primary_key_binding_mismatch_is_rejected_before_poll() { #[tokio::test] async fn unchanged_primary_key_binding_accepts_empty_stream() { - let (scan, state) = fixture(Some(series_schema()), vec![]); - let metadata = scan.scanner.lock().unwrap().metadata(); + let (scan, state, metadata) = named_fixture_with_two_tags(); let source = RawLabelSource::PrimaryKey { metric_name: Arc::from("cpu"), physical_metadata: metadata, logical_tags: Arc::from([ BoundTag { - column_id: 1, - name: Arc::from("zone"), + column_id: 7, + name: Arc::from("host"), }, BoundTag { - column_id: 2, + column_id: 8, name: Arc::from("rack"), }, ]), @@ -1171,7 +1205,8 @@ async fn error_after_emitted_frame_is_not_replayed() { assert_eq!(frame.schema(), plan.schema()); let decoded = datatypes::series::frame::unpack_raw_series_frame( plan.series_schema(), - RawLabelMode::Explicit, + FrameLabelMode::Explicit, + None, &frame, ) .unwrap(); @@ -1354,7 +1389,8 @@ async fn tracked_native_frame_holds_and_releases_quota_on_cancel() { assert_eq!( datatypes::series::frame::unpack_raw_series_frame( plan.series_schema(), - RawLabelMode::Explicit, + FrameLabelMode::Explicit, + None, &frame, ) .unwrap()[0] @@ -1484,7 +1520,7 @@ async fn raw_series_admission_captures_scanner_ordering_before_execution() { .unwrap(); assert_eq!(plan.raw_series_ordering(), ordering); assert_eq!( - raw_series_frame_ordering(plan.schema().as_ref()).unwrap(), + series_frame_ordering(plan.schema().as_ref()).unwrap(), ordering ); { diff --git a/tests/compatibility/wire/raw_series_label_decode/README.md b/tests/compatibility/wire/raw_series_label_decode/README.md index a7d5c36858f..048b5bf54d9 100644 --- a/tests/compatibility/wire/raw_series_label_decode/README.md +++ b/tests/compatibility/wire/raw_series_label_decode/README.md @@ -1,13 +1,14 @@ # Raw-series label decode description compatibility -`v1_metadata.json` is a hand-authored, frozen set of Arrow schema metadata maps -for the v1 raw-series frame. It records the established layout/content/labels/ -time metadata plus the new `greptime.series.label_decode` entry, whose value is a -JSON label decode description with an explicit `version`. The Rust unit test -parses these literal maps through `raw_series_frame_label_decode`; the fixture is -not produced by the schema builder. `v1_physical_types.json`, described under +`v2_metadata.json` is a hand-authored, frozen set of Arrow schema metadata maps +for the v2 series frame. It records the established layout/content/labels/time +metadata plus the `greptime.series.label_decode` entry, whose value is a JSON +label decode description with an explicit `version`. The Rust unit test parses +these literal maps through `series_frame_label_decode`; the fixture is not +produced by the schema builder. `v1_physical_types.json`, described under [Physical types](#physical-types), is the one generated fixture here: it records -the serde text of every `ConcreteDataType` variant. +the serde text of every `ConcreteDataType` variant, which the frame layout version +does not change, so it keeps its original name. Run the focused check: @@ -22,6 +23,17 @@ column, and that the reader rejects a key frame without a description, an explic frame that declares one, an unknown version, a missing or unknown field, a wrong field type, and a non-JSON value. +It also pins the declarations the unified frame family refuses. Layout is now +`v2`: a frame declaring `v1` or any other value is refused, and so is a +`greptime.series.*` key this revision does not register. The three modes are not +independent - `content` fixes `time`, and `labels` decides whether a description +must be present - so `content = evaluated` with `time = timestamps`, `content = +evaluated` with `labels = key`, and `content = raw` with a +`greptime.series.grid` are all contradictions and are refused rather than +half-trusted. An evaluated frame is therefore always `explicit` + `grid_spans` and +always declares its grid; a raw frame is always `timestamps` and never declares +one. + A timestamp-typed tag column has its own case in the crate's unit tests rather than here, so this fixture stays a single fixed layout. diff --git a/tests/compatibility/wire/raw_series_label_decode/v1_metadata.json b/tests/compatibility/wire/raw_series_label_decode/v2_metadata.json similarity index 52% rename from tests/compatibility/wire/raw_series_label_decode/v1_metadata.json rename to tests/compatibility/wire/raw_series_label_decode/v2_metadata.json index 2e4e6d7f24b..43ded1e4bf6 100644 --- a/tests/compatibility/wire/raw_series_label_decode/v1_metadata.json +++ b/tests/compatibility/wire/raw_series_label_decode/v2_metadata.json @@ -2,109 +2,181 @@ { "name": "key_frame_declares_dense_description", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps", - "greptime.series.label_decode": "{\"version\":1,\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}" + "greptime.series.label_decode": "{\"version\":1,\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}", + "greptime.series.ordering": "contiguous" }, "encoding": "dense" }, { "name": "key_frame_declares_sparse_description", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps", - "greptime.series.label_decode": "{\"version\":1,\"metric_name\":\"cpu_usage\",\"encoding\":\"sparse\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}" + "greptime.series.label_decode": "{\"version\":1,\"metric_name\":\"cpu_usage\",\"encoding\":\"sparse\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}", + "greptime.series.ordering": "contiguous" }, "encoding": "sparse" }, { "name": "missing_description_rejected", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", - "greptime.series.time": "timestamps" + "greptime.series.time": "timestamps", + "greptime.series.ordering": "contiguous" }, "encoding": null }, { "name": "unknown_version_rejected", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps", - "greptime.series.label_decode": "{\"version\":2,\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}" + "greptime.series.label_decode": "{\"version\":2,\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}", + "greptime.series.ordering": "contiguous" }, "encoding": null }, { "name": "missing_field_rejected", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps", - "greptime.series.label_decode": "{\"version\":1,\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}" + "greptime.series.label_decode": "{\"version\":1,\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}", + "greptime.series.ordering": "contiguous" }, "encoding": null }, { "name": "unknown_field_rejected", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps", - "greptime.series.label_decode": "{\"version\":1,\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}],\"future\":0}" + "greptime.series.label_decode": "{\"version\":1,\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}],\"future\":0}", + "greptime.series.ordering": "contiguous" }, "encoding": null }, { "name": "wrong_field_type_rejected", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps", - "greptime.series.label_decode": "{\"version\":\"1\",\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}" + "greptime.series.label_decode": "{\"version\":\"1\",\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}", + "greptime.series.ordering": "contiguous" }, "encoding": null }, { "name": "unlisted_logical_tag_rejected", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps", - "greptime.series.label_decode": "{\"version\":1,\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}" + "greptime.series.label_decode": "{\"version\":1,\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}", + "greptime.series.ordering": "contiguous" }, "encoding": null }, { "name": "explicit_frame_with_description_rejected", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "explicit", "greptime.series.time": "timestamps", - "greptime.series.label_decode": "{\"version\":1,\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}" + "greptime.series.label_decode": "{\"version\":1,\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}", + "greptime.series.ordering": "contiguous" }, "encoding": null }, { "name": "malformed_json_rejected", "metadata": { - "greptime.series.layout": "v1", + "greptime.series.layout": "v2", "greptime.series.content": "raw", "greptime.series.labels": "key", "greptime.series.time": "timestamps", - "greptime.series.label_decode": "greptime" + "greptime.series.label_decode": "greptime", + "greptime.series.ordering": "contiguous" + }, + "encoding": null + }, + { + "name": "wrong_layout_rejected", + "metadata": { + "greptime.series.layout": "v3", + "greptime.series.content": "raw", + "greptime.series.labels": "key", + "greptime.series.time": "timestamps", + "greptime.series.label_decode": "{\"version\":1,\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}", + "greptime.series.ordering": "contiguous" + }, + "encoding": null + }, + { + "name": "unregistered_reserved_metadata_rejected", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "raw", + "greptime.series.labels": "key", + "greptime.series.time": "timestamps", + "greptime.series.label_decode": "{\"version\":1,\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}", + "greptime.series.future": "x", + "greptime.series.ordering": "contiguous" + }, + "encoding": null + }, + { + "name": "evaluated_with_timestamps_rejected", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "evaluated", + "greptime.series.labels": "explicit", + "greptime.series.time": "timestamps", + "greptime.series.grid": "{\"start_ms\":0,\"step_ms\":10000,\"len\":4}", + "greptime.series.ordering": "contiguous" + }, + "encoding": null + }, + { + "name": "evaluated_with_key_labels_rejected", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "evaluated", + "greptime.series.labels": "key", + "greptime.series.time": "grid_spans", + "greptime.series.label_decode": "{\"version\":1,\"metric_name\":\"cpu_usage\",\"encoding\":\"dense\",\"primary_key\":[{\"column_id\":1,\"name\":\"host\",\"physical_type\":{\"String\":{\"size_type\":\"Utf8\"}}},{\"column_id\":2,\"name\":\"shard\",\"physical_type\":{\"UInt32\":{}}}],\"logical_tags\":[{\"column_id\":1,\"name\":\"host\"},{\"column_id\":2,\"name\":\"shard\"}]}", + "greptime.series.grid": "{\"start_ms\":0,\"step_ms\":10000,\"len\":4}", + "greptime.series.ordering": "contiguous" + }, + "encoding": null + }, + { + "name": "raw_with_grid_rejected", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "raw", + "greptime.series.labels": "explicit", + "greptime.series.time": "timestamps", + "greptime.series.grid": "{\"start_ms\":0,\"step_ms\":10000,\"len\":4}", + "greptime.series.ordering": "contiguous" }, "encoding": null } diff --git a/tests/compatibility/wire/raw_series_ordering/README.md b/tests/compatibility/wire/raw_series_ordering/README.md index 630dc25e2a2..2ddc1463dd2 100644 --- a/tests/compatibility/wire/raw_series_ordering/README.md +++ b/tests/compatibility/wire/raw_series_ordering/README.md @@ -1,10 +1,11 @@ # Raw-series ordering schema metadata compatibility -`v1_metadata.json` is a hand-authored, frozen set of Arrow schema metadata maps -for the v1 raw-series frame. It records the established layout/content/labels/ -time metadata and the new optional ordering entry. The Rust unit test parses -these literal maps through `raw_series_frame_ordering`; the fixture is not -produced by the schema builder. +`v2_metadata.json` is a hand-authored, frozen set of Arrow schema metadata maps +for the v2 series frame. It records the established layout/content/labels/time +metadata and mandatory ordering entry, for raw frames and for evaluated ones. +The Rust unit test parses these literal maps through `series_frame_ordering` and +through `SeriesFrameDeclaration::parse`; the fixture is not produced by the +schema builder. Run the focused check: @@ -12,8 +13,25 @@ Run the focused check: cargo nextest run -p common-query -E 'test(raw_series_ordering_metadata_fixture)' ``` -The fixture proves that absent ordering metadata defaults to `Contiguous`, the -two known values decode, and an unknown value is rejected. It does not prove -interoperability with an old binary, nor does it test an end-to-end client or -service. Separate pack/IPC tests check the declared ordering schema and batch -schema agreement for both orderings, including empty streams. +The fixture proves that missing ordering metadata and unknown values are rejected, +the two known values decode, and that a whole declaration is accepted or refused +consistently with its layout and content: a +`v3` layout, an unregistered `greptime.series.*` key, a raw frame that declares a +grid, and any content/time or content/labels contradiction are refusals. +Evaluated declarations use explicit `contiguous` ordering; streaming round trips +for evaluated frames are covered by the crate's evaluated tests. + +Because the declaration is cross-checked, a frame that names `evaluated` content +is only an evaluated frame when it also declares `explicit` labels, `grid_spans` +time, and a `greptime.series.grid`. `read_key_then_time` parses as a declaration +(a raw frame may legitimately claim it), but no writer produces one on an +evaluated frame: computed values have no storage key to be ordered by, so both the +schema builder and `unpack_evaluated_series` refuse it. The fixture keeps a +hand-written evaluated frame that states it and verifies that declaration +parsing refuses it. + +The fixture also pins rejection of evaluated `read_key_then_time` metadata. It +does not prove interoperability with an old binary or test an end-to-end client +or service. Separate pack/IPC tests check the declared +ordering schema and batch schema agreement for both orderings, including empty +streams, and check the same for the evaluated frame across Arrow IPC. diff --git a/tests/compatibility/wire/raw_series_ordering/v1_metadata.json b/tests/compatibility/wire/raw_series_ordering/v1_metadata.json deleted file mode 100644 index c5b3c0ea69b..00000000000 --- a/tests/compatibility/wire/raw_series_ordering/v1_metadata.json +++ /dev/null @@ -1,45 +0,0 @@ -[ - { - "name": "missing_defaults_to_contiguous", - "metadata": { - "greptime.series.layout": "v1", - "greptime.series.content": "raw", - "greptime.series.labels": "explicit", - "greptime.series.time": "timestamps" - }, - "ordering": "contiguous" - }, - { - "name": "known_contiguous", - "metadata": { - "greptime.series.layout": "v1", - "greptime.series.content": "raw", - "greptime.series.labels": "explicit", - "greptime.series.time": "timestamps", - "greptime.series.ordering": "contiguous" - }, - "ordering": "contiguous" - }, - { - "name": "known_read_key_then_time", - "metadata": { - "greptime.series.layout": "v1", - "greptime.series.content": "raw", - "greptime.series.labels": "explicit", - "greptime.series.time": "timestamps", - "greptime.series.ordering": "read_key_then_time" - }, - "ordering": "read_key_then_time" - }, - { - "name": "unknown_ordering_rejected", - "metadata": { - "greptime.series.layout": "v1", - "greptime.series.content": "raw", - "greptime.series.labels": "explicit", - "greptime.series.time": "timestamps", - "greptime.series.ordering": "future_value" - }, - "ordering": null - } -] diff --git a/tests/compatibility/wire/raw_series_ordering/v2_metadata.json b/tests/compatibility/wire/raw_series_ordering/v2_metadata.json new file mode 100644 index 00000000000..4b01c12091d --- /dev/null +++ b/tests/compatibility/wire/raw_series_ordering/v2_metadata.json @@ -0,0 +1,100 @@ +[ + { + "name": "missing_ordering_rejected", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "raw", + "greptime.series.labels": "explicit", + "greptime.series.time": "timestamps" + }, + "ordering": null, + "declaration": "rejected" + }, + { + "name": "known_contiguous", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "raw", + "greptime.series.labels": "explicit", + "greptime.series.time": "timestamps", + "greptime.series.ordering": "contiguous" + }, + "ordering": "contiguous", + "declaration": "accepted" + }, + { + "name": "known_read_key_then_time", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "raw", + "greptime.series.labels": "explicit", + "greptime.series.time": "timestamps", + "greptime.series.ordering": "read_key_then_time" + }, + "ordering": "read_key_then_time", + "declaration": "accepted" + }, + { + "name": "unknown_ordering_rejected", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "raw", + "greptime.series.labels": "explicit", + "greptime.series.time": "timestamps", + "greptime.series.ordering": "future_value" + }, + "ordering": null, + "declaration": "rejected" + }, + { + "name": "evaluated_contiguous", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "evaluated", + "greptime.series.labels": "explicit", + "greptime.series.time": "grid_spans", + "greptime.series.grid": "{\"start_ms\":0,\"step_ms\":10000,\"len\":4}", + "greptime.series.ordering": "contiguous" + }, + "ordering": "contiguous", + "declaration": "accepted" + }, + { + "name": "evaluated_read_key_then_time_rejected", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "evaluated", + "greptime.series.labels": "explicit", + "greptime.series.time": "grid_spans", + "greptime.series.grid": "{\"start_ms\":0,\"step_ms\":10000,\"len\":4}", + "greptime.series.ordering": "read_key_then_time" + }, + "ordering": "read_key_then_time", + "declaration": "rejected" + }, + { + "name": "wrong_layout_rejected", + "metadata": { + "greptime.series.layout": "v3", + "greptime.series.content": "raw", + "greptime.series.labels": "explicit", + "greptime.series.time": "timestamps", + "greptime.series.ordering": "contiguous" + }, + "ordering": "contiguous", + "declaration": "rejected" + }, + { + "name": "unregistered_reserved_metadata_rejected", + "metadata": { + "greptime.series.layout": "v2", + "greptime.series.content": "raw", + "greptime.series.labels": "explicit", + "greptime.series.time": "timestamps", + "greptime.series.future": "x", + "greptime.series.ordering": "contiguous" + }, + "ordering": "contiguous", + "declaration": "rejected" + } +]