diff --git a/src/common/query/src/series_frame.rs b/src/common/query/src/series_frame.rs index aa344d5a775..5fb942ca156 100644 --- a/src/common/query/src/series_frame.rs +++ b/src/common/query/src/series_frame.rs @@ -43,10 +43,11 @@ use common_recordbatch::{DfRecordBatch, DfSendableRecordBatchStream}; use datafusion_common::DataFusionError; use datatypes::arrow::datatypes::{Schema as ArrowSchema, SchemaRef as ArrowSchemaRef}; use datatypes::error::InvalidSeriesSnafu; +use datatypes::series::aggregate_state::{AggregateStateSchema, StateSeries}; use datatypes::series::frame::{ FrameContent, FrameLabelMode, LABEL_DECODE_VERSION, LabelDecodeColumn, LabelDecodeEncoding, LabelDecodeSpec, LabelDecodeTag, SeriesFrameBuilder, SeriesFrameCursor, SeriesFrameDeclaration, - SeriesFrameDecoder, series_frame_label_decode, series_frame_schema, + SeriesFrameDecoder, StateFrameRow, series_frame_label_decode, series_frame_schema, }; use datatypes::series::raw::RawSeriesChunk; use datatypes::series::samples::{Series, SeriesSchema}; @@ -71,6 +72,18 @@ const ORDERING_KEY: &str = "greptime.series.ordering"; pub type SendableEvaluatedSeriesStream = Pin> + Send>>; +/// A stream of aggregate-state chunks and their producer-local finality. +/// +/// Export accepts the existing field-major [`StateSeries`] representation and columnarises each +/// chunk once. Import is intentionally asymmetric: [`SendableStateFrameStream`] retains typed Arrow +/// handles so consumers do not rebuild a nested state tree. +pub type SendableStateSeriesStream = + Pin> + Send>>; + +/// A stream of decoded state rows retaining typed handles into their Arrow batch. +pub type SendableStateFrameStream = + Pin> + Send>>; + /// Builds the canonical raw frame schema with the producer's label binding and ordering. /// /// Ordering records a producer guarantee; it is not inferred from frame contents. `Interleaved` @@ -282,6 +295,124 @@ pub fn pack_evaluated_series( })) } +/// Packs aggregate-state chunks into bounded DataFusion record batches. +/// +/// The producer supplies one [`StateSeries`] per group chunk; the cursor columnarises it into the +/// frame once. Aggregate state requires a grid, explicit labels and contiguous ordering. The state +/// schema is revalidated with its own constructor before any stream is returned. +pub fn pack_aggregate_state( + source: SendableStateSeriesStream, + state: Arc, + schema: Arc, +) -> datafusion_common::Result { + if schema.eval_grid.is_none() { + return Err(invalid("an aggregate state frame needs an evaluation grid")); + } + AggregateStateSchema::try_new(state.kind, state.components.to_vec()).map_err(codec_error)?; + let frame_schema = datatypes::series::frame::aggregate_state_frame_schema( + &state, + &schema, + FrameLabelMode::Explicit, + ) + .map_err(codec_error)?; + let mut metadata = frame_schema.metadata().clone(); + metadata.insert(ORDERING_KEY.to_string(), "contiguous".to_string()); + let frame_schema = Arc::new(ArrowSchema::new_with_metadata( + frame_schema.fields().clone(), + metadata, + )); + let builder = SeriesFrameBuilder::try_new_aggregate_state( + &state, + &schema, + FrameLabelMode::Explicit, + SeriesFrameBuilder::DEFAULT_MAX_ROWS, + SeriesFrameBuilder::DEFAULT_MAX_BYTES, + ) + .map_err(codec_error)?; + let decoder = + SeriesFrameDecoder::try_new_aggregate_state(&state, &schema, FrameLabelMode::Explicit) + .map_err(codec_error)?; + // Decoding the empty batch validates the exact declared layout before publishing this stream. + decoder + .decode_aggregate_state(&DfRecordBatch::new_empty(frame_schema.clone())) + .map_err(codec_error)?; + Ok(Box::pin(PackStateStream { + source: Some(source), + state, + schema, + batches: PackBatches::new(builder, frame_schema), + })) +} + +/// Unpacks aggregate-state frames while retaining Arrow-backed state rows. +/// +/// Kind and ordered components must both agree: the kind describes the accumulator semantics while +/// components describe its physical values. Expected fields and grid are checked independently. +pub fn unpack_aggregate_state( + source: DfSendableRecordBatchStream, + expected_state: Option>, + expected_series: Option>, +) -> datafusion_common::Result { + let frame_schema = source.schema(); + let declaration = SeriesFrameDeclaration::parse(frame_schema.as_ref()).map_err(codec_error)?; + if declaration.content() != FrameContent::AggregateState { + return Err(invalid(format!( + "expected an aggregate state frame, found {}", + declaration.content().as_str() + ))); + } + ensure_contiguous_ordering(series_frame_ordering(frame_schema.as_ref())?)?; + let state = declaration + .aggregate_state() + .ok_or_else(|| invalid("aggregate state declaration has no aggregate state"))?; + if let Some(expected) = expected_state { + if state.kind != expected.kind { + return Err(invalid( + "aggregate state kind does not match expected state", + )); + } + if state.components != expected.components { + return Err(invalid( + "aggregate state components do not match expected state", + )); + } + } + let schema = Arc::new(declaration.series().clone()); + if let Some(expected) = expected_series { + if expected.fields.len() != schema.fields.len() + || expected + .fields + .iter() + .zip(schema.fields.iter()) + .any(|(a, b)| a.name != b.name) + { + return Err(invalid( + "aggregate state fields do not match expected series", + )); + } + if expected.eval_grid.as_ref().map(|grid| grid.spec()) + != schema.eval_grid.as_ref().map(|grid| grid.spec()) + { + return Err(invalid( + "aggregate state grid does not match expected series", + )); + } + } + let decoder = + SeriesFrameDecoder::try_new_aggregate_state(state, &schema, FrameLabelMode::Explicit) + .map_err(codec_error)?; + // Validate even empty streams before returning them, rather than deferring declaration errors. + decoder + .decode_aggregate_state(&DfRecordBatch::new_empty(frame_schema.clone())) + .map_err(codec_error)?; + Ok(Box::pin(UnpackStateStream { + source: Some(source), + frame_schema, + pending: Vec::new().into_iter(), + decoder, + })) +} + /// Unpacks DataFusion batches declared by [`pack_raw_series`]. /// /// The frame is the sole source of decode information: the payload fields, the label mode, the @@ -357,7 +488,10 @@ pub fn unpack_evaluated_series( let frame_schema = source.schema(); let declaration = SeriesFrameDeclaration::parse(frame_schema.as_ref()).map_err(codec_error)?; if declaration.content() != FrameContent::Evaluated { - return Err(invalid("expected an evaluated frame, found a raw one")); + return Err(invalid(format!( + "expected an evaluated frame, found {}", + declaration.content().as_str() + ))); } ensure_contiguous_ordering(series_frame_ordering(frame_schema.as_ref())?)?; let schema = Arc::new(declaration.series().clone()); @@ -809,6 +943,166 @@ impl datafusion::physical_plan::RecordBatchStream for PackStream { } } +/// The pack stream of [`pack_aggregate_state`]. +struct PackStateStream { + source: Option, + state: Arc, + schema: Arc, + batches: PackBatches, +} + +impl PackStateStream { + fn finish(&mut self) { + self.source = None; + self.batches.release(); + } +} + +impl Stream for PackStateStream { + type Item = datafusion_common::Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = &mut *self; + loop { + match this.batches.advance() { + Ok(Some(batch)) => return this.batches.declare(batch), + Ok(None) => {} + Err(error) => { + this.finish(); + return Poll::Ready(Some(Err(codec_error(error)))); + } + } + let polled = match this.source.as_mut() { + Some(source) => source.as_mut().poll_next(cx), + None => return Poll::Ready(None), + }; + match polled { + Poll::Pending => { + return match this.batches.flush() { + Ok(Some(batch)) => this.batches.declare(batch), + Ok(None) => Poll::Pending, + Err(error) => { + this.finish(); + Poll::Ready(Some(Err(codec_error(error)))) + } + }; + } + Poll::Ready(None) => { + let flushed = this.batches.flush(); + let declared = this.batches.frame_schema.clone(); + this.finish(); + return match flushed { + Ok(Some(batch)) => { + match DfRecordBatch::try_new(declared, batch.columns().to_vec()) { + Ok(batch) => Poll::Ready(Some(Ok(batch))), + Err(error) => Poll::Ready(Some(Err(DataFusionError::from(error)))), + } + } + Ok(None) => Poll::Ready(None), + Err(error) => Poll::Ready(Some(Err(codec_error(error)))), + }; + } + Poll::Ready(Some(Err(error))) => { + this.finish(); + return Poll::Ready(Some(Err(upstream_error(error)))); + } + Poll::Ready(Some(Ok((series, is_last)))) => { + match SeriesFrameCursor::try_new_aggregate_state( + &this.state, + &this.schema, + series, + is_last, + ) { + Ok(cursor) => this.batches.install(cursor), + Err(error) => { + this.finish(); + return Poll::Ready(Some(Err(codec_error(error)))); + } + } + } + } + } + } +} + +impl datafusion::physical_plan::RecordBatchStream for PackStateStream { + fn schema(&self) -> ArrowSchemaRef { + self.batches.frame_schema.clone() + } +} + +/// The unpack stream of [`unpack_aggregate_state`]. +struct UnpackStateStream { + source: Option, + frame_schema: ArrowSchemaRef, + pending: std::vec::IntoIter, + decoder: SeriesFrameDecoder, +} + +impl UnpackStateStream { + fn finish(&mut self) { + self.source = None; + self.pending = Vec::new().into_iter(); + } +} + +impl Stream for UnpackStateStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = &mut *self; + let mut empty_batches = 0; + loop { + if let Some(row) = this.pending.next() { + return Poll::Ready(Some(Ok(row))); + } + let polled = match this.source.as_mut() { + Some(source) => source.as_mut().poll_next(cx), + None => return Poll::Ready(None), + }; + match polled { + Poll::Pending => return Poll::Pending, + Poll::Ready(None) => { + this.finish(); + return Poll::Ready(None); + } + Poll::Ready(Some(Err(error))) => { + this.finish(); + return Poll::Ready(Some(Err(BoxedError::new( + crate::error::Error::GeneralDataFusion { + error, + location: snafu::Location::default(), + }, + )))); + } + Poll::Ready(Some(Ok(batch))) => { + let result = if batch.schema() != this.frame_schema { + Err(invalid("aggregate state frame schema mismatch")) + } else { + this.decoder + .decode_aggregate_state(&batch) + .map_err(codec_error) + }; + match result { + Ok(rows) if rows.is_empty() => { + empty_batches += 1; + if empty_batches == MAX_EMPTY_BATCHES_PER_POLL { + cx.waker().wake_by_ref(); + return Poll::Pending; + } + } + Ok(rows) => this.pending = rows.into_iter(), + Err(error) => { + this.finish(); + return Poll::Ready(Some(Err(unpack_error(error)))); + } + } + } + } + } + } +} + /// The pack stream of [`pack_evaluated_series`]. struct PackEvaluatedStream { source: Option, diff --git a/src/common/query/src/series_frame/tests.rs b/src/common/query/src/series_frame/tests.rs index 659a83566b8..5f2ccc725ee 100644 --- a/src/common/query/src/series_frame/tests.rs +++ b/src/common/query/src/series_frame/tests.rs @@ -14,6 +14,7 @@ mod evaluated; mod lifecycle; +mod state; use std::collections::{HashMap, VecDeque}; use std::io::Cursor; diff --git a/src/common/query/src/series_frame/tests/state.rs b/src/common/query/src/series_frame/tests/state.rs new file mode 100644 index 00000000000..0f4fa0ee059 --- /dev/null +++ b/src/common/query/src/series_frame/tests/state.rs @@ -0,0 +1,391 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Cursor; +use std::sync::Arc; + +use common_error::ext::BoxedError; +use common_recordbatch::DfSendableRecordBatchStream; +use datatypes::arrow::compute::concat_batches; +use datatypes::arrow::ipc::{reader::StreamReader, writer::StreamWriter}; +use datatypes::arrow::record_batch::RecordBatch; +use datatypes::series::aggregate_state::{ + AggregateStateKind, AggregateStateSchema, AggregateStateValues, StateComponent, StateSeries, +}; +use datatypes::series::frame::{FrameLabelMode, StateFrameRow}; +use datatypes::series::samples::{FieldSpec, SeriesSchema}; +use datatypes::series::time::{GridSpan, TimeAxis}; +use datatypes::series::{Label, SeriesLabels}; +use futures_util::StreamExt; + +use super::super::{ + SendableStateSeriesStream, pack_aggregate_state, pack_evaluated_series, unpack_aggregate_state, + unpack_evaluated_series, +}; +use super::{df_source, evaluated_grid, evaluated_schema}; + +fn group() -> SeriesLabels { + SeriesLabels::try_new( + Some(Arc::from("cpu")), + vec![Label { + name: Arc::from("host"), + value: Arc::from("a"), + }], + ) + .unwrap() +} + +fn state_schema(kind: AggregateStateKind) -> Arc { + let components = match kind { + AggregateStateKind::Group => vec![], + AggregateStateKind::Sum => vec![StateComponent::Value, StateComponent::Compensation], + AggregateStateKind::Count => vec![StateComponent::Count], + AggregateStateKind::Avg => vec![ + StateComponent::Value, + StateComponent::Mean, + StateComponent::Compensation, + StateComponent::Count, + StateComponent::IncrementalMean, + ], + AggregateStateKind::Min | AggregateStateKind::Max => vec![StateComponent::Value], + AggregateStateKind::Stddev | AggregateStateKind::Stdvar => vec![ + StateComponent::Value, + StateComponent::Mean, + StateComponent::Count, + ], + }; + Arc::new(AggregateStateSchema::try_new(kind, components).unwrap()) +} + +fn series_schema(fields: &[&str]) -> Arc { + Arc::new(SeriesSchema { + fields: fields + .iter() + .map(|name| FieldSpec { + name: Arc::from(*name), + }) + .collect(), + eval_grid: Some(evaluated_grid()), + }) +} + +fn wrong_grid_schema() -> Arc { + Arc::new(SeriesSchema { + fields: vec![FieldSpec { + name: Arc::from("a"), + }] + .into(), + eval_grid: Some(Arc::new( + datatypes::series::time::EvalGrid::try_new( + datatypes::series::time::GridSpec::try_new(1, 1000, 512).unwrap(), + ) + .unwrap(), + )), + }) +} + +fn input(fields: Vec>>) -> (StateSeries, bool) { + let time = TimeAxis::from_grid_spans(evaluated_grid(), vec![GridSpan::new(1, 3)]).unwrap(); + ( + StateSeries { + metric: group(), + time, + states: fields, + }, + true, + ) +} + +fn source(items: Vec<(StateSeries, bool)>) -> SendableStateSeriesStream { + Box::pin(futures_util::stream::iter( + items.into_iter().map(Ok::<_, BoxedError>), + )) +} + +async fn packed( + rows: Vec<(StateSeries, bool)>, + state: Arc, + schema: Arc, +) -> Vec { + let mut stream = pack_aggregate_state(source(rows), state, schema).unwrap(); + let mut batches = Vec::new(); + while let Some(batch) = stream.next().await { + batches.push(batch.unwrap()); + } + batches +} + +fn unpack_source( + batches: Vec, + schema: Arc, +) -> DfSendableRecordBatchStream { + df_source(batches, schema).0 +} + +async fn rows( + batches: Vec, + state: Option>, + schema: Option>, +) -> Vec { + let frame = batches + .first() + .map(|batch| batch.schema()) + .unwrap_or_else(|| { + datatypes::series::frame::aggregate_state_frame_schema( + state.as_ref().unwrap(), + schema.as_ref().unwrap(), + FrameLabelMode::Explicit, + ) + .unwrap() + }); + let mut stream = unpack_aggregate_state(unpack_source(batches, frame), state, schema).unwrap(); + let mut rows = Vec::new(); + while let Some(row) = stream.next().await { + rows.push(row.unwrap()); + } + rows +} + +fn values(value: f64) -> Option { + Some(AggregateStateValues { + value, + compensation: value / 10.0, + ..Default::default() + }) +} + +fn assert_state_row(row: &StateFrameRow, expected: &[f64], last: bool) { + assert_eq!(row.metric(), &group()); + assert!(row.is_last() == last); + assert_eq!( + row.time() + .positions() + .map(|(position, _)| position) + .collect::>(), + vec![1, 2, 3] + ); + let fields = row.fields().unwrap(); + for (position, value) in expected.iter().enumerate() { + assert!(fields.is_present(0, position).unwrap()); + assert_eq!( + fields + .field_values(0, StateComponent::Value) + .unwrap() + .unwrap() + .value(position), + *value + ); + } +} + +fn assert_group_row(row: &StateFrameRow, last: bool) { + assert_eq!(row.metric(), &group()); + assert_eq!(row.is_last(), last); + assert_eq!( + row.time() + .positions() + .map(|(position, _)| position) + .collect::>(), + vec![1, 2, 3] + ); + let fields = row.fields().unwrap(); + for field in 0..fields.len() { + for position in 0..3 { + assert!(fields.is_present(field, position).unwrap()); + } + } +} + +#[tokio::test] +async fn aggregate_state_roundtrip_preserves_components_and_group_rows() { + let sum = state_schema(AggregateStateKind::Sum); + let schema = series_schema(&["a"]); + let input_rows = vec![input(vec![vec![values(1.0), values(2.0), values(3.0)]])]; + let batches = packed(input_rows, sum.clone(), schema.clone()).await; + let decoded = rows(batches, Some(sum), Some(schema)).await; + assert_eq!(decoded.len(), 1); + assert_state_row(&decoded[0], &[1.0, 2.0, 3.0], true); + + let group_state = state_schema(AggregateStateKind::Group); + let group_schema = series_schema(&["a"]); + let group_batches = packed( + vec![input(vec![vec![values(0.0), values(0.0), values(0.0)]])], + group_state.clone(), + group_schema.clone(), + ) + .await; + let group_rows = rows(group_batches, Some(group_state), Some(group_schema)).await; + assert_eq!(group_rows.len(), 1); + assert_group_row(&group_rows[0], true); +} + +#[tokio::test] +async fn group_presence_is_per_field() { + let state = state_schema(AggregateStateKind::Group); + let schema = series_schema(&["a", "b"]); + let batches = packed( + vec![input(vec![ + vec![values(0.0), None, values(0.0)], + vec![None, values(0.0), None], + ])], + state.clone(), + schema.clone(), + ) + .await; + let decoded = rows(batches, Some(state), Some(schema)).await; + let fields = decoded[0].fields().unwrap(); + assert_eq!( + (0..3) + .map(|i| fields.is_present(0, i).unwrap()) + .collect::>(), + vec![true, false, true] + ); + assert_eq!( + (0..3) + .map(|i| fields.is_present(1, i).unwrap()) + .collect::>(), + vec![false, true, false] + ); +} + +#[tokio::test] +async fn aggregate_state_ipc_concat_roundtrip() { + let state = state_schema(AggregateStateKind::Sum); + let schema = series_schema(&["a"]); + let batches = packed( + vec![input(vec![vec![values(1.0), values(2.0), values(3.0)]])], + state.clone(), + schema.clone(), + ) + .await; + let expected = rows(batches.clone(), Some(state.clone()), Some(schema.clone())).await; + let mut bytes = Cursor::new(Vec::new()); + { + let mut writer = StreamWriter::try_new(&mut bytes, batches[0].schema().as_ref()).unwrap(); + for batch in &batches { + writer.write(batch).unwrap(); + } + writer.finish().unwrap(); + } + bytes.set_position(0); + let reader = StreamReader::try_new(bytes, None).unwrap(); + let read = reader.map(Result::unwrap).collect::>(); + let concatenated = concat_batches(&batches[0].schema(), &read).unwrap(); + let decoded = rows(vec![concatenated], Some(state), Some(schema)).await; + assert_eq!(decoded.len(), expected.len()); + assert_state_row(&decoded[0], &[1.0, 2.0, 3.0], true); +} + +#[tokio::test] +async fn aggregate_state_refusals_name_mismatched_parts() { + let state = state_schema(AggregateStateKind::Sum); + let schema = series_schema(&["a"]); + let batches = packed( + vec![input(vec![vec![values(1.0), values(2.0), values(3.0)]])], + state.clone(), + schema.clone(), + ) + .await; + for (expected, expected_schema, part) in [ + ( + Some(state_schema(AggregateStateKind::Count)), + Some(schema.clone()), + "kind", + ), + ( + Some(Arc::new(AggregateStateSchema { + kind: AggregateStateKind::Sum, + components: vec![StateComponent::Compensation, StateComponent::Value].into(), + })), + Some(schema.clone()), + "components", + ), + (Some(state.clone()), Some(wrong_grid_schema()), "grid"), + ( + Some(state.clone()), + Some(series_schema(&["a", "b"])), + "fields", + ), + ] { + let frame = batches[0].schema(); + let (source, _, _) = df_source(batches.clone(), frame); + let error = unpack_aggregate_state(source, expected, expected_schema) + .err() + .unwrap() + .to_string(); + assert!( + error.to_lowercase().contains(part), + "{error} did not name {part}" + ); + } +} + +#[tokio::test] +async fn aggregate_state_rejects_evaluated_and_evaluated_rejects_state() { + let state = state_schema(AggregateStateKind::Sum); + let schema = series_schema(&["a"]); + let batches = packed( + vec![input(vec![vec![values(1.0), values(2.0), values(3.0)]])], + state, + schema, + ) + .await; + let eval = + pack_evaluated_series(Box::pin(futures_util::stream::empty()), evaluated_schema()).unwrap(); + let frame = eval.schema(); + let (source, _, _) = df_source(vec![], frame.clone()); + assert!( + unpack_aggregate_state(source, None, None) + .err() + .unwrap() + .to_string() + .contains("evaluated") + ); + let (source, _, _) = df_source(batches.clone(), batches[0].schema()); + assert!( + unpack_evaluated_series(source, None) + .err() + .unwrap() + .to_string() + .contains("aggregate_state") + ); + let eval = + pack_evaluated_series(Box::pin(futures_util::stream::empty()), evaluated_schema()).unwrap(); + let frame = eval.schema(); + let (source, _, _) = df_source(vec![], frame); + assert!( + unpack_aggregate_state(source, None, None) + .err() + .unwrap() + .to_string() + .contains("evaluated") + ); +} + +#[tokio::test] +async fn aggregate_state_empty_stream_terminates() { + let state = state_schema(AggregateStateKind::Sum); + let schema = series_schema(&["a"]); + let batches = packed(vec![], state.clone(), schema.clone()).await; + assert!(batches.is_empty()); + let frame = datatypes::series::frame::aggregate_state_frame_schema( + &state, + &schema, + datatypes::series::frame::FrameLabelMode::Explicit, + ) + .unwrap(); + let (source, _, _) = df_source(vec![], frame); + let mut decoded = unpack_aggregate_state(source, Some(state), Some(schema)).unwrap(); + assert!(decoded.next().await.is_none()); +} diff --git a/src/datatypes/src/series.rs b/src/datatypes/src/series.rs index 81d0dd4750b..08f4b36bc3b 100644 --- a/src/datatypes/src/series.rs +++ b/src/datatypes/src/series.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod aggregate_state; pub mod frame; pub mod histogram; pub mod raw; diff --git a/src/datatypes/src/series/aggregate_state.rs b/src/datatypes/src/series/aggregate_state.rs new file mode 100644 index 00000000000..7dd072f8290 --- /dev/null +++ b/src/datatypes/src/series/aggregate_state.rs @@ -0,0 +1,328 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use arrow::array::{Array, BooleanArray, Float64Array, ListArray, StructArray}; +use arrow::datatypes::DataType; + +use crate::error::{InvalidSeriesSnafu, Result}; +use crate::series::SeriesLabels; +use crate::series::time::TimeAxis; + +/// Aggregate whose accumulator is represented by a state frame. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub enum AggregateStateKind { + Sum, + Count, + Avg, + Min, + Max, + Group, + Stddev, + Stdvar, +} + +impl AggregateStateKind { + /// Canonical metadata spelling. + pub const fn as_str(self) -> &'static str { + match self { + Self::Sum => "sum", + Self::Count => "count", + Self::Avg => "avg", + Self::Min => "min", + Self::Max => "max", + Self::Group => "group", + Self::Stddev => "stddev", + Self::Stdvar => "stdvar", + } + } + /// Parses a canonical aggregate name without treating unknown kinds as another state. + pub fn parse(text: &str) -> Result { + match text { + "sum" => Ok(Self::Sum), + "count" => Ok(Self::Count), + "avg" => Ok(Self::Avg), + "min" => Ok(Self::Min), + "max" => Ok(Self::Max), + "group" => Ok(Self::Group), + "stddev" => Ok(Self::Stddev), + "stdvar" => Ok(Self::Stdvar), + _ => invalid("unknown aggregate state kind"), + } + } + fn components(self) -> &'static [StateComponent] { + use StateComponent as C; + match self { + Self::Sum => &[C::Value, C::Compensation], + Self::Count => &[C::Count], + Self::Avg => &[ + C::Value, + C::Mean, + C::Compensation, + C::Count, + C::IncrementalMean, + ], + Self::Min | Self::Max => &[C::Value], + Self::Group => &[], + Self::Stddev | Self::Stdvar => &[C::Value, C::Mean, C::Count], + } + } +} + +/// One named accumulator column. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub enum StateComponent { + Value, + Mean, + Compensation, + Count, + IncrementalMean, +} + +impl StateComponent { + /// Canonical metadata spelling. + pub const fn as_str(self) -> &'static str { + match self { + Self::Value => "value", + Self::Mean => "mean", + Self::Compensation => "compensation", + Self::Count => "count", + Self::IncrementalMean => "incremental_mean", + } + } + /// Parses a canonical component name, refusing unknown wire values. + pub fn parse(text: &str) -> Result { + match text { + "value" => Ok(Self::Value), + "mean" => Ok(Self::Mean), + "compensation" => Ok(Self::Compensation), + "count" => Ok(Self::Count), + "incremental_mean" => Ok(Self::IncrementalMean), + _ => invalid("unknown aggregate state component"), + } + } +} + +/// Values for all accumulator components at one field and grid position. +#[derive(Clone, Copy, Debug, Default, PartialEq)] +pub struct AggregateStateValues { + /// Accumulator value. + pub value: f64, + /// Running mean. + pub mean: f64, + /// Compensation retained for stable summation. + pub compensation: f64, + /// Observation count represented as a floating-point accumulator. + pub count: f64, + /// Whether the mean is incremental. + pub incremental_mean: bool, +} + +/// Validated aggregate kind and ordered component layout. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AggregateStateSchema { + pub kind: AggregateStateKind, + pub components: Arc<[StateComponent]>, +} + +impl AggregateStateSchema { + /// Creates a schema only when its ordered components exactly describe its kind. + pub fn try_new(kind: AggregateStateKind, components: Vec) -> Result { + if kind != AggregateStateKind::Group && components.is_empty() { + return invalid("aggregate state components must not be empty"); + } + if components.as_slice() != kind.components() { + return invalid("aggregate state components do not match kind"); + } + Ok(Self { + kind, + components: components.into(), + }) + } + /// Parses the aggregate kind and ordered comma-separated component metadata. + pub fn from_text(kind: &str, components: &str) -> Result { + let kind = AggregateStateKind::parse(kind)?; + let parsed = if components.is_empty() { + Vec::new() + } else { + components + .split(',') + .map(StateComponent::parse) + .collect::>>()? + }; + Self::try_new(kind, parsed) + } + /// Returns the canonical ordered component metadata text. + pub fn components_text(&self) -> String { + self.components + .iter() + .map(|component| component.as_str()) + .collect::>() + .join(",") + } +} + +/// A group-labelled sparse aggregate-state chunk on a shared time axis. +#[derive(Clone, Debug)] +pub struct StateSeries { + /// Complete group identity. + pub metric: SeriesLabels, + /// Grid positions carried by this chunk. + pub time: TimeAxis, + /// Field-major states; `None` denotes a field position without a contribution. + pub states: Vec>>, +} + +fn invalid(reason: &str) -> Result { + InvalidSeriesSnafu { + reason: reason.to_string(), + } + .fail() +} + +/// Columnar state fields retained in Arrow form so consumers can read component values in place. +/// Null items are field-local holes, not zero-valued states. +#[derive(Clone, Debug)] +pub struct StateFieldsBatch { + kind: AggregateStateKind, + components: Arc<[StateComponent]>, + fields: Vec, +} + +impl StateFieldsBatch { + /// Validates the ordered component layout once and keeps the row's typed Arrow arrays. + pub fn try_from_arrow(fields: StructArray, schema: &AggregateStateSchema) -> Result { + if fields.num_columns() == 0 { + return invalid("state fields array has no fields"); + } + let mut items = Vec::with_capacity(fields.num_columns()); + for (column, field) in fields.columns().iter().zip(fields.fields()) { + let DataType::List(item) = field.data_type() else { + return invalid("state field is not a list"); + }; + let DataType::Struct(members) = item.data_type() else { + return invalid("state list item is not a struct"); + }; + if members.len() != schema.components.len() { + return invalid("state component count mismatch"); + } + for (member, expected) in members.iter().zip(schema.components.iter()) { + if member.name() != expected.as_str() { + return invalid("state component order mismatch"); + } + let ty = if *expected == StateComponent::IncrementalMean { + DataType::Boolean + } else { + DataType::Float64 + }; + if member.data_type() != &ty { + return invalid("state component type mismatch"); + } + } + let list = column + .as_any() + .downcast_ref::() + .ok_or_else(|| invalid_series("state field list type mismatch"))?; + if list.is_null(0) { + return invalid("null state field list"); + } + let item_array = list.value(0); + let item_struct = item_array + .as_any() + .downcast_ref::() + .ok_or_else(|| invalid_series("state item struct type mismatch"))?; + items.push(item_struct.clone()); + } + Ok(Self { + kind: schema.kind, + components: schema.components.clone(), + fields: items, + }) + } + /// Number of logical series fields in this row. + pub fn len(&self) -> usize { + self.fields.len() + } + /// Whether this wrapper contains no series fields. + pub fn is_empty(&self) -> bool { + self.fields.is_empty() + } + /// The item struct for one field's list, sliced to the decoded row. + pub fn field(&self, index: usize) -> Option<&StructArray> { + self.fields.get(index) + } + /// Aggregate kind represented by these columns. + pub fn kind(&self) -> AggregateStateKind { + self.kind + } + /// Declared accumulator components in wire order. + pub fn components(&self) -> &[StateComponent] { + &self.components + } + /// Returns a numeric component array, or `None` if the component is not declared. + pub fn field_values( + &self, + field: usize, + component: StateComponent, + ) -> Result> { + let Some(index) = self.components.iter().position(|value| *value == component) else { + return Ok(None); + }; + self.fields + .get(field) + .ok_or_else(|| invalid_series("state field index out of range"))? + .column(index) + .as_any() + .downcast_ref::() + .map(Some) + .ok_or_else(|| invalid_series("state float component type mismatch")) + } + /// Returns a Boolean component array, or `None` if the component is not declared. + pub fn field_bool_values( + &self, + field: usize, + component: StateComponent, + ) -> Result> { + let Some(index) = self.components.iter().position(|value| *value == component) else { + return Ok(None); + }; + self.fields + .get(field) + .ok_or_else(|| invalid_series("state field index out of range"))? + .column(index) + .as_any() + .downcast_ref::() + .map(Some) + .ok_or_else(|| invalid_series("state boolean component type mismatch")) + } + /// Reads this field's item validity at one carried position; sibling fields never fill holes. + pub fn is_present(&self, field: usize, position: usize) -> Result { + let items = self + .fields + .get(field) + .ok_or_else(|| invalid_series("state field index out of range"))?; + if position >= items.len() { + return invalid("state position out of range"); + } + Ok(!items.is_null(position)) + } +} + +fn invalid_series(reason: &str) -> crate::error::Error { + InvalidSeriesSnafu { + reason: reason.to_string(), + } + .build() +} diff --git a/src/datatypes/src/series/frame.rs b/src/datatypes/src/series/frame.rs index e8934d5f4f4..bc92636bc57 100644 --- a/src/datatypes/src/series/frame.rs +++ b/src/datatypes/src/series/frame.rs @@ -80,7 +80,7 @@ use arrow::array::{ Array, ArrayRef, BinaryArray, BooleanArray, Float64Array, Int64Array, ListArray, StringArray, StructArray, UInt32Array, UInt64Array, }; -use arrow::buffer::{Buffer, NullBuffer, OffsetBuffer, ScalarBuffer}; +use arrow::buffer::{BooleanBuffer, Buffer, NullBuffer, OffsetBuffer, ScalarBuffer}; use arrow::compute::{concat, take}; use arrow::datatypes::{DataType, Field, Fields, Schema, SchemaRef}; use arrow::record_batch::{RecordBatch, RecordBatchOptions}; @@ -88,6 +88,9 @@ use serde::{Deserialize, Serialize}; use crate::data_type::ConcreteDataType; use crate::error::{InvalidSeriesSnafu, Result}; +use crate::series::aggregate_state::{ + AggregateStateSchema, AggregateStateValues, StateComponent, StateFieldsBatch, StateSeries, +}; use crate::series::histogram::{HistogramBatch, fields as histogram_fields}; use crate::series::raw::{RawSeriesChunk, RawTimestamps}; use crate::series::samples::{ @@ -105,12 +108,26 @@ const GRID: &str = "greptime.series.grid"; const LABEL_DECODE: &str = "greptime.series.label_decode"; /// Metadata key of the producer ordering carried by a complete frame declaration. const ORDERING: &str = "greptime.series.ordering"; +/// Metadata key naming the aggregate whose accumulator a state frame carries. +const AGGREGATE_KIND: &str = "greptime.series.aggregate.kind"; +/// Metadata key listing the accumulator components a state frame carries, in column order. +const AGGREGATE_COMPONENTS: &str = "greptime.series.aggregate.components"; /// Every `greptime.series.*` key a frame this revision writes may carry. /// /// Frame schemas carry an explicit `contiguous` default; streaming producers may replace it with /// their stronger guarantee. -const RESERVED_KEYS: [&str; 7] = [LAYOUT, CONTENT, LABELS, TIME, GRID, LABEL_DECODE, ORDERING]; +const RESERVED_KEYS: [&str; 9] = [ + LAYOUT, + CONTENT, + LABELS, + TIME, + GRID, + LABEL_DECODE, + ORDERING, + AGGREGATE_KIND, + AGGREGATE_COMPONENTS, +]; /// The only layout this revision writes and reads. /// @@ -125,14 +142,24 @@ pub const FRAME_LAYOUT_VERSION: &str = "v2"; /// reader refuses to guess at, while the layout versions the column and metadata set around it. pub const LABEL_DECODE_VERSION: u32 = 1; -/// What one frame's rows hold: raw samples with their own timestamps, or evaluated samples -/// addressed on the grid the frame declares. +/// What one frame's rows hold. +/// +/// The content decides the row layout, the time mode and the payload columns together, so a frame +/// states it once and every other declaration is derived from it rather than chosen separately. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum FrameContent { /// Samples read from storage, each row carrying the timestamps it was read at. Raw, /// Values computed on an evaluation grid, each row carrying the grid positions it holds. Evaluated, + /// Partial aggregate accumulator state, addressed on the grid the frame declares. + /// + /// A state row is not a query value: it names a group and the positions it has contributions + /// at, and its payload carries the accumulator components that aggregate declared. It is + /// addressed like an evaluated row because a group's state lives at the same grid positions a + /// value would, but the two are not interchangeable — a consumer that folds values must refuse + /// state, and one that merges state must refuse values. + AggregateState, } impl FrameContent { @@ -141,6 +168,7 @@ impl FrameContent { match self { Self::Raw => "raw", Self::Evaluated => "evaluated", + Self::AggregateState => "aggregate_state", } } @@ -149,6 +177,7 @@ impl FrameContent { match text.as_bytes() { b"raw" => Some(Self::Raw), b"evaluated" => Some(Self::Evaluated), + b"aggregate_state" => Some(Self::AggregateState), _ => None, } } @@ -162,7 +191,7 @@ impl FrameContent { const fn time_text(self) -> &'static str { match self { Self::Raw => "timestamps", - Self::Evaluated => "grid_spans", + Self::Evaluated | Self::AggregateState => "grid_spans", } } @@ -170,7 +199,7 @@ impl FrameContent { const fn time_member(self) -> &'static str { match self { Self::Raw => "timestamps", - Self::Evaluated => "spans", + Self::Evaluated | Self::AggregateState => "spans", } } } @@ -409,7 +438,7 @@ pub fn series_frame_schema( )] .into(), ), - FrameContent::Evaluated => DataType::Struct( + FrameContent::Evaluated | FrameContent::AggregateState => DataType::Struct( vec![Field::new( "spans", list( @@ -517,6 +546,57 @@ pub fn series_frame_schema( ))) } +/// Builds an aggregate state frame schema with explicit labels and a declared grid. +pub fn aggregate_state_frame_schema( + state: &AggregateStateSchema, + schema: &SeriesSchema, + labels: FrameLabelMode, +) -> Result { + if schema.eval_grid.is_none() { + return invalid_series("an aggregate state frame needs an evaluation grid"); + } + if labels != FrameLabelMode::Explicit { + return invalid_series("an aggregate state frame requires explicit labels"); + } + let components: Fields = state + .components + .iter() + .map(|component| { + let (ty, nullable) = match component { + StateComponent::IncrementalMean => (DataType::Boolean, true), + _ => (DataType::Float64, true), + }; + Field::new(component.as_str(), ty, nullable) + }) + .collect(); + let member = DataType::List(Arc::new(Field::new( + "item", + DataType::Struct(components), + true, + ))); + let fields: Fields = schema + .fields + .iter() + .map(|field| Field::new(field.name.as_ref(), member.clone(), false)) + .collect(); + let mut ordinary = series_frame_schema(schema, labels, None)?; + let mut metadata = ordinary.metadata().clone(); + metadata.insert( + CONTENT.to_string(), + FrameContent::AggregateState.as_str().to_string(), + ); + metadata.insert( + TIME.to_string(), + FrameContent::AggregateState.time_text().to_string(), + ); + metadata.insert(AGGREGATE_KIND.to_string(), state.kind.as_str().to_string()); + metadata.insert(AGGREGATE_COMPONENTS.to_string(), state.components_text()); + let mut columns = ordinary.fields().to_vec(); + columns[3] = Field::new("fields", DataType::Struct(fields), false).into(); + ordinary = Arc::new(Schema::new_with_metadata(columns, metadata)); + Ok(ordinary) +} + /// Builds the typed schema for a raw frame, refusing a series schema with an evaluation grid. /// /// A key frame is unreadable without its description, so this constructor serves explicit-label @@ -561,8 +641,10 @@ fn parse_grid(text: &str) -> Result> { #[derive(Clone, Debug)] pub struct SeriesFrameDeclaration { series: SeriesSchema, + content: FrameContent, labels: FrameLabelMode, label_decode: Option, + aggregate_state: Option, } fn validate_ordering(schema: &Schema, content: FrameContent) -> Result<()> { @@ -622,6 +704,40 @@ impl SeriesFrameDeclaration { return Err(err("an evaluated frame must declare its evaluation grid")); } (FrameContent::Evaluated, Some(text)) => Some(parse_grid(text)?), + (FrameContent::AggregateState, None) => { + return Err(err( + "an aggregate state frame must declare its evaluation grid", + )); + } + (FrameContent::AggregateState, Some(text)) => Some(parse_grid(text)?), + }; + let has_kind = schema.metadata().contains_key(AGGREGATE_KIND); + let has_components = schema.metadata().contains_key(AGGREGATE_COMPONENTS); + let aggregate_state = match content { + FrameContent::AggregateState => { + if !has_kind || !has_components { + return Err(err( + "an aggregate state frame must declare kind and components", + )); + } + if schema.metadata().contains_key(LABEL_DECODE) { + return Err(err( + "an aggregate state frame cannot carry a label decode description", + )); + } + Some(AggregateStateSchema::from_text( + schema.metadata()[AGGREGATE_KIND].as_str(), + schema.metadata()[AGGREGATE_COMPONENTS].as_str(), + )?) + } + FrameContent::Raw | FrameContent::Evaluated => { + if has_kind || has_components { + return Err(err( + "raw or evaluated frames cannot carry aggregate state metadata", + )); + } + None + } }; let label_decode = match schema.metadata().get(LABEL_DECODE) { None => None, @@ -670,6 +786,8 @@ impl SeriesFrameDeclaration { )); } Ok(Self { + content, + aggregate_state, series: SeriesSchema { fields: members .iter() @@ -691,10 +809,12 @@ impl SeriesFrameDeclaration { /// What the frame's rows hold. pub fn content(&self) -> FrameContent { - match self.series.eval_grid { - Some(_) => FrameContent::Evaluated, - None => FrameContent::Raw, - } + self.content + } + + /// The validated accumulator kind and component layout, present only on state frames. + pub fn aggregate_state(&self) -> Option<&AggregateStateSchema> { + self.aggregate_state.as_ref() } /// How the frame names its series. @@ -735,6 +855,13 @@ impl FrameShape { }; Self { content, labels } } + + fn aggregate_state(labels: FrameLabelMode) -> Self { + Self { + content: FrameContent::AggregateState, + labels, + } + } } /// Bounded multi-row builder for finite series frame batches. @@ -789,6 +916,28 @@ impl SeriesFrameBuilder { }) } + /// Creates a bounded builder for validated aggregate state rows. + pub fn try_new_aggregate_state( + state: &AggregateStateSchema, + schema: &SeriesSchema, + labels: FrameLabelMode, + max_rows: usize, + max_bytes: usize, + ) -> Result { + if max_rows == 0 || max_bytes == 0 { + return invalid_series("series frame limits must be nonzero"); + } + Ok(Self { + schema: schema.clone(), + frame_schema: aggregate_state_frame_schema(state, schema, labels)?, + shape: FrameShape::aggregate_state(labels), + max_rows, + max_bytes: max_bytes.min(i32::MAX as usize), + rows: Vec::new(), + bytes: 0, + }) + } + /// The frame schema of every batch this builder emits. pub fn schema(&self) -> &SchemaRef { &self.frame_schema @@ -885,6 +1034,7 @@ enum ChunkWalk { Raw(RawWalk), /// An evaluated chunk: absolute grid positions, carried as runs. Evaluated(EvaluatedWalk), + State(StateWalk), } /// The walk state of one raw chunk. @@ -901,6 +1051,13 @@ struct RawWalk { pos: usize, } +struct StateWalk { + series: StateSeries, + is_last: bool, + runs: Vec, + done: bool, +} + /// The walk state of one evaluated chunk. /// /// The runs are collected once, so a row is one intersection of a run with a sample segment and the @@ -1000,6 +1157,79 @@ impl SeriesFrameCursor { }) } + /// Validates and walks one aggregate state chunk. + pub fn try_new_aggregate_state( + state: &AggregateStateSchema, + schema: &SeriesSchema, + series: StateSeries, + is_last: bool, + ) -> Result { + if schema.eval_grid.is_none() { + return invalid_series("an aggregate state chunk needs an evaluation grid"); + } + if series.states.len() != schema.fields.len() + || series + .states + .iter() + .any(|field| field.len() != series.time.len()) + { + return invalid_series("aggregate state field shape mismatch"); + } + if series.time.grid().map(|grid| grid.spec()) + != schema.eval_grid.as_ref().map(|grid| grid.spec()) + { + return invalid_series("aggregate state chunk grid does not match schema"); + } + let mut states = series.states; + // A position is carried when some field of it was folded, which is what the axis says. When + // every position is carried - the shape a fold produces whenever each group has a sample at + // each of its points - the axis and the states are already the ones this walk reads, so + // nothing is rebuilt. Deciding that first keeps the common case free of a mask, a position + // list and a second axis. + let every_position_carried = state.kind + == crate::series::aggregate_state::AggregateStateKind::Group + || (0..series.time.len()).all(|row| states.iter().any(|field| field[row].is_some())); + let time = if every_position_carried { + series.time + } else { + let contributing = (0..series.time.len()) + .map(|row| states.iter().any(|field| field[row].is_some())) + .collect::>(); + let carried = series + .time + .positions() + .filter_map(|(position, row)| contributing[row].then_some(position)) + .collect::>(); + let spans = positions_to_spans(&carried); + for field in &mut states { + let mut row = 0; + field.retain(|_| { + let keep = contributing[row]; + row += 1; + keep + }); + } + TimeAxis::from_grid_spans(series.time.grid().expect("validated grid").clone(), spans)? + }; + let runs: Vec = time.runs().collect(); + let series = StateSeries { + metric: series.metric, + time, + states, + }; + Ok(Self { + schema: schema.clone(), + shape: FrameShape::aggregate_state(FrameLabelMode::Explicit), + walk: ChunkWalk::State(StateWalk { + series, + is_last, + runs, + done: false, + }), + pending: None, + }) + } + /// Appends rows to the builder until one batch is ready or this chunk is exhausted. /// /// `None` means the cursor is exhausted; calling again returns `None` without touching the @@ -1040,10 +1270,37 @@ impl SeriesFrameCursor { match &mut self.walk { ChunkWalk::Raw(walk) => walk.next_row(), ChunkWalk::Evaluated(walk) => walk.next_row(available_bytes), + ChunkWalk::State(walk) => walk.next_row(), } } } +impl StateWalk { + fn next_row(&mut self) -> Result> { + if self.done { + return Ok(None); + } + self.done = true; + let states = std::mem::take(&mut self.series.states); + let time = RowTime::Spans( + self.runs + .iter() + .map(|run| GridSpan::new(run.position, run.len)) + .collect(), + ); + let mut row = FrameRow { + key: None, + metric: Some(self.series.metric.clone()), + is_last: self.is_last, + time, + payload: RowPayload::States(states), + bytes: 0, + }; + row.bytes = measure_row(&row)?; + Ok(Some(row)) + } +} + impl RawWalk { /// Builds the next row, or `None` when the chunk is exhausted. /// @@ -1081,7 +1338,7 @@ impl RawWalk { metric: chunk.metric.clone(), is_last: chunk.is_last && *pos + take == chunk.time.len(), time: RowTime::Timestamps(time_axis.slice(*pos, take)), - samples, + payload: RowPayload::Samples(samples), bytes: 0, }; row.bytes = measure_row(&row)?; @@ -1130,7 +1387,7 @@ impl EvaluatedWalk { metric: Some(series.metric.clone()), is_last: *is_last, time: RowTime::Spans(Vec::new()), - samples, + payload: RowPayload::Samples(samples), bytes: 0, }; row.bytes = measure_row(&row)?; @@ -1188,7 +1445,7 @@ impl EvaluatedWalk { metric: Some(series.metric.clone()), is_last: *is_last && next_run + 1 == runs.len() && take == remaining, time: RowTime::Spans(vec![GridSpan::new(current.position + next_offset, take)]), - samples: Vec::new(), + payload: RowPayload::Samples(Vec::new()), bytes: 0, }; candidate.bytes = add_checked(measure_row(&candidate)?, sample_growth)?; @@ -1217,12 +1474,16 @@ impl EvaluatedWalk { } } if let Some(row) = &mut row { - row.samples = series - .fields - .iter() - .enumerate() - .map(|(index, field)| segment(field, segments[index])?.slice(offsets[index], total)) - .collect::>>()?; + row.payload = RowPayload::Samples( + series + .fields + .iter() + .enumerate() + .map(|(index, field)| { + segment(field, segments[index])?.slice(offsets[index], total) + }) + .collect::>>()?, + ); } *run = next_run; *offset = next_offset; @@ -1283,10 +1544,15 @@ struct FrameRow { is_last: bool, /// The positions this row's samples are addressed at. time: RowTime, - samples: Vec, + payload: RowPayload, bytes: usize, } +enum RowPayload { + Samples(Vec), + States(Vec>>), +} + /// How one row addresses the time of its samples. enum RowTime { /// The timestamps of this row's samples, strictly increasing within the row. @@ -1384,6 +1650,16 @@ fn histogram_table_charge(values: &HistogramSamples) -> Result { Ok(bytes) } +fn state_charges(fields: &[Vec>]) -> Result { + fields.iter().try_fold(0usize, |bytes, values| { + let mut charge = FIELD_CHARGE; + for _ in values.iter().flatten() { + charge = add_checked(charge, 8 * 4 + 1)?; + } + add_checked(bytes, charge) + }) +} + fn sample_charges(samples: &[Samples]) -> Result { samples.iter().try_fold(0usize, |bytes, sample| { add_checked(bytes, sample_charge(sample)?) @@ -1419,7 +1695,13 @@ fn measure_row(row: &FrameRow) -> Result { mul_checked(spans.len(), 2)?, )?, }; - bytes = add_checked(bytes, sample_charges(&row.samples)?)?; + bytes = add_checked( + bytes, + match &row.payload { + RowPayload::Samples(samples) => sample_charges(samples)?, + RowPayload::States(fields) => state_charges(fields)?, + }, + )?; if let Some(metric) = &row.metric { if let Some(name) = metric.name() { checked_offset(name.len())?; @@ -1522,7 +1804,7 @@ fn encode_rows(schema: &SchemaRef, shape: FrameShape, rows: &[FrameRow]) -> Resu let times = list_array(&time_lengths, time_child, time_item.as_ref().clone(), None)?; struct_array(time_fields.clone(), vec![Arc::new(times)])? } - FrameContent::Evaluated => { + FrameContent::Evaluated | FrameContent::AggregateState => { let DataType::Struct(span_fields) = time_item.data_type() else { return invalid_series("grid span schema mismatch"); }; @@ -1572,6 +1854,130 @@ fn encode_rows(schema: &SchemaRef, shape: FrameShape, rows: &[FrameRow]) -> Resu } }; + if shape.content == FrameContent::AggregateState { + // A state frame's per-field list carries one item per grid position the row holds, and an + // item is absent exactly where that field has no contribution at that position. The + // absence lives on the item — the child struct's validity — not on the list's own + // validity, which is per row: a row always has a list, with zero or more items in it. + let mut field_arrays: Vec = Vec::with_capacity(field_types.len()); + for (index, field) in field_types.iter().enumerate() { + let mut lengths = Vec::with_capacity(count); + let mut positions = 0usize; + for row in rows { + let RowPayload::States(fields) = &row.payload else { + return invalid_series("state frame row carries sample payload"); + }; + let values = fields + .get(index) + .ok_or_else(|| err("state field count mismatch"))?; + if values.len() != row.time.len()? { + return invalid_series("state list length mismatch"); + } + lengths.push(values.len()); + positions = add_checked(positions, values.len())?; + } + + let item = match field.data_type() { + DataType::List(item) => item, + _ => return invalid_series("state field schema mismatch"), + }; + let DataType::Struct(components) = item.data_type() else { + return invalid_series("state list item is not a component struct"); + }; + // Absence lives on the item, so the child struct's validity is one flag per position. + // A row of a state frame always carries states; a caller that hands in the wrong payload + // is refused here rather than panicking inside an iterator. + let mut present = Vec::with_capacity(positions); + for row in rows { + let RowPayload::States(fields) = &row.payload else { + return invalid_series("state frame row carries sample payload"); + }; + present.extend(fields[index].iter().map(Option::is_some)); + } + debug_assert_eq!(present.len(), positions); + + // Every component of a field is absent at exactly the same positions - the field's own - + // so they share the one mask, which the child struct already needs. + let nulls = present + .iter() + .any(|present| !present) + .then(|| NullBuffer::from(present.clone())); + let arrays: Vec = components + .iter() + .map(|component| { + let component_name = component.name(); + let array: ArrayRef = match component_name.as_str() { + "value" | "mean" | "compensation" | "count" => { + let mut floats = Vec::with_capacity(positions); + for row in rows { + let RowPayload::States(fields) = &row.payload else { + return invalid_series( + "state frame row carries sample payload", + ); + }; + for value in &fields[index] { + // A missing position's value is unreadable, so it holds the same + // zero a null slot has always held here. + floats.push(value.map_or(0.0, |state| { + match component_name.as_str() { + "value" => state.value, + "mean" => state.mean, + "compensation" => state.compensation, + "count" => state.count, + _ => unreachable!(), + } + })); + } + } + Arc::new(Float64Array::new(ScalarBuffer::from(floats), nulls.clone())) + } + "incremental_mean" => { + let mut booleans = Vec::with_capacity(positions); + for row in rows { + let RowPayload::States(fields) = &row.payload else { + return invalid_series( + "state frame row carries sample payload", + ); + }; + booleans.extend(fields[index].iter().map(|value| { + value.is_some_and(|state| state.incremental_mean) + })); + } + Arc::new(BooleanArray::new( + BooleanBuffer::from(booleans), + nulls.clone(), + )) + } + _ => return invalid_series("unknown state component"), + }; + Ok(array) + }) + .collect::>>()?; + + let child = if components.is_empty() { + Arc::new(StructArray::new_empty_fields( + positions, + Some(NullBuffer::from(present)), + )) as ArrayRef + } else { + Arc::new( + StructArray::try_new( + components.clone(), + arrays, + Some(NullBuffer::from(present)), + ) + .map_err(|error| err(error.to_string()))?, + ) as ArrayRef + }; + // The list itself is valid for every row: absence is an absent item, never an absent + // list, so a reader can tell "this field holds nothing here" from "this row has no + // list at all". + let list = list_array(&lengths, child, item.as_ref().clone(), None)?; + field_arrays.push(Arc::new(list) as ArrayRef); + } + let fields = struct_array(field_types.clone(), field_arrays)?; + return assemble_frame_batch(schema, shape, rows, time, fields); + } let mut empty_table: Option = None; let mut field_arrays = Vec::with_capacity(field_types.len()); for (index, field) in field_types.iter().enumerate() { @@ -1602,12 +2008,13 @@ fn encode_rows(schema: &SchemaRef, shape: FrameShape, rows: &[FrameRow]) -> Resu let mut table_parts: Vec = Vec::new(); let mut histogram_valid = Vec::with_capacity(count); for row in rows { - let dense = row - .samples - .get(index) - .ok_or_else(|| err("raw frame row field count mismatch"))? - .dense() - .ok_or_else(|| err("sample segment is not dense"))?; + let dense = match &row.payload { + RowPayload::Samples(samples) => samples.get(index), + _ => return invalid_series("sample frame row carries state payload"), + } + .ok_or_else(|| err("raw frame row field count mismatch"))? + .dense() + .ok_or_else(|| err("sample segment is not dense"))?; match dense { DenseSamplesRef::Float(values) => { float_lengths.push(values.len()); @@ -1689,7 +2096,23 @@ fn encode_rows(schema: &SchemaRef, shape: FrameShape, rows: &[FrameRow]) -> Resu )?) as ArrayRef); } let fields = struct_array(field_types.clone(), field_arrays)?; + assemble_frame_batch(schema, shape, rows, time, fields) +} +/// Builds the batch from the parts every content shares: the key, finality, time and field +/// columns, plus the labels an explicit frame carries. +/// +/// Keeping the assembly in one place is what makes a state row and an evaluated row leave the +/// encoder under the same column order and the same options; only how their field columns were +/// built differs, and that is decided before this call. +fn assemble_frame_batch( + schema: &SchemaRef, + shape: FrameShape, + rows: &[FrameRow], + time: StructArray, + fields: StructArray, +) -> Result { + let count = rows.len(); let mut columns: Vec = vec![ Arc::new(binary_array(rows, shape)?), Arc::new(BooleanArray::from( @@ -1805,6 +2228,20 @@ const HISTOGRAM_TABLE_CHARGE: usize = 28; /// Item widths of histogram table list columns `7..=13`, in that order. const HISTOGRAM_LIST_WIDTHS: [usize; 7] = [8, 4, 4, 8, 4, 4, 8]; +fn positions_to_spans(positions: &[usize]) -> Vec { + let mut spans: Vec = Vec::new(); + for position in positions { + if let Some(last) = spans.last_mut() + && last.start() + last.len() == *position + { + *last = GridSpan::new(last.start(), last.len() + 1); + continue; + } + spans.push(GridSpan::new(*position, 1)); + } + spans +} + fn list_array( lengths: &[usize], values: ArrayRef, @@ -1876,6 +2313,111 @@ fn mul_checked(len: usize, width: usize) -> Result { .ok_or_else(|| err("raw series frame offset overflow")) } +/// One decoded state row retaining Arrow handles for its per-field state columns. +/// +/// The arrays remain columnar so consumers can read typed components in place rather than building +/// a nested accumulator tree. Null list items remain field-local absence, not numeric zero. +pub struct StateFrameRow { + metric: SeriesLabels, + time: TimeAxis, + is_last: bool, + fields: StructArray, + state: AggregateStateSchema, +} + +impl StateFrameRow { + /// Complete group identity for this row. + pub fn metric(&self) -> &SeriesLabels { + &self.metric + } + /// Grid positions carried by this row. + pub fn time(&self) -> &TimeAxis { + &self.time + } + /// Whether this row closes its producer-local chunk chain. + pub fn is_last(&self) -> bool { + self.is_last + } + /// Columnar state arrays validated against the declared aggregate layout. + pub fn fields(&self) -> Result { + StateFieldsBatch::try_from_arrow(self.fields.clone(), &self.state) + } + + /// The export-side form of this row: one group, its carried positions, and its per-field state. + /// + /// This is the dual of `decode_aggregate_state`: it reads the row's columnar arrays back into the + /// form a producer hands to `pack_aggregate_state`. Absence stays per field — a null item is + /// `None` for that field at that position, never a zero and never promoted to a sibling field. + pub fn to_state_series(&self) -> Result { + let fields = self.fields()?; + let mut states = Vec::with_capacity(fields.len()); + for field in 0..fields.len() { + let mut values = Vec::with_capacity(self.time.len()); + for position in 0..self.time.len() { + if !fields.is_present(field, position)? { + values.push(None); + continue; + } + let mut value = AggregateStateValues::default(); + for &component in fields.components() { + match component { + StateComponent::Value => { + value.value = state_float(&fields, field, component, position)? + } + StateComponent::Mean => { + value.mean = state_float(&fields, field, component, position)? + } + StateComponent::Compensation => { + value.compensation = state_float(&fields, field, component, position)? + } + StateComponent::Count => { + value.count = state_float(&fields, field, component, position)? + } + StateComponent::IncrementalMean => { + value.incremental_mean = fields + .field_bool_values(field, component)? + .and_then(|array| { + (!array.is_null(position)).then(|| array.value(position)) + }) + .ok_or_else(|| { + InvalidSeriesSnafu { + reason: "present aggregate state has a missing component" + .to_string(), + } + .build() + })?; + } + } + } + values.push(Some(value)); + } + states.push(values); + } + Ok(StateSeries { + metric: self.metric.clone(), + time: self.time.clone(), + states, + }) + } +} + +fn state_float( + fields: &StateFieldsBatch, + field: usize, + component: StateComponent, + position: usize, +) -> Result { + fields + .field_values(field, component)? + .and_then(|array| (!array.is_null(position)).then(|| array.value(position))) + .ok_or_else(|| { + InvalidSeriesSnafu { + reason: "present aggregate state has a missing component".to_string(), + } + .build() + }) +} + /// Reuses one prepared frame schema while decoding all batches of a stream. /// /// The decoder is built from the schema a consumer expects and then checks every batch against it: @@ -1886,6 +2428,7 @@ pub struct SeriesFrameDecoder { schema: SeriesSchema, shape: FrameShape, expected: SchemaRef, + aggregate_state: Option, } impl SeriesFrameDecoder { @@ -1895,13 +2438,177 @@ impl SeriesFrameDecoder { labels: FrameLabelMode, decode: Option<&LabelDecodeSpec>, ) -> Result { + let shape = FrameShape::of(schema, labels); Ok(Self { schema: schema.clone(), - shape: FrameShape::of(schema, labels), + shape, expected: series_frame_schema(schema, labels, decode)?, + aggregate_state: None, }) } + /// Builds a decoder for the declared aggregate state and its grid-addressed series fields. + pub fn try_new_aggregate_state( + state: &AggregateStateSchema, + schema: &SeriesSchema, + labels: FrameLabelMode, + ) -> Result { + if schema.eval_grid.is_none() { + return invalid_series("an aggregate state decoder requires an evaluation grid"); + } + Ok(Self { + schema: schema.clone(), + shape: FrameShape::aggregate_state(labels), + expected: aggregate_state_frame_schema(state, schema, labels)?, + aggregate_state: Some(state.clone()), + }) + } + + /// Reads only the explicit labels of each aggregate-state row. + /// + /// This validates the declared frame schema and label columns without inspecting time or state. + pub fn decode_aggregate_state_labels(&self, batch: &RecordBatch) -> Result> { + if self.shape.labels != FrameLabelMode::Explicit { + return invalid_series("aggregate state routing requires explicit labels"); + } + let cols = batch.columns(); + let names = down::(&cols[4], "name")?; + let labels = down::(&cols[5], "labels")?; + let mut rows = Vec::with_capacity(batch.num_rows()); + for row in 0..batch.num_rows() { + if labels.is_null(row) { + return invalid_series("null required aggregate state label"); + } + let values = labels.value(row); + let label_struct = down::(&values, "labels")?; + if label_struct.null_count() != 0 { + return invalid_series("null label struct element"); + } + let label_names = down::(label_struct.column(0), "label names")?; + let label_values = down::(label_struct.column(1), "label values")?; + if label_names.null_count() != 0 || label_values.null_count() != 0 { + return invalid_series("null label element"); + } + let tags = (0..label_names.len()) + .map(|i| Label { + name: Arc::from(label_names.value(i)), + value: Arc::from(label_values.value(i)), + }) + .collect(); + // An aggregation drops the metric name, and the frame says so by leaving it null. That + // is an identity with no name, not a malformed row. + let name = (!names.is_null(row)).then(|| Arc::from(names.value(row))); + rows.push(SeriesLabels::try_new(name, tags)?); + } + Ok(rows) + } + + /// Decodes one aggregate-state row without materializing its accumulator values. + pub fn decode_aggregate_state(&self, batch: &RecordBatch) -> Result> { + let state = self + .aggregate_state + .as_ref() + .ok_or_else(|| err("decoder has no aggregate state declaration"))?; + if batch.schema().as_ref() != self.expected.as_ref() { + return invalid_series("aggregate state frame schema mismatch"); + } + validate_ordering(batch.schema().as_ref(), FrameContent::AggregateState)?; + let cols = batch.columns(); + let lasts = down::(&cols[1], "is_last")?; + let time = down::(&cols[2], "time")?; + let fields = down::(&cols[3], "fields")?; + let names = down::(&cols[4], "name")?; + let labels = down::(&cols[5], "labels")?; + let grid = self + .schema + .eval_grid + .clone() + .ok_or_else(|| err("an aggregate state frame needs an evaluation grid"))?; + let mut rows = Vec::with_capacity(batch.num_rows()); + for row in 0..batch.num_rows() { + if lasts.is_null(row) || time.is_null(row) || fields.is_null(row) || labels.is_null(row) + { + return invalid_series("null required aggregate state parent"); + } + let spans_array = down::(time.column(0), "spans")?; + if spans_array.is_null(row) { + return invalid_series("null aggregate state spans"); + } + let spans_values = spans_array.value(row); + let spans_struct = down::(&spans_values, "spans")?; + if spans_struct.null_count() != 0 { + return invalid_series("null aggregate state span"); + } + let starts = down::(spans_struct.column(0), "span starts")?; + let lengths = down::(spans_struct.column(1), "span lengths")?; + let mut spans = Vec::with_capacity(starts.len()); + for index in 0..starts.len() { + let start = usize::try_from(starts.value(index)) + .map_err(|_| err("grid span start exceeds the supported range"))?; + let len = usize::try_from(lengths.value(index)) + .map_err(|_| err("grid span length exceeds the supported range"))?; + if len == 0 { + return invalid_series("grid spans must not be empty"); + } + spans.push(GridSpan::new(start, len)); + } + let axis = TimeAxis::from_grid_spans(grid.clone(), spans)?; + let label_values = labels.value(row); + let label_struct = down::(&label_values, "labels")?; + if label_struct.null_count() != 0 { + return invalid_series("null label struct element"); + } + let label_names = down::(label_struct.column(0), "label names")?; + let label_values = down::(label_struct.column(1), "label values")?; + if label_names.null_count() != 0 || label_values.null_count() != 0 { + return invalid_series("null label element"); + } + let tags = (0..label_names.len()) + .map(|i| Label { + name: Arc::from(label_names.value(i)), + value: Arc::from(label_values.value(i)), + }) + .collect(); + let metric = SeriesLabels::try_new( + (!names.is_null(row)).then(|| Arc::from(names.value(row))), + tags, + )?; + let row_fields = fields.slice(row, 1); + let validated = StateFieldsBatch::try_from_arrow(row_fields.clone(), state)?; + if validated.len() != self.schema.fields.len() { + return invalid_series("aggregate state field count mismatch"); + } + for field in 0..validated.len() { + let items = validated + .field(field) + .ok_or_else(|| err("null state field list"))?; + if items.len() != axis.len() { + return invalid_series("state list length does not match carried positions"); + } + if state.kind != crate::series::aggregate_state::AggregateStateKind::Group { + for component in 0..state.components.len() { + let values = items.column(component); + for position in 0..items.len() { + if !items.is_null(position) && values.is_null(position) { + return invalid_series( + "present state item has a missing component", + ); + } + } + } + } + } + rows.push(StateFrameRow { + metric, + time: axis, + is_last: lasts.value(row), + fields: row_fields, + state: state.clone(), + }); + } + Ok(rows) + } + /// Decodes all rows of one batch as raw chunks. pub fn decode_raw(&self, batch: &RecordBatch) -> Result> { if self.shape.content != FrameContent::Raw { @@ -1917,7 +2624,10 @@ impl SeriesFrameDecoder { .key .ok_or_else(|| err("a raw frame row carries no key"))?; let time = RawTimestamps::try_new(timestamps)?; - let fields = SeriesFields::try_from_vec(row.samples)?; + let RowPayload::Samples(samples) = row.payload else { + return invalid_series("raw frame row carries state payload"); + }; + let fields = SeriesFields::try_from_vec(samples)?; out.push(RawSeriesChunk::try_new( &self.schema, key, @@ -1964,7 +2674,10 @@ impl SeriesFrameDecoder { "an evaluated chunk without positions must close its series", ); } - let fields = SeriesFields::try_from_vec(row.samples)?; + let RowPayload::Samples(samples) = row.payload else { + return invalid_series("evaluated frame row carries state payload"); + }; + let fields = SeriesFields::try_from_vec(samples)?; out.push(( Series::try_new(&self.schema, metric, time, fields)?, row.is_last, @@ -2041,7 +2754,7 @@ impl SeriesFrameDecoder { } RowTime::Timestamps(tv.values().clone()) } - FrameContent::Evaluated => { + FrameContent::Evaluated | FrameContent::AggregateState => { let typed = down::(time_struct.column(0), "spans")?; if typed.is_null(row) { return invalid_series("null required evaluated frame parent"); @@ -2154,7 +2867,7 @@ impl SeriesFrameDecoder { metric, is_last: lasts.value(row), time, - samples: decoded, + payload: RowPayload::Samples(decoded), bytes: 0, }); } @@ -2207,3 +2920,125 @@ fn err(reason: impl Into) -> crate::error::Error { fn arrow(result: std::result::Result) -> Result { result.map_err(|e| err(e.to_string())) } + +#[cfg(test)] +mod aggregate_state_contribution_tests { + use std::sync::Arc; + + use super::*; + use crate::series::aggregate_state::{ + AggregateStateKind as Kind, AggregateStateValues as Values, StateComponent as Component, + }; + use crate::series::samples::FieldSpec; + + fn schema(len: usize) -> SeriesSchema { + SeriesSchema { + fields: ["a", "b"] + .into_iter() + .map(|name| FieldSpec { + name: Arc::from(name), + }) + .collect::>() + .into(), + eval_grid: Some(Arc::new( + EvalGrid::try_new(GridSpec::try_new(0, 1, len as u64).unwrap()).unwrap(), + )), + } + } + + fn cursor( + kind: Kind, + schema: &SeriesSchema, + spans: Vec, + states: Vec>>, + ) -> SeriesFrameCursor { + let components = match kind { + Kind::Group => vec![], + _ => vec![Component::Value, Component::Compensation], + }; + let state = AggregateStateSchema::try_new(kind, components).unwrap(); + SeriesFrameCursor::try_new_aggregate_state( + &state, + schema, + StateSeries { + metric: SeriesLabels::try_new(None, vec![]).unwrap(), + time: TimeAxis::from_grid_spans(schema.eval_grid.clone().unwrap(), spans).unwrap(), + states, + }, + true, + ) + .unwrap() + } + + #[test] + fn sparse_rows_are_filtered_and_group_keeps_all_rows() { + let schema = schema(12); + let sparse = cursor( + Kind::Sum, + &schema, + vec![GridSpan::new(0, 5), GridSpan::new(7, 4)], + vec![ + vec![ + None, + Some(Values::default()), + None, + None, + Some(Values::default()), + None, + None, + None, + None, + ], + vec![ + None, + None, + None, + Some(Values::default()), + None, + None, + Some(Values::default()), + None, + None, + ], + ], + ); + let ChunkWalk::State(walk) = sparse.walk else { + panic!("expected state walk") + }; + assert_eq!( + walk.series.time.positions().collect::>(), + vec![(1, 0), (3, 1), (4, 2), (8, 3)] + ); + assert_eq!( + walk.series + .time + .runs() + .map(|run| (run.position, run.len)) + .collect::>(), + vec![(1, 1), (3, 2), (8, 1)] + ); + assert_eq!( + walk.series.states[0], + vec![Some(Values::default()), None, Some(Values::default()), None] + ); + assert_eq!( + walk.series.states[1], + vec![None, Some(Values::default()), None, Some(Values::default())] + ); + + let group = cursor( + Kind::Group, + &schema, + vec![GridSpan::new(2, 3), GridSpan::new(8, 2)], + vec![vec![None; 5], vec![None; 5]], + ); + let ChunkWalk::State(walk) = group.walk else { + panic!("expected state walk") + }; + assert_eq!( + walk.series.time.positions().collect::>(), + vec![(2, 0), (3, 1), (4, 2), (8, 3), (9, 4)] + ); + assert_eq!(walk.series.states, vec![vec![None; 5], vec![None; 5]]); + } +} diff --git a/src/datatypes/src/series/frame/tests.rs b/src/datatypes/src/series/frame/tests.rs index 6854ad6b006..b16839c5bad 100644 --- a/src/datatypes/src/series/frame/tests.rs +++ b/src/datatypes/src/series/frame/tests.rs @@ -28,6 +28,7 @@ use crate::series::histogram::{BucketSideView, CounterResetHint, HistogramBatchB use crate::series::samples::{ DenseSamplesRef, FieldSpec, SampleRef, Samples, SeriesFields, SeriesSchema, }; +use crate::series::time::GridRange; use crate::series::{Label, SeriesLabels}; fn test_schema(names: &[&str]) -> SeriesSchema { @@ -1358,7 +1359,7 @@ fn evaluated_incremental_measure_matches_full_measure_at_exact_byte_budget() { metric: Some(metric.clone()), is_last: true, time: RowTime::Spans(vec![GridSpan::new(0, 8), GridSpan::new(10, 1)]), - samples: vec![samples.clone()], + payload: RowPayload::Samples(vec![samples.clone()]), bytes: 0, }; let exact_budget = measure_row(&whole).unwrap(); @@ -4673,3 +4674,785 @@ fn the_frozen_text_covers_every_datatype_variant() { "one fixture entry per datatype variant, no duplicates" ); } + +#[cfg(test)] +mod aggregate_state_acceptance { + use super::*; + use crate::series::aggregate_state::{ + AggregateStateKind as Kind, AggregateStateSchema, AggregateStateValues as Values, + StateComponent as Component, StateSeries, + }; + + fn state_schema(fields: &[&str], len: usize) -> SeriesSchema { + let base = test_schema(fields); + SeriesSchema { + fields: base.fields, + eval_grid: Some(Arc::new( + crate::series::time::EvalGrid::try_new( + crate::series::time::GridSpec::try_new(0, 1, len as u64).unwrap(), + ) + .unwrap(), + )), + } + } + + fn state_frame( + kind: Kind, + schema: &SeriesSchema, + spans: Vec, + states: Vec>>, + last: bool, + ) -> (RecordBatch, SeriesFrameDecoder) { + let components = match kind { + Kind::Sum => vec![Component::Value, Component::Compensation], + Kind::Count => vec![Component::Count], + Kind::Avg => vec![ + Component::Value, + Component::Mean, + Component::Compensation, + Component::Count, + Component::IncrementalMean, + ], + Kind::Min | Kind::Max => vec![Component::Value], + Kind::Group => vec![], + Kind::Stddev | Kind::Stdvar => { + vec![Component::Value, Component::Mean, Component::Count] + } + }; + let state = AggregateStateSchema::try_new(kind, components).unwrap(); + let time = TimeAxis::from_grid_spans(schema.eval_grid.clone().unwrap(), spans).unwrap(); + let cursor = SeriesFrameCursor::try_new_aggregate_state( + &state, + schema, + StateSeries { + metric: metric(Some("m"), &[("job", "x")]), + time, + states, + }, + last, + ) + .unwrap(); + let mut builder = SeriesFrameBuilder::try_new_aggregate_state( + &state, + schema, + FrameLabelMode::Explicit, + 8, + 1 << 20, + ) + .unwrap(); + let mut cursor = cursor; + assert!(cursor.append_to(&mut builder).unwrap().is_none()); + let batch = builder.flush().unwrap().unwrap(); + let decoder = + SeriesFrameDecoder::try_new_aggregate_state(&state, schema, FrameLabelMode::Explicit) + .unwrap(); + (batch, decoder) + } + + fn values(n: usize, seed: f64) -> Vec> { + (0..n) + .map(|i| { + Some(Values { + value: seed + i as f64, + mean: seed + 10.0 + i as f64, + compensation: seed + 20.0 + i as f64, + count: seed + 30.0 + i as f64, + incremental_mean: i % 2 == 0, + }) + }) + .collect() + } + + #[test] + fn group_state_keeps_field_local_presence_through_ipc_and_slices() { + let schema = state_schema(&["a", "b"], 6); + let (batch, decoder) = state_frame( + Kind::Group, + &schema, + vec![GridSpan::new(0, 4)], + vec![ + vec![Some(Values::default()), None, Some(Values::default()), None], + vec![None; 4], + ], + true, + ); + let expected = decoder.decode_aggregate_state(&batch).unwrap(); + let baseline_metric = expected[0].metric().clone(); + let baseline_positions = expected[0].time().positions().collect::>(); + let baseline_last = expected[0].is_last(); + let check = |rows: Vec| { + let row = &rows[0]; + let fields = row.fields().unwrap(); + let presence = |field| { + (0..4) + .map(|i| fields.is_present(field, i).unwrap()) + .collect::>() + }; + assert_eq!(presence(0), vec![true, false, true, false]); + assert_eq!(presence(1), vec![false; 4]); + assert_eq!(row.metric(), &baseline_metric); + assert_eq!( + row.time().positions().collect::>(), + baseline_positions + ); + assert_eq!(row.is_last(), baseline_last); + }; + check(expected); + let mut ipc = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut ipc, &batch.schema()).unwrap(); + writer.write(&batch).unwrap(); + writer.finish().unwrap(); + } + let batches = StreamReader::try_new(std::io::Cursor::new(ipc), None) + .unwrap() + .map(|batch| batch.unwrap()) + .collect::>(); + let joined = concat_batches(&batch.schema(), &batches).unwrap(); + check(decoder.decode_aggregate_state(&joined).unwrap()); + check(decoder.decode_aggregate_state(&joined.slice(0, 1)).unwrap()); + } + + #[test] + fn aggregate_state_kinds_round_trip_values_presence_labels_axis_and_finality() { + let kinds = [ + Kind::Sum, + Kind::Count, + Kind::Avg, + Kind::Min, + Kind::Max, + Kind::Group, + Kind::Stddev, + Kind::Stdvar, + ]; + for kind in kinds { + let schema = state_schema(&["x", "y"], 5); + let (batch, decoder) = state_frame( + kind, + &schema, + vec![GridSpan::new(1, 2), GridSpan::new(4, 1)], + vec![ + values(3, 1.0), + vec![ + None, + Some(values(3, 1.0)[1].unwrap()), + Some(values(3, 1.0)[2].unwrap()), + ], + ], + true, + ); + let rows = decoder.decode_aggregate_state(&batch).unwrap(); + assert_eq!(rows.len(), 1); + let row = &rows[0]; + assert_eq!(row.metric(), &metric(Some("m"), &[("job", "x")])); + assert_eq!( + row.time().positions().map(|(p, _)| p).collect::>(), + vec![1, 2, 4] + ); + assert!(row.is_last()); + let fields = row.fields().unwrap(); + assert_eq!(fields.kind(), kind); + assert_eq!( + (0..3) + .map(|i| fields.is_present(0, i).unwrap()) + .collect::>(), + vec![true; 3] + ); + assert_eq!( + (0..3) + .map(|i| fields.is_present(1, i).unwrap()) + .collect::>(), + vec![false, true, true] + ); + for field in 0..2 { + for (component, expected) in [ + (Component::Value, [1.0, 2.0, 3.0]), + (Component::Mean, [11.0, 12.0, 13.0]), + (Component::Compensation, [21.0, 22.0, 23.0]), + (Component::Count, [31.0, 32.0, 33.0]), + ] { + if let Some(array) = fields.field_values(field, component).unwrap() { + for (i, value) in expected.iter().enumerate() { + if fields.is_present(field, i).unwrap() { + assert_eq!(array.value(i), *value); + } + } + } + } + } + } + } + + #[test] + fn aggregate_state_all_kinds_keep_values_and_presence_through_ipc_slicing_and_concat() { + let kinds = [ + Kind::Sum, + Kind::Count, + Kind::Avg, + Kind::Min, + Kind::Max, + Kind::Group, + Kind::Stddev, + Kind::Stdvar, + ]; + let components = [ + Component::Value, + Component::Mean, + Component::Compensation, + Component::Count, + Component::IncrementalMean, + ]; + + for kind in kinds { + let schema = state_schema(&["x", "y"], 5); + let source_values = values(4, 3.0); + let states = || { + vec![ + source_values.clone(), + vec![ + None, + source_values[1], + Some(values(4, 30.0)[2].unwrap()), + None, + ], + ] + }; + // Two rows, so slicing and re-concatenating at an offset has something to shift: with + // one row, a decoder that ignored the offset would still agree with the baseline. + let (first, decoder) = state_frame( + kind, + &schema, + vec![GridSpan::new(0, 3), GridSpan::new(4, 1)], + states(), + false, + ); + let (second, _) = state_frame( + kind, + &schema, + vec![GridSpan::new(0, 3), GridSpan::new(4, 1)], + states(), + true, + ); + let batch = concat_batches(&first.schema(), &[first, second]).unwrap(); + let baseline = decoder.decode_aggregate_state(&batch).unwrap(); + assert_eq!(baseline.len(), 2); + + // This migrates #10080's property: IPC bytes plus batch concatenation must not change + // field-local presence, accumulator values, labels, axis, or finality. + let mut ipc = Vec::new(); + { + let mut writer = StreamWriter::try_new(&mut ipc, &batch.schema()).unwrap(); + writer.write(&batch).unwrap(); + writer.finish().unwrap(); + } + let read_back = StreamReader::try_new(std::io::Cursor::new(ipc), None) + .unwrap() + .map(|batch| batch.unwrap()) + .collect::>(); + assert_eq!(read_back.len(), 1); + let ipc_batch = &read_back[0]; + assert_eq!(ipc_batch.num_rows(), baseline.len()); + // Both halves carry a non-zero offset after concatenation, so the decoder has to honour + // the child arrays' offsets rather than reading from their first element. + let sliced = vec![ipc_batch.slice(1, 1), ipc_batch.slice(0, 1)]; + let joined = concat_batches(&batch.schema(), &sliced).unwrap(); + assert_eq!( + joined.num_rows(), + baseline.len(), + "{kind:?}: every row survives slicing and concatenation" + ); + + let check = |rows: &[StateFrameRow], expected: &[&StateFrameRow], context: &str| { + assert_eq!(rows.len(), expected.len(), "{context}: row count"); + for (row, expected) in rows.iter().zip(expected) { + let expected: &StateFrameRow = expected; + assert_eq!(row.metric(), expected.metric(), "{context}: labels"); + assert_eq!(row.is_last(), expected.is_last(), "{context}: finality"); + assert_eq!( + row.time().positions().collect::>(), + expected.time().positions().collect::>(), + "{context}: positions" + ); + let actual_fields = row.fields().unwrap(); + let expected_fields = expected.fields().unwrap(); + assert_eq!( + actual_fields.kind(), + expected_fields.kind(), + "{context}: kind" + ); + assert_eq!( + actual_fields.components(), + expected_fields.components(), + "{context}: components" + ); + if kind == Kind::Avg { + // Pin the full set once, so iterating a candidate list cannot silently + // skip every component a kind does not have. + assert_eq!( + actual_fields.components(), + &components, + "{context}: av components" + ); + } + for field in 0..expected_fields.len() { + let positions = expected_fields + .field(field) + .expect("a declared field has an item array") + .len(); + for position in 0..positions { + assert_eq!( + actual_fields.is_present(field, position).unwrap(), + expected_fields.is_present(field, position).unwrap(), + "{context}: presence {kind:?} field {field} position {position}" + ); + } + for component in components { + // Only the incremental mean is boolean; the rest are float64. Asking + // for the wrong one is itself an error, so the branch is by component. + if component == Component::IncrementalMean { + let expected_values = + expected_fields.field_bool_values(field, component).unwrap(); + let actual_values = + actual_fields.field_bool_values(field, component).unwrap(); + match (expected_values, actual_values) { + (Some(expected_values), Some(actual_values)) => { + for position in 0..expected_values.len() { + if expected_fields.is_present(field, position).unwrap() + { + assert_eq!( + actual_values.value(position), + expected_values.value(position), + "{context}: {kind:?} {component:?} \ + field {field} position {position}" + ); + } + } + } + (None, None) => {} + _ => panic!( + "{context}: {kind:?} {component:?} component changed" + ), + } + continue; + } + let expected_values = + expected_fields.field_values(field, component).unwrap(); + let actual_values = + actual_fields.field_values(field, component).unwrap(); + match (expected_values, actual_values) { + (Some(expected_values), Some(actual_values)) => { + for position in 0..expected_values.len() { + if expected_fields.is_present(field, position).unwrap() { + assert_eq!( + actual_values.value(position), + expected_values.value(position), + "{context}: {kind:?} {component:?} field {field} \ + position {position}" + ); + } + } + } + (None, None) => {} + _ => panic!("{context}: {kind:?} {component:?} component changed"), + } + } + } + } + }; + // The slices are concatenated in reverse row order on purpose: the decoded order must + // follow the concatenation, not the batch the rows came from. + check( + &decoder.decode_aggregate_state(&joined).unwrap(), + &[&baseline[1], &baseline[0]], + "ipc+concat", + ); + // A slice of the joined batch: the decoder must honour the offset rather than reading + // the first rows of the underlying arrays. + let tail = joined.slice(1, joined.num_rows() - 1); + check( + &decoder.decode_aggregate_state(&tail).unwrap(), + &[&baseline[0]], + "sliced", + ); + } + } + + #[test] + fn aggregate_state_cursor_moves_dense_states_without_filtering() { + let schema = state_schema(&["a", "b"], 4); + let state = AggregateStateSchema::try_new( + Kind::Avg, + vec![ + Component::Value, + Component::Mean, + Component::Compensation, + Component::Count, + Component::IncrementalMean, + ], + ) + .unwrap(); + let states = vec![values(4, 1.0), values(4, 5.0)]; + let allocation = states[0].as_ptr(); + // A grid axis rather than a span axis, so the test can tell the chunk's own axis from a + // rebuilt one. + let time = + TimeAxis::Grid(GridRange::try_new(schema.eval_grid.clone().unwrap(), 0, 4).unwrap()); + let cursor = SeriesFrameCursor::try_new_aggregate_state( + &state, + &schema, + StateSeries { + metric: metric(Some("m"), &[]), + time, + states, + }, + true, + ) + .unwrap(); + let ChunkWalk::State(walk) = cursor.walk else { + panic!("expected state walk") + }; + // The dense fast path preserves the caller's field allocation and values verbatim. + assert_eq!(walk.series.states[0].as_ptr(), allocation); + assert_eq!(walk.series.states, vec![values(4, 1.0), values(4, 5.0)]); + assert_eq!( + walk.series + .time + .positions() + .map(|(p, _)| p) + .collect::>(), + vec![0, 1, 2, 3] + ); + // And it keeps the axis it was handed. A rebuilt axis always states spans, so a grid axis + // surviving here is what says the dense case did not rebuild one. + assert!( + matches!(walk.series.time, TimeAxis::Grid(_)), + "a dense chunk kept its own axis: {:?}", + walk.series.time + ); + } + + /// Every component of one field is absent exactly where that field is, so they share one mask. + /// + /// A position is carried when any field has a state there, so a hole needs a second field: the + /// two fields below cover each other's positions and each is absent where the other is present. + #[test] + fn state_components_share_their_fields_presence_mask() { + let schema = state_schema(&["x", "y"], 4); + let values = values(3, 1.0); + let (batch, decoder) = state_frame( + Kind::Avg, + &schema, + vec![GridSpan::new(0, 3)], + vec![ + vec![Some(values[0].unwrap()), None, Some(values[2].unwrap())], + vec![None, Some(values[1].unwrap()), None], + ], + true, + ); + let rows = decoder.decode_aggregate_state(&batch).unwrap(); + assert_eq!(rows.len(), 1); + let fields = rows[0].fields().unwrap(); + assert_eq!(fields.len(), 2); + let mut masks = Vec::new(); + // Field 0 is present at {0,2} and field 1 at {1}, so their masks differ in both content and + // hole count. + for (field, holes) in [1usize, 2].into_iter().enumerate() { + for component in [ + Component::Value, + Component::Mean, + Component::Compensation, + Component::Count, + ] { + let array = fields.field_values(field, component).unwrap().unwrap(); + let nulls = array.nulls().expect("a hole in this field"); + assert_eq!(nulls.null_count(), holes, "field {field} {component:?}"); + masks.push(nulls.inner().inner().as_ptr()); + } + let boolean = fields + .field_bool_values(field, Component::IncrementalMean) + .unwrap() + .expect("avg declares the incremental mean"); + masks.push( + boolean + .nulls() + .expect("the same hole") + .inner() + .inner() + .as_ptr(), + ); + } + // One mask per field, shared by that field's components: five components of field 0 agree, + // then five of field 1. + assert!( + masks[..5].windows(2).all(|pair| pair[0] == pair[1]) + && masks[5..].windows(2).all(|pair| pair[0] == pair[1]), + "one mask per field, shared by its components: {masks:?}" + ); + assert_ne!( + masks[0], masks[5], + "two fields with different holes are two masks" + ); + } + + /// A field with no hole states no mask at all, which is what a null buffer builder produces. + #[test] + fn a_field_without_holes_carries_no_null_buffer() { + let schema = state_schema(&["x"], 3); + let (batch, decoder) = state_frame( + Kind::Sum, + &schema, + vec![GridSpan::new(0, 3)], + vec![values(3, 1.0)], + true, + ); + let rows = decoder.decode_aggregate_state(&batch).unwrap(); + let fields = rows[0].fields().unwrap(); + let array = fields + .field_values(0, Component::Value) + .unwrap() + .expect("sum declares the value"); + assert!(array.nulls().is_none()); + let mean = fields + .field_bool_values(0, Component::IncrementalMean) + .unwrap(); + assert!(mean.is_none(), "sum declares no incremental mean"); + } + + #[test] + fn aggregate_state_cursor_compacts_sparse_fields_and_preserves_runs() { + let schema = state_schema(&["a", "b"], 7); + let state = AggregateStateSchema::try_new( + Kind::Sum, + vec![Component::Value, Component::Compensation], + ) + .unwrap(); + let a = values(4, 1.0); + let b = values(4, 5.0); + let input = vec![ + vec![Some(a[0].unwrap()), None, Some(a[2].unwrap()), None], + vec![None, Some(b[1].unwrap()), None, Some(b[3].unwrap())], + ]; + let time = TimeAxis::from_grid_spans( + schema.eval_grid.clone().unwrap(), + vec![ + GridSpan::new(0, 1), + GridSpan::new(2, 1), + GridSpan::new(4, 1), + GridSpan::new(6, 1), + ], + ) + .unwrap(); + let cursor = SeriesFrameCursor::try_new_aggregate_state( + &state, + &schema, + StateSeries { + metric: metric(Some("m"), &[]), + time, + states: input, + }, + false, + ) + .unwrap(); + let ChunkWalk::State(walk) = cursor.walk else { + panic!("expected state walk") + }; + assert_eq!( + walk.series + .time + .positions() + .map(|(p, _)| p) + .collect::>(), + vec![0, 2, 4, 6] + ); + assert_eq!( + walk.runs + .iter() + .map(|run| (run.position, run.len)) + .collect::>(), + vec![(0, 1), (2, 1), (4, 1), (6, 1)] + ); + assert_eq!( + walk.series.states[0], + vec![Some(a[0].unwrap()), None, Some(a[2].unwrap()), None] + ); + assert_eq!( + walk.series.states[1], + vec![None, Some(b[1].unwrap()), None, Some(b[3].unwrap())] + ); + } + + #[test] + fn sparse_state_runs_and_empty_batch_decode() { + let schema = state_schema(&["x"], 10); + let (batch, decoder) = state_frame( + Kind::Sum, + &schema, + vec![GridSpan::new(0, 3), GridSpan::new(5, 3)], + vec![values(6, 0.0)], + true, + ); + let rows = decoder.decode_aggregate_state(&batch).unwrap(); + assert_eq!( + rows[0] + .time() + .runs() + .map(|r| (r.position, r.len)) + .collect::>(), + vec![(0, 3), (5, 3)] + ); + assert!( + decoder + .decode_aggregate_state(&batch.slice(0, 0)) + .unwrap() + .is_empty() + ); + } +} + +#[cfg(test)] +mod aggregate_state_decoder_rejections { + use super::*; + use crate::series::aggregate_state::{ + AggregateStateKind as Kind, AggregateStateSchema, AggregateStateValues as Values, + StateComponent as Component, StateSeries, + }; + + fn fixture() -> (RecordBatch, SeriesFrameDecoder) { + let base = test_schema(&["x"]); + let schema = SeriesSchema { + fields: base.fields, + eval_grid: Some(Arc::new( + crate::series::time::EvalGrid::try_new( + crate::series::time::GridSpec::try_new(0, 1, 3).unwrap(), + ) + .unwrap(), + )), + }; + let state = AggregateStateSchema::try_new( + Kind::Sum, + vec![Component::Value, Component::Compensation], + ) + .unwrap(); + let time = + TimeAxis::from_grid_spans(schema.eval_grid.clone().unwrap(), vec![GridSpan::new(0, 3)]) + .unwrap(); + let cursor = SeriesFrameCursor::try_new_aggregate_state( + &state, + &schema, + StateSeries { + metric: metric(Some("m"), &[]), + time, + states: vec![vec![ + Some(Values { + value: 1.0, + compensation: 2.0, + ..Values::default() + }); + 3 + ]], + }, + true, + ) + .unwrap(); + let mut cursor = cursor; + let mut builder = SeriesFrameBuilder::try_new_aggregate_state( + &state, + &schema, + FrameLabelMode::Explicit, + 4, + 4096, + ) + .unwrap(); + assert!(cursor.append_to(&mut builder).unwrap().is_none()); + let batch = builder.flush().unwrap().unwrap(); + let decoder = + SeriesFrameDecoder::try_new_aggregate_state(&state, &schema, FrameLabelMode::Explicit) + .unwrap(); + (batch, decoder) + } + + fn replace_field(batch: &RecordBatch, replacement: ArrayRef) -> RecordBatch { + let old = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap(); + let mut columns = old.columns().to_vec(); + columns[0] = replacement; + let fields = Arc::new( + StructArray::try_new(old.fields().clone(), columns, old.nulls().cloned()).unwrap(), + ); + let mut batch_columns = batch.columns().to_vec(); + batch_columns[3] = fields; + RecordBatch::try_new(batch.schema(), batch_columns).unwrap() + } + + #[test] + fn present_state_item_with_missing_component_is_rejected() { + let (batch, decoder) = fixture(); + let list = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap() + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let item = list + .values() + .as_any() + .downcast_ref::() + .unwrap(); + let mut columns = item.columns().to_vec(); + columns[0] = Arc::new(Float64Array::from(vec![None, Some(1.0), Some(1.0)])); + let broken_item = Arc::new( + StructArray::try_new(item.fields().clone(), columns, item.nulls().cloned()).unwrap(), + ); + let broken_list = Arc::new( + ListArray::try_new( + match list.data_type() { + DataType::List(f) => f.clone(), + _ => unreachable!(), + }, + list.offsets().clone(), + broken_item, + list.nulls().cloned(), + ) + .unwrap(), + ); + let broken = replace_field(&batch, broken_list); + assert!(decoder.decode_aggregate_state(&broken).is_err()); + } + + #[test] + fn state_list_length_must_match_carried_positions() { + let (batch, decoder) = fixture(); + let list = batch + .column(3) + .as_any() + .downcast_ref::() + .unwrap() + .column(0) + .as_any() + .downcast_ref::() + .unwrap(); + let broken = Arc::new( + ListArray::try_new( + match list.data_type() { + DataType::List(f) => f.clone(), + _ => unreachable!(), + }, + OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 2])), + list.values().slice(0, 2), + None, + ) + .unwrap(), + ); + assert!( + decoder + .decode_aggregate_state(&replace_field(&batch, broken)) + .is_err() + ); + } +} diff --git a/src/promql/src/aggregate.rs b/src/promql/src/aggregate.rs index 7d99ff1567a..0fe0ebcf8e9 100644 --- a/src/promql/src/aggregate.rs +++ b/src/promql/src/aggregate.rs @@ -16,13 +16,11 @@ //! is keyed by absolute grid index, never by local row number; sparse state and trimmed output //! do not expand the full grid. -mod state; +pub(crate) mod state; #[cfg(test)] mod tests; -pub use state::{ - AccumulatorState, FloatMerger, PartialAggregate, PartialGroup, StateField, merge_count_values, -}; +pub use state::{AccumulatorState, FloatMerger, PartialAggregate, StateField, merge_count_values}; use std::cmp::Reverse; use std::collections::hash_map::Entry; diff --git a/src/promql/src/aggregate/state.rs b/src/promql/src/aggregate/state.rs index 82f14887aa6..66d2ca4601c 100644 --- a/src/promql/src/aggregate/state.rs +++ b/src/promql/src/aggregate/state.rs @@ -31,9 +31,14 @@ use std::collections::{BTreeMap, HashMap}; use std::mem::size_of; use std::sync::Arc; +use datafusion::arrow::array::Array; use datafusion::error::{DataFusionError, Result}; use datatypes::series::SeriesLabels; +use datatypes::series::aggregate_state::{ + AggregateStateKind, AggregateStateValues, StateComponent, StateFieldsBatch, StateSeries, +}; use datatypes::series::samples::{SampleRef, Series, SeriesSchema}; +use datatypes::series::time::{GridSpan, TimeAxis}; use crate::aggregate::{ AggregateOp, BLOCK_POINTS, Counts, FloatAggregator, FloatState, Group, LabelKey, @@ -75,89 +80,120 @@ impl AggregateOp { } /// The state of one field of one group at one grid point. -#[derive(Clone, Copy, Debug, Default, PartialEq)] -pub struct AccumulatorState { - pub value: f64, - pub mean: f64, - pub compensation: f64, - pub count: f64, - pub incremental_mean: bool, -} - -/// The state of one group: its points in ascending order, each with one state per field, `None` -/// where no sample of that field was folded in. -#[derive(Clone, Debug)] -pub struct PartialGroup { - pub metric: SeriesLabels, - pub points: Vec<(usize, Vec>)>, -} +/// +/// The fold and the frame carry the same five components, so they share one type: the frame's own +/// [`AggregateStateValues`] is the definition, and this is the name the aggregate side reads it by. +pub use datatypes::series::aggregate_state::AggregateStateValues as AccumulatorState; /// The state of one execution partition, with its groups in the order they first appeared. +/// +/// A group is the frame's own [`StateSeries`]: field-major states addressed on that group's sparse +/// absolute grid positions. Exporting therefore hands the same representation over instead of +/// transposing a second one into it. #[derive(Clone, Debug)] pub struct PartialAggregate { pub op: AggregateOp, - pub groups: Vec, + pub groups: Vec, +} + +/// One Arrow-backed aggregate state row, independent of executor plumbing. +#[derive(Clone, Debug)] +pub(crate) struct StateRow { + pub metric: SeriesLabels, + pub time: TimeAxis, + pub fields: StateFieldsBatch, + pub kind: AggregateStateKind, } impl FloatAggregator { /// The state folded so far, for a merge with the state of other partitions. - pub fn finish_partial(self) -> PartialAggregate { + pub fn finish_partial(self) -> Result { self.finish_partial_inner() } - /// [`Self::finish_partial`] that admits what it builds first: the fold state and the group, - /// point and state vectors, at the exact capacity [`Self::finish_partial_inner`] gives them. A - /// refusal consumes the aggregator, so the caller drops the execution. + /// [`Self::finish_partial`] that admits what it builds first: the fold state and the groups it + /// is exported as, at the exact capacity [`Self::finish_partial_inner`] gives them. A refusal + /// consumes the aggregator, so the caller drops the execution. pub(crate) fn finish_partial_budgeted( self, mut budget: impl FnMut(usize) -> Result<()>, ) -> Result { let width = self.schema.fields.len(); - let mut partial_bytes = self.groups.len().saturating_mul(size_of::()); + let mut partial_bytes = self.groups.len().saturating_mul(size_of::()); for group in &self.groups { let points = seen_points(group, width); partial_bytes = partial_bytes - .saturating_add(points.saturating_mul(size_of::())) + .saturating_add(size_of::()) + .saturating_add( + points.saturating_mul(size_of::().saturating_add(size_of::())), + ) + .saturating_add( + width.saturating_mul(size_of::>>()), + ) .saturating_add( points .saturating_mul(width) - .saturating_mul(size_of::>()), + .saturating_mul(size_of::>()), ); } budget(self.state_bytes().saturating_add(partial_bytes))?; - Ok(self.finish_partial_inner()) + self.finish_partial_inner() } - fn finish_partial_inner(self) -> PartialAggregate { + /// Turns the fold into the frame's own groups: field-major states on sparse absolute positions. + /// + /// The blocks are walked once, in ascending position order, so a position's states are appended + /// to every field at the same time and adjacent positions coalesce into one span as they come. + /// What comes out is the representation the frame writes; no per-point vector is built. + fn finish_partial_inner(self) -> Result { let width = self.schema.fields.len(); - let mut groups: Vec = Vec::with_capacity(self.groups.len()); + let grid = self.schema.eval_grid.as_ref().ok_or_else(|| { + DataFusionError::Execution("aggregate state is indexed on an evaluated grid".into()) + })?; + let grid_len = grid.spec().len as usize; + let mut groups: Vec = Vec::with_capacity(self.groups.len()); for group in self.groups { - let mut points: Vec = Vec::with_capacity(seen_points(&group, width)); - for (&block, states) in &group.blocks { - for (offset, point) in states.chunks_exact(width).enumerate() { - if point.iter().any(|state| state.seen) { - points.push(( - block * BLOCK_POINTS + offset, - point.iter().map(|state| state.partial()).collect(), - )); + let points = seen_points(&group, width); + let mut spans: Vec = Vec::with_capacity(points.min(1)); + let mut states: Vec>> = + (0..width).map(|_| Vec::with_capacity(points)).collect(); + for (&block, block_states) in &group.blocks { + for (offset, point) in block_states.chunks_exact(width).enumerate() { + if !point.iter().any(|state| state.seen) { + continue; + } + let position = block * BLOCK_POINTS + offset; + if position >= grid_len { + return Err(DataFusionError::Execution(format!( + "aggregate state point {position} exceeds a grid of {grid_len}" + ))); + } + match spans.last_mut() { + Some(span) if span.start() + span.len() == position => { + *span = GridSpan::new(span.start(), span.len() + 1); + } + _ => spans.push(GridSpan::new(position, 1)), + } + for (field, state) in point.iter().enumerate() { + states[field].push(state.partial()); } } } - groups.push(PartialGroup { + let time = TimeAxis::from_grid_spans(Arc::clone(grid), spans) + .map_err(|error| DataFusionError::Execution(error.to_string()))?; + groups.push(StateSeries { metric: group.metric, - points, + time, + states, }); } - PartialAggregate { + Ok(PartialAggregate { op: self.op, groups, - } + }) } } -/// One point of a partial group: its grid position and one state per field. -type Point = (usize, Vec>); - /// The points of `group` that carry a state: the ones a partial of it holds. fn seen_points(group: &Group, width: usize) -> usize { group.blocks.values().fold(0usize, |count, states| { @@ -170,16 +206,33 @@ fn seen_points(group: &Group, width: usize) -> usize { }) } -/// Bytes one partial group keeps: its labels and its point and state vectors at the capacity they -/// were built with, so spare capacity is paid for rather than skipped. -fn group_bytes(group: &PartialGroup) -> usize { - metric_bytes(&group.metric) - .saturating_add(group.points.capacity().saturating_mul(size_of::())) - .saturating_add(group.points.iter().fold(0usize, |bytes, (_, states)| { +/// Bytes one exported group keeps: its labels and the field vectors, at the capacity they were +/// built with, so spare capacity is paid for rather than skipped. +fn group_bytes(group: &StateSeries) -> usize { + metric_bytes(&group.metric).saturating_add(group_field_bytes(group)) +} + +/// Bytes a group's axis and field vectors hold, without its labels. +pub(crate) fn group_field_bytes(group: &StateSeries) -> usize { + size_of::() + .saturating_add( + group + .time + .runs() + .count() + .saturating_mul(size_of::().saturating_add(size_of::())), + ) + .saturating_add( + group + .states + .capacity() + .saturating_mul(size_of::>>()), + ) + .saturating_add(group.states.iter().fold(0usize, |bytes, field| { bytes.saturating_add( - states + field .capacity() - .saturating_mul(size_of::>()), + .saturating_mul(size_of::>()), ) })) } @@ -277,23 +330,33 @@ impl FloatMerger { ))); } for group in &partial.groups { - for (point, states) in &group.points { - if *point >= points || states.len() != width { - return Err(DataFusionError::Execution( - "aggregate state does not match the grid or fields of the merge".into(), - )); - } + let carried = group.time.len(); + if group.states.len() != width + || group.states.iter().any(|field| field.len() != carried) + { + return Err(DataFusionError::Execution( + "aggregate state does not match the grid or fields of the merge".into(), + )); + } + if group.time.positions().any(|(point, _)| point >= points) { + return Err(DataFusionError::Execution( + "aggregate state does not match the grid or fields of the merge".into(), + )); } } let input_live = partial.groups.iter().fold( partial .groups .capacity() - .saturating_mul(size_of::()), + .saturating_mul(size_of::()), |bytes, group| bytes.saturating_add(group_bytes(group)), ); for group in partial.groups { - let metric = group.metric; + let StateSeries { + metric, + time, + states, + } = group; // `label_key` copies the metric name into its own `Arc`, so the copy is admitted // before it is made. A group that is already merged needs a key too, and its // admission comes before anything else this call can allocate. @@ -308,7 +371,95 @@ impl FloatMerger { Some(index) => index, None => self.open_group(metric, key, live, &mut budget)?, }; - self.merge_points(index, group.points, live, &mut budget)?; + self.merge_series(index, &time, &states, live, &mut budget)?; + } + Ok(()) + } + + /// Merges one Arrow-backed state row directly, without materializing a partial aggregate. + pub(crate) fn merge_row( + &mut self, + row: &StateRow, + mut budget: impl FnMut(usize) -> Result<()>, + ) -> Result<()> { + let op = crate::native::state::op_of_state(row.kind); + if op != self.op { + return Err(DataFusionError::Execution(format!( + "cannot merge {:?} state into {:?}", + op, self.op + ))); + } + let width = self.schema.fields.len(); + if row.fields.len() != width { + return Err(DataFusionError::Execution( + "aggregate state does not match the grid or fields of the merge".into(), + )); + } + let grid_len = self + .schema + .eval_grid + .as_ref() + .expect("checked at construction") + .spec() + .len as usize; + let mut opened = None; + let key_workspace = key_workspace_bytes(&row.metric); + let mut live = None; + for (point, position) in row.time.positions() { + if point >= grid_len { + return Err(DataFusionError::Execution( + "aggregate state does not match the grid or fields of the merge".into(), + )); + } + let mut present = false; + for field in 0..width { + present |= row + .fields + .is_present(field, position) + .map_err(crate::grid_series::external)?; + } + if !present { + return Err(DataFusionError::Execution( + "aggregate state names a position with no contribution".into(), + )); + } + let index = match opened { + Some(index) => index, + None => { + budget(self.retained.saturating_add(key_workspace))?; + let key = label_key(&row.metric); + let live_bytes = key_bytes(&key); + let index = match self.group_index.get(&key).copied() { + Some(index) => index, + None => { + self.open_group(row.metric.clone(), key, live_bytes, &mut budget)? + } + }; + live = Some(live_bytes); + opened = Some(index); + index + } + }; + let mut field = 0; + let mut state_error = None; + let states = std::iter::from_fn(|| { + if field == width || state_error.is_some() { + return None; + } + let current = field; + field += 1; + match state_at(&row.fields, current, position) { + Ok(state) => Some(state), + Err(error) => { + state_error = Some(error); + None + } + } + }); + self.write_point(index, point, states, live.unwrap_or(0), &mut budget)?; + if let Some(error) = state_error { + return Err(error); + } } Ok(()) } @@ -381,10 +532,29 @@ impl FloatMerger { /// Adds the points of one partial group to group `index`. A missing block is admitted with its /// states and B-tree node before either is allocated; an existing one only merges state. - fn merge_points( + fn merge_series( &mut self, index: usize, - points: Vec, + time: &TimeAxis, + states: &[Vec>], + live: usize, + budget: &mut impl FnMut(usize) -> Result<()>, + ) -> Result<()> { + // The axis carries the absolute positions and the row each one sits at, so a group's states + // are read field by field at that row: this is the same order the point-major form had. + for (point, row) in time.positions() { + let point_states = states.iter().map(|field| field[row]); + self.write_point(index, point, point_states, live, budget)?; + } + Ok(()) + } + + /// Writes one point's states, admitting a missing block before allocating it. + fn write_point( + &mut self, + index: usize, + point: usize, + states: impl Iterator>, live: usize, budget: &mut impl FnMut(usize) -> Result<()>, ) -> Result<()> { @@ -393,33 +563,29 @@ impl FloatMerger { .saturating_mul(width) .saturating_mul(size_of::()); let blocks = &mut self.groups[index].blocks; - for (point, states) in points { - let block_index = point / BLOCK_POINTS; - let entries = blocks.len(); - let block = match blocks.entry(block_index) { - BTreeEntry::Occupied(entry) => entry.into_mut(), - BTreeEntry::Vacant(entry) => { - budget( - self.retained - .saturating_add(live) - .saturating_add(block_bytes) - .saturating_add(btree_backing_bytes(entries.saturating_add(1))), - )?; - self.retained = self - .retained + let block_index = point / BLOCK_POINTS; + let entries = blocks.len(); + let block = match blocks.entry(block_index) { + BTreeEntry::Occupied(entry) => entry.into_mut(), + BTreeEntry::Vacant(entry) => { + budget( + self.retained + .saturating_add(live) .saturating_add(block_bytes) - .saturating_add(btree_backing_bytes(entries.saturating_add(1))) - .saturating_sub(btree_backing_bytes(entries)); - entry.insert( - vec![FloatState::default(); BLOCK_POINTS * width].into_boxed_slice(), - ) - } - }; - let offset = (point % BLOCK_POINTS) * width; - for (slot, state) in block[offset..offset + width].iter_mut().zip(states) { - if let Some(state) = state { - slot.merge(self.op, FloatState::from_partial(state)); - } + .saturating_add(btree_backing_bytes(entries.saturating_add(1))), + )?; + self.retained = self + .retained + .saturating_add(block_bytes) + .saturating_add(btree_backing_bytes(entries.saturating_add(1))) + .saturating_sub(btree_backing_bytes(entries)); + entry.insert(vec![FloatState::default(); BLOCK_POINTS * width].into_boxed_slice()) + } + }; + let offset = (point % BLOCK_POINTS) * width; + for (slot, state) in block[offset..offset + width].iter_mut().zip(states) { + if let Some(state) = state { + slot.merge(self.op, FloatState::from_partial(state)); } } Ok(()) @@ -440,6 +606,58 @@ impl FloatMerger { } } +/// One present field's state at one position, read from the row's component arrays. +/// +/// The components come from the row's own declaration; the native aggregate caller checks +/// compatibility before this runs, so a component list that disagrees is caught before this runs. +fn state_at( + fields: &StateFieldsBatch, + field: usize, + position: usize, +) -> Result> { + if !fields + .is_present(field, position) + .map_err(crate::grid_series::external)? + { + return Ok(None); + } + let mut state = AccumulatorState::default(); + for &component in fields.components() { + match component { + StateComponent::IncrementalMean => { + state.incremental_mean = fields + .field_bool_values(field, component) + .map_err(crate::grid_series::external)? + .and_then(|array| (!array.is_null(position)).then(|| array.value(position))) + .ok_or_else(|| { + DataFusionError::Execution( + "present aggregate state has a missing component".into(), + ) + })?; + } + _ => { + let value = fields + .field_values(field, component) + .map_err(crate::grid_series::external)? + .and_then(|array| (!array.is_null(position)).then(|| array.value(position))) + .ok_or_else(|| { + DataFusionError::Execution( + "present aggregate state has a missing component".into(), + ) + })?; + match component { + StateComponent::Value => state.value = value, + StateComponent::Mean => state.mean = value, + StateComponent::Compensation => state.compensation = value, + StateComponent::Count => state.count = value, + StateComponent::IncrementalMean => unreachable!(), + } + } + } + } + Ok(Some(state)) +} + impl FloatState { fn partial(&self) -> Option { self.seen.then_some(AccumulatorState { @@ -575,3 +793,137 @@ pub fn merge_count_values( } count_series(schema, groups) } + +#[cfg(test)] +mod tests { + use super::*; + use datafusion::arrow::array::{Float64Array, ListArray, StructArray}; + use datafusion::arrow::buffer::{NullBuffer, OffsetBuffer, ScalarBuffer}; + use datafusion::arrow::datatypes::{DataType, Field}; + use datatypes::series::aggregate_state::{AggregateStateSchema, StateComponent}; + use datatypes::series::samples::FieldSpec; + use datatypes::series::time::{EvalGrid, GridRange, GridSpec, TimeAxis}; + + fn fields_batch() -> StateFieldsBatch { + let item_fields: datafusion::arrow::datatypes::Fields = vec![ + Field::new("value", DataType::Float64, true), + Field::new("compensation", DataType::Float64, true), + ] + .into(); + let make_field = + |values: Vec>, compensation: Vec>, validity: Vec| { + let item = StructArray::new( + item_fields.clone(), + vec![ + Arc::new(Float64Array::from(values)), + Arc::new(Float64Array::from(compensation)), + ], + Some(NullBuffer::from(validity)), + ); + Arc::new(ListArray::new( + Arc::new(Field::new( + "item", + DataType::Struct(item_fields.clone()), + true, + )), + OffsetBuffer::new(ScalarBuffer::from(vec![0i32, 3])), + Arc::new(item), + None, + )) + }; + let field_a = make_field( + vec![Some(2.), Some(3.), Some(4.)], + vec![Some(0.), Some(0.), Some(0.)], + vec![true, true, true], + ); + let field_b = make_field( + vec![Some(5.), None, Some(7.)], + vec![Some(0.), None, Some(0.)], + vec![true, false, true], + ); + let outer_fields = vec![ + Field::new("a", field_a.data_type().clone(), false), + Field::new("b", field_b.data_type().clone(), false), + ] + .into(); + let outer = StructArray::new(outer_fields, vec![field_a, field_b], None); + let schema = AggregateStateSchema::try_new( + AggregateStateKind::Sum, + vec![StateComponent::Value, StateComponent::Compensation], + ) + .unwrap(); + StateFieldsBatch::try_from_arrow(outer, &schema).unwrap() + } + + fn schema() -> Arc { + Arc::new(SeriesSchema { + fields: vec![ + FieldSpec { name: "a".into() }, + FieldSpec { name: "b".into() }, + ] + .into(), + eval_grid: Some(Arc::new( + EvalGrid::try_new(GridSpec::try_new(0, 1, 3).unwrap()).unwrap(), + )), + }) + } + + #[test] + fn merge_row_matches_partial_merge_with_field_hole() { + let schema = schema(); + let metric = SeriesLabels::try_new(Some("m".into()), Vec::new()).unwrap(); + let row = StateRow { + metric: metric.clone(), + time: TimeAxis::Grid( + GridRange::try_new(schema.eval_grid.clone().unwrap(), 0, 3).unwrap(), + ), + fields: fields_batch(), + kind: AggregateStateKind::Sum, + }; + let mut from_row = FloatMerger::try_new(&schema, AggregateOp::Sum).unwrap(); + from_row.merge_row(&row, |_| Ok(())).unwrap(); + let mut from_partial = FloatMerger::try_new(&schema, AggregateOp::Sum).unwrap(); + from_partial + .merge(PartialAggregate { + op: AggregateOp::Sum, + groups: vec![StateSeries { + metric, + time: TimeAxis::Grid( + GridRange::try_new(schema.eval_grid.clone().unwrap(), 0, 3).unwrap(), + ), + states: vec![ + vec![ + Some(AccumulatorState { + value: 2.0, + ..Default::default() + }), + Some(AccumulatorState { + value: 3.0, + ..Default::default() + }), + Some(AccumulatorState { + value: 4.0, + ..Default::default() + }), + ], + vec![ + Some(AccumulatorState { + value: 5.0, + ..Default::default() + }), + None, + Some(AccumulatorState { + value: 7.0, + ..Default::default() + }), + ], + ], + }], + }) + .unwrap(); + assert_eq!( + format!("{:?}", from_row.finish().unwrap()), + format!("{:?}", from_partial.finish().unwrap()), + ); + } +} diff --git a/src/promql/src/aggregate/tests.rs b/src/promql/src/aggregate/tests.rs index 0e04941632c..a8aab466e29 100644 --- a/src/promql/src/aggregate/tests.rs +++ b/src/promql/src/aggregate/tests.rs @@ -1394,22 +1394,25 @@ fn assert_flat_close( fn to_layout(mut partial: PartialAggregate) -> PartialAggregate { let kept = partial.op.state_fields(); for group in &mut partial.groups { - for (_, states) in &mut group.points { - for state in states.iter_mut().flatten() { - let mut stripped = AccumulatorState::default(); - for field in kept { - match field { - StateField::Value => stripped.value = state.value, - StateField::Mean => stripped.mean = state.mean, - StateField::Compensation => stripped.compensation = state.compensation, - StateField::Count => stripped.count = state.count, - StateField::IncrementalMean => { - stripped.incremental_mean = state.incremental_mean - } + for state in group + .states + .iter_mut() + .flat_map(|field| field.iter_mut()) + .flatten() + { + let mut stripped = AccumulatorState::default(); + for field in kept { + match field { + StateField::Value => stripped.value = state.value, + StateField::Mean => stripped.mean = state.mean, + StateField::Compensation => stripped.compensation = state.compensation, + StateField::Count => stripped.count = state.count, + StateField::IncrementalMean => { + stripped.incremental_mean = state.incremental_mean } } - *state = stripped; } + *state = stripped; } } partial @@ -1432,7 +1435,7 @@ fn merged_and_serial(op: AggregateOp, partitions: &[&[f64]]) -> (f64, f64) { aggregator.update(&series).unwrap(); } merger - .merge(to_layout(aggregator.finish_partial())) + .merge(to_layout(aggregator.finish_partial().unwrap())) .unwrap(); } let all: Vec = partitions @@ -1471,11 +1474,22 @@ fn finish_partial_reserves_one_slot_per_seen_point() { aggregator.update(&series).unwrap(); let expected = aggregator .state_bytes() - .saturating_add(std::mem::size_of::()) + .saturating_add(std::mem::size_of::< + datatypes::series::aggregate_state::StateSeries, + >()) + .saturating_add(std::mem::size_of::()) .saturating_add( - SEEN.saturating_mul(std::mem::size_of::<(usize, Vec>)>()), + SEEN.saturating_mul( + std::mem::size_of::() + .saturating_add(std::mem::size_of::()), + ), ) - .saturating_add(SEEN.saturating_mul(std::mem::size_of::>())); + .saturating_add(std::mem::size_of::< + Vec>, + >()) + .saturating_add(SEEN.saturating_mul(std::mem::size_of::< + Option, + >())); let mut peak = 0usize; let partial = aggregator .finish_partial_budgeted(|bytes| { @@ -1483,9 +1497,11 @@ fn finish_partial_reserves_one_slot_per_seen_point() { Ok(()) }) .unwrap(); - // The vector is reserved once, at the points that carry a state, so pushing does not double it. - assert_eq!(partial.groups[0].points.len(), SEEN); - assert_eq!(partial.groups[0].points.capacity(), SEEN); + // Each field vector is reserved once, at the points that carry a state, so pushing does not + // double it and the export holds one vector per field rather than one per point. + assert_eq!(partial.groups[0].states.len(), 1); + assert_eq!(partial.groups[0].states[0].len(), SEEN); + assert_eq!(partial.groups[0].states[0].capacity(), SEEN); assert_eq!(peak, expected); } @@ -1496,20 +1512,17 @@ fn merge_charges_the_capacity_of_an_input_partial() { let mut aggregator = FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&[])).unwrap(); aggregator.update(&series).unwrap(); - let partial = aggregator.finish_partial(); - assert_eq!(partial.groups[0].points.capacity(), 1); + let partial = aggregator.finish_partial().unwrap(); + assert_eq!(partial.groups[0].states[0].capacity(), 1); let mut spare = partial.clone(); - let point = spare.groups[0].points.pop().unwrap(); - let mut points = Vec::with_capacity(1024); - points.push(point); - spare.groups[0].points = points; + spare.groups[0].states[0].reserve(1023); let peak = |partial: PartialAggregate| { let mut merger = FloatMerger::try_new(&schema, AggregateOp::Sum).unwrap(); let (peak, result) = peak_admission(|budget| merger.merge_budgeted(partial, budget)); result.unwrap(); peak }; - let slack = 1023 * std::mem::size_of::<(usize, Vec>)>(); + let slack = 1023 * std::mem::size_of::>(); assert!( peak(spare) >= peak(partial).saturating_add(slack), "spare capacity was not admitted" @@ -1526,8 +1539,8 @@ fn merge_charges_one_block_per_new_block_not_per_point() { let mut aggregator = FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&[])).unwrap(); aggregator.update(&series).unwrap(); - let partial = aggregator.finish_partial(); - assert_eq!(partial.groups[0].points.len(), 64); + let partial = aggregator.finish_partial().unwrap(); + assert_eq!(partial.groups[0].states[0].len(), 64); let block = BLOCK_POINTS * 64 * std::mem::size_of::(); let mut merger = FloatMerger::try_new(&schema, AggregateOp::Sum).unwrap(); let (peak, result) = peak_admission(|budget| merger.merge_budgeted(partial, budget)); @@ -1545,12 +1558,20 @@ fn merge_charges_one_block_per_new_block_not_per_point() { #[test] fn merge_admits_groups_and_blocks_before_it_inserts_them() { let schema = schema(&["value"]); + let empty_axis = || { + datatypes::series::time::TimeAxis::from_grid_spans( + schema.eval_grid.clone().unwrap(), + Vec::new(), + ) + .unwrap() + }; let partial = |ids: std::ops::Range| PartialAggregate { op: AggregateOp::Sum, groups: ids - .map(|id| PartialGroup { + .map(|id| datatypes::series::aggregate_state::StateSeries { metric: metric("m", &[("i", &id.to_string())]), - points: Vec::new(), + time: empty_axis(), + states: vec![Vec::new()], }) .collect(), }; @@ -1564,7 +1585,7 @@ fn merge_admits_groups_and_blocks_before_it_inserts_them() { assert!(merger.merge_budgeted(partial(100..108), refused).is_err()); assert_eq!(merger.state_bytes(), retained); let mut invalid = partial(200..201); - invalid.groups[0].points.push((256, Vec::new())); + invalid.groups[0].states[0].push(None); assert!(merger.merge(invalid).is_err()); assert_eq!(merger.state_bytes(), retained); assert!( @@ -1584,7 +1605,7 @@ fn merge_admits_the_key_before_it_builds_it() { let mut aggregator = FloatAggregator::try_new(&schema, AggregateOp::Sum, Grouping::By(&[])).unwrap(); aggregator.update(&series).unwrap(); - aggregator.finish_partial() + aggregator.finish_partial().unwrap() }; let mut merger = FloatMerger::try_new(&schema, AggregateOp::Sum).unwrap(); merger.merge(partial()).unwrap(); @@ -1622,9 +1643,14 @@ fn a_long_metric_name_is_held_by_the_group_and_by_its_key() { let retained = |name: &str| { let partial = PartialAggregate { op: AggregateOp::Sum, - groups: vec![PartialGroup { + groups: vec![datatypes::series::aggregate_state::StateSeries { metric: metric(name, &[]), - points: Vec::new(), + time: datatypes::series::time::TimeAxis::from_grid_spans( + schema.eval_grid.clone().unwrap(), + Vec::new(), + ) + .unwrap(), + states: vec![Vec::new()], }], }; let mut merger = FloatMerger::try_new(&schema, AggregateOp::Sum).unwrap(); @@ -1747,7 +1773,7 @@ fn partials_merge_to_the_serial_result() { aggregator.update(series).unwrap(); } merger - .merge(to_layout(aggregator.finish_partial())) + .merge(to_layout(aggregator.finish_partial().unwrap())) .unwrap(); } let merged = merger.finish().unwrap(); @@ -1760,7 +1786,7 @@ fn partials_merge_to_the_serial_result() { } let mut merger = FloatMerger::try_new(&schema, op).unwrap(); merger - .merge(to_layout(aggregator.finish_partial())) + .merge(to_layout(aggregator.finish_partial().unwrap())) .unwrap(); assert_flat_close( &merger.finish().unwrap(), @@ -1876,7 +1902,7 @@ fn equal_zero_extremes_follow_the_merge_order() { aggregator .update(&input(&schema, metric("m", &[]), 0, &[vec![Some(value)]])) .unwrap(); - merger.merge(aggregator.finish_partial()).unwrap(); + merger.merge(aggregator.finish_partial().unwrap()).unwrap(); } let value = values_of(&merger.finish().unwrap(), 0, 128).1[0].unwrap(); assert_eq!( @@ -1897,15 +1923,27 @@ fn a_merge_rejects_state_that_does_not_fit() { aggregator .update(&input(&schema, metric("m", &[]), 5, &[vec![Some(1.0)]])) .unwrap(); - aggregator.finish_partial() + aggregator.finish_partial().unwrap() }; let mut merger = FloatMerger::try_new(&schema, AggregateOp::Sum).unwrap(); assert!(merger.merge(partial(AggregateOp::Avg)).is_err()); + // A field with a state at a position the axis does not carry: the shape no longer fits. let mut wide = partial(AggregateOp::Sum); - wide.groups[0].points[0].1.push(None); + wide.groups[0].states[0].push(None); assert!(merger.merge(wide).is_err()); + // An axis whose positions belong to another grid: the merge is bound to its own. let mut late = partial(AggregateOp::Sum); - late.groups[0].points[0].0 = 128; + // The fixture's grid holds 128 positions, so a position from a longer grid is out of bounds. + let other_grid = Arc::new(EvalGrid::try_new(GridSpec::try_new(0, 1, 256).unwrap()).unwrap()); + late.groups[0].time = datatypes::series::time::TimeAxis::from_grid_spans( + other_grid, + vec![datatypes::series::time::GridSpan::new(200, 1)], + ) + .unwrap(); + late.groups[0].states[0] = vec![Some(AccumulatorState { + value: 1.0, + ..Default::default() + })]; assert!(merger.merge(late).is_err()); assert!(merger.finish().unwrap().is_empty()); } diff --git a/src/promql/src/extension_plan.rs b/src/promql/src/extension_plan.rs index a592f5bcdfa..453efb41222 100644 --- a/src/promql/src/extension_plan.rs +++ b/src/promql/src/extension_plan.rs @@ -13,8 +13,6 @@ // limitations under the License. mod absent; -mod aggregate_exchange; -mod aggregate_finalize; mod empty_metric; mod fused_series; mod histogram_fold; @@ -31,7 +29,6 @@ mod test_util; mod union_distinct_on; pub use absent::{Absent, AbsentExec, AbsentStream}; -pub use aggregate_finalize::AggregateFinalize; use common_query::native_histogram::{SUM_FIELD, native_histogram_value_type}; use common_query::prometheus::is_prometheus_stale_nan; use datafusion::arrow::array::{Array, Float64Array, StructArray}; diff --git a/src/promql/src/extension_plan/aggregate_exchange.rs b/src/promql/src/extension_plan/aggregate_exchange.rs deleted file mode 100644 index 162e4bd0c25..00000000000 --- a/src/promql/src/extension_plan/aggregate_exchange.rs +++ /dev/null @@ -1,486 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -//! The private Arrow layout of a partition's partial aggregate state, and the fold that writes it. -//! [`decode_state`] reads a batch back into the rows a `FloatMerger` merges. Only that producer and -//! that merge read [`state_schema`]: it is not a wire format, and has no serde or version. -//! -//! TODO(discord9): Replace this temporary state-row bridge with series frames emitted directly by -//! FusedSeries and consumed by native final aggregation, avoiding the extra state-row encoding, -//! decoding and reconstruction. This private layout is not the final intermediate representation. - -#[cfg(test)] -mod tests; - -use std::mem::size_of; -use std::sync::{Arc, Mutex}; - -use datafusion::arrow::array::{ - Array, ArrayRef, BooleanArray, Float64Array, StringArray, UInt32Array, UInt64Array, -}; -use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef}; -use datafusion::arrow::record_batch::RecordBatch; -use datafusion::error::{DataFusionError, Result as DfResult}; -use datafusion::execution::memory_pool::{MemoryConsumer, MemoryPool, MemoryReservation}; -use datatypes::series::samples::SeriesSchema; -use datatypes::series::{Label, SeriesLabels}; - -use crate::aggregate::{ - AccumulatorState, AggregateOp, FloatAggregator, Grouping, PartialAggregate, PartialGroup, -}; -use crate::evaluated::EvaluatedChunk; -use crate::grid_series::series_bytes; -use crate::native::plan::{ - AggregateKind, AggregateStage, BoundAggregate, Grouping as BoundGrouping, -}; - -/// Rows per encoded batch; the exchange has no session context to read a batch size from. -const BATCH_ROWS: usize = 4096; -/// One column's `ArrayRef`, its `Field` and its buffers are all well under this. -const COLUMN_OVERHEAD: usize = 256; -/// The value capacity a string column starts with, nulls or not (Arrow's `StringArray::from`). -const STRING_CAPACITY: usize = 1024; - -/// One state row: a group's labels, the point and field it is of, and its state. Label columns are -/// nullable, since absent and empty differ; `__name__` has the metric name's column. -pub(crate) fn state_schema(tags: &[String]) -> SchemaRef { - let mut fields = vec![Field::new("metric_name", DataType::Utf8, true)]; - let labels = ordinary_tags(tags).count(); - fields.extend((0..labels).map(|at| Field::new(format!("tag_{at}"), DataType::Utf8, true))); - fields.extend([ - Field::new("point", DataType::UInt64, false), - Field::new("field", DataType::UInt32, false), - ]); - for name in ["value", "mean", "compensation", "count"] { - fields.push(Field::new(name, DataType::Float64, false)); - } - fields.push(Field::new("incremental_mean", DataType::Boolean, false)); - Arc::new(Schema::new(fields)) -} - -/// The kernel a `Complete` aggregate binds; `quantile` and the other stages are not a fold. -pub(crate) fn aggregate_op(op: &BoundAggregate) -> DfResult { - if op.stage != AggregateStage::Complete { - return Err(DataFusionError::NotImplemented(format!( - "the aggregate state exchange folds only `Complete` aggregates, not `{:?}`", - op.stage - ))); - } - match op.kind { - AggregateKind::Sum => Ok(AggregateOp::Sum), - AggregateKind::Count => Ok(AggregateOp::Count), - AggregateKind::Avg => Ok(AggregateOp::Avg), - AggregateKind::Min => Ok(AggregateOp::Min), - AggregateKind::Max => Ok(AggregateOp::Max), - AggregateKind::Group => Ok(AggregateOp::Group), - AggregateKind::Stddev => Ok(AggregateOp::Stddev), - AggregateKind::Stdvar => Ok(AggregateOp::Stdvar), - AggregateKind::Quantile { quantile } => Err(DataFusionError::NotImplemented(format!( - "the aggregate state exchange does not carry `quantile {quantile}`" - ))), - } -} - -/// One partition's fold, and the state it encodes when its input ends. -pub(crate) struct PartialFold { - tags: Arc<[String]>, - pool: Arc, - /// What this fold holds beside the kernel's state, so each charge can be added to it. - held: usize, - reservation: MemoryReservation, - /// The plan's charge for exported batches, which live until its queue drains. - export_budget: Arc>>, - fold: Option, -} - -impl PartialFold { - /// Starts one partition's fold, its `export_budget` registered against `pool` at first export. - pub(crate) fn try_new( - op: BoundAggregate, - schema: Arc, - tags: Arc<[String]>, - export_budget: Arc>>, - pool: &Arc, - ) -> DfResult { - let kind = aggregate_op(&op)?; - let per_tag = size_of::() + size_of::() + size_of::(); - let held = tags.iter().map(|tag| tag.len() + per_tag).sum::() + 8 * per_tag; - let reservation = MemoryConsumer::new("AggregateExchangeState").register(pool); - reservation.try_resize(held)?; - // `__name__` has the metric name's column; a list without it is reused, not copied. - let tags: Arc<[String]> = if tags.iter().any(|tag| tag == "__name__") { - ordinary_tags(&tags) - .map(ToOwned::to_owned) - .collect::>() - .into() - } else { - tags - }; - // The grouping names are a scratch copy of the binding's list, which can be longer than the - // tag list, so they are admitted while the kernel is built and given back after. - let scratch = match &op.group { - BoundGrouping::By(names) | BoundGrouping::Without(names) => { - names.len() * size_of::<&str>() - } - }; - reservation.try_resize(held.saturating_add(scratch))?; - let fold = { - let names: Vec<&str> = match &op.group { - BoundGrouping::By(names) | BoundGrouping::Without(names) => { - names.iter().map(AsRef::as_ref).collect() - } - }; - let grouping = match &op.group { - BoundGrouping::By(_) => Grouping::By(&names), - BoundGrouping::Without(_) => Grouping::Without(&names), - }; - FloatAggregator::try_new_budgeted(&schema, kind, grouping, |total| { - reservation.try_resize(held.saturating_add(scratch).saturating_add(total)) - })? - }; - reservation.try_resize(held)?; - Ok(Self { - tags, - pool: Arc::clone(pool), - held, - reservation, - export_budget, - fold: Some(fold), - }) - } - - /// Folds one chunk, held while it is folded and dropped after. The kernel validates the chunk - /// against the schema this fold was built with before it charges anything. - pub(crate) fn update(&mut self, chunk: EvaluatedChunk) -> DfResult<()> { - let fold = self.fold.as_mut().ok_or_else(finished)?; - let pending = self.held + series_bytes(&chunk.series); - fold.update_budgeted(&chunk.series, |fold_bytes| { - self.reservation.try_resize(pending + fold_bytes) - })?; - drop(chunk); - self.reservation.try_resize(self.held + fold.state_bytes()) - } - - /// Encodes the fold's state once the caller has read the input to its end. The batches stay - /// charged to [`Self::export_budget`] after the states are given back, since they outlive them. - pub(crate) fn finish(&mut self) -> DfResult> { - let fold = self.fold.take().ok_or_else(finished)?; - let partial = - fold.finish_partial_budgeted(|total| self.reservation.try_resize(self.held + total))?; - let batches = self.encode(&partial)?; - drop(partial); - self.reservation.try_resize(self.held)?; - Ok(batches) - } - - /// Encodes `partial` as one row buffer, admitted in full before it allocates, and hands it over - /// as slices of at most [`BATCH_ROWS`] rows that share it. - fn encode(&self, partial: &PartialAggregate) -> DfResult> { - let mut rows = 0usize; - let mut label_bytes = 0usize; - for group in &partial.groups { - let labels = group.metric.name().map_or(0, str::len).saturating_add( - ordinary_tags(&self.tags) - .filter_map(|tag| group.metric.get(tag).map(|value| tag.len() + value.len())) - .sum::(), - ); - let present: usize = group - .points - .iter() - .map(|p| p.1.iter().flatten().count()) - .sum(); - rows += present; - label_bytes += labels * present; - } - if rows > 0 { - self.grow_export(export_bytes(rows, label_bytes, self.tags.len()))?; - } - let mut metric_name: Vec> = Vec::with_capacity(rows); - let mut tags: Vec> = self.tags.iter().map(|_| Vec::with_capacity(rows)).collect(); - let (mut point, mut field) = (Vec::with_capacity(rows), Vec::with_capacity(rows)); - let mut components: [Vec; 4] = std::array::from_fn(|_| Vec::with_capacity(rows)); - let mut incremental_mean: Vec = Vec::with_capacity(rows); - for group in &partial.groups { - // A group's labels are the same for every row it writes, so resolve each tag once - // instead of once per row. `SeriesLabels::get` binary-searches the group's tags, and - // this loop runs once per (point, field) of the group. - let resolved: Vec> = - self.tags.iter().map(|tag| group.metric.get(tag)).collect(); - for (at, states) in &group.points { - for (index, state) in states.iter().enumerate() { - let Some(state) = state else { - continue; - }; - metric_name.push(group.metric.name()); - for (column, value) in tags.iter_mut().zip(resolved.iter()) { - column.push(*value); - } - point.push(*at as u64); - field.push(index as u32); - components[0].push(state.value); - components[1].push(state.mean); - components[2].push(state.compensation); - components[3].push(state.count); - incremental_mean.push(state.incremental_mean); - } - } - } - let mut columns: Vec = Vec::with_capacity(8 + tags.len()); - columns.push(Arc::new(StringArray::from(metric_name))); - for values in tags { - columns.push(Arc::new(StringArray::from(values))); - } - columns.push(Arc::new(UInt64Array::from(point))); - columns.push(Arc::new(UInt32Array::from(field))); - for values in components { - columns.push(Arc::new(Float64Array::from(values))); - } - columns.push(Arc::new(BooleanArray::from(incremental_mean))); - let batch = RecordBatch::try_new(state_schema(&self.tags), columns)?; - Ok((0..rows) - .step_by(BATCH_ROWS) - .map(|offset| batch.slice(offset, BATCH_ROWS.min(rows - offset))) - .collect()) - } - - /// Admits `bytes` to the plan's export reservation, registered against this pool at first use. - fn grow_export(&self, bytes: usize) -> DfResult<()> { - let mut budget = self.export_budget.lock().unwrap_or_else(|e| e.into_inner()); - let consumer = MemoryConsumer::new("AggregateExchangeExport"); - budget - .get_or_insert_with(|| consumer.register(&self.pool)) - .try_grow(bytes) - } -} - -/// Reads one state batch back into the rows a `FloatMerger` merges. -/// -/// The encoder writes a group's points in ascending order, one row per (point, field), so a label -/// change opens the next group and everything else extends the current one: consecutive rows of a -/// point fill that point's state vector, and a later point is appended to the same group. Rows of -/// one group split across batches, or concatenated back to back, start a further point entry -/// instead of failing; the merge addresses points by index, so both entries fold into one slot. -pub(crate) fn decode_state( - batch: &RecordBatch, - schema: &Arc, - op: AggregateOp, - tags: &[String], - mut budget: impl FnMut(usize) -> DfResult<()>, -) -> DfResult { - // The expected layout builds a name and a field per column, so it is admitted before it is - // built, together with the column lookups below. - let mut total = (tags.len() + 9) - .saturating_mul(size_of::<&str>() + size_of::() + size_of::()); - budget(total)?; - let layout = state_schema(tags); - let batch_schema = batch.schema_ref(); - if batch_schema.as_ref() != layout.as_ref() { - return Err(DataFusionError::Execution(format!( - "not the layout {layout:?}" - ))); - } - let grid = schema.eval_grid.as_ref().ok_or_else(|| { - DataFusionError::Execution("aggregate state is indexed on an evaluated grid".into()) - })?; - let width = schema.fields.len(); - // The name list and the label column lookups are workspace, sized exactly and charged before - // either allocates. - total = total.saturating_add(tags.len() * (size_of::<&str>() + size_of::<&StringArray>())); - budget(total)?; - let mut names: Vec<&str> = Vec::with_capacity(tags.len()); - names.extend(ordinary_tags(tags)); - let metric_name: &StringArray = column(batch, "metric_name")?; - let mut tag_columns: Vec<&StringArray> = Vec::with_capacity(names.len()); - for at in 0..names.len() { - tag_columns.push(column(batch, &format!("tag_{at}"))?); - } - let point: &UInt64Array = column(batch, "point")?; - let field: &UInt32Array = column(batch, "field")?; - let value: &Float64Array = column(batch, "value")?; - let mean: &Float64Array = column(batch, "mean")?; - let compensation: &Float64Array = column(batch, "compensation")?; - let count: &Float64Array = column(batch, "count")?; - let incremental_mean: &BooleanArray = column(batch, "incremental_mean")?; - // A null in a state column is not a zero: that state was never folded. Only the label columns - // ahead of these are nullable. - for at in names.len() + 1..batch.num_columns() { - let nulls = batch.column(at).null_count(); - let name = batch_schema.field(at).name(); - if nulls > 0 { - return Err(DataFusionError::Execution(format!( - "the aggregate state column `{name}` holds {nulls} nulls, a state being a value" - ))); - } - } - let mut groups: Vec = Vec::new(); - for row in 0..batch.num_rows() { - let at = usize::try_from(point.value(row)).unwrap_or(usize::MAX); - let field = field.value(row) as usize; - if at >= grid.spec().len as usize || field >= width { - return Err(DataFusionError::Execution(format!( - "aggregate state row {row} is of point {at} of the grid's {} or of field {field} \ - of the aggregate's {width}", - grid.spec().len - ))); - } - // The encoder writes a group's points in ascending order and its fields together, so only a - // label change opens a group: a row repeating the labels of the row before it is the same - // group, and is that point's next field or the group's next point. Comparing a label change - // against a change of point would open one group per row, since consecutive points of one - // group differ by definition. - let differs = |column: &StringArray| { - column.is_null(row - 1) != column.is_null(row) - || (!column.is_null(row) && column.value(row - 1) != column.value(row)) - }; - let opens = - row == 0 || differs(metric_name) || tag_columns.iter().any(|column| differs(column)); - if opens { - // A push outgrowing the group vector doubles it, so the block it is about to allocate - // is admitted here; the point's tuple and state vector and the label scratch follow - // before either allocates, then what the labels carry once they are read. - let growth = if groups.len() == groups.capacity() { - groups.capacity().max(1) * 2 * size_of::() - } else { - 0 - }; - total = total.saturating_add( - growth - + size_of::<(usize, Vec>)>() - + width * size_of::>() - + (names.len() + 1) * size_of::<(&str, &str)>(), - ); - budget(total)?; - let name = (!metric_name.is_null(row)).then(|| metric_name.value(row)); - let mut carried: Vec<(&str, &str)> = Vec::with_capacity(names.len()); - for (column, label) in tag_columns.iter().zip(&names) { - if !column.is_null(row) { - carried.push((*label, column.value(row))); - } - } - // The label vector, the `Arc` header each name and value points at, and the bytes - // they hold. One extra pair is the metric name. - total = total.saturating_add( - (names.len() + 1) * (size_of::