feat(datatypes): add shared native series samples and fields

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
This commit is contained in:
discord9
2026-10-15 12:47:45 +08:00
parent fbf5f0c544
commit bee536ff27
3 changed files with 1463 additions and 0 deletions
+1
View File
@@ -13,6 +13,7 @@
// limitations under the License.
pub mod histogram;
pub mod samples;
pub mod time;
#[cfg(test)]
+620
View File
@@ -0,0 +1,620 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use arrow::array::{Array, Float64Array, UInt32Array, UInt32Builder};
use arrow::compute::concat;
use snafu::OptionExt;
use crate::error::{InvalidSeriesSnafu, Result};
use crate::series::MetricName;
use crate::series::histogram::{HistogramBatch, HistogramView};
use crate::series::time::{EvalGrid, TimeAxis};
#[cfg(test)]
mod tests;
fn invalid<T>(reason: impl Into<String>) -> Result<T> {
Err(InvalidSeriesSnafu {
reason: reason.into(),
}
.build())
}
fn arrow_result<T>(r: std::result::Result<T, arrow::error::ArrowError>) -> Result<T> {
r.map_err(|e| {
InvalidSeriesSnafu {
reason: e.to_string(),
}
.build()
})
}
/// Histogram references for each time position, backed by one immutable table.
#[derive(Clone, Debug)]
pub struct HistogramSamples {
indices: UInt32Array,
batch: Arc<HistogramBatch>,
}
impl HistogramSamples {
/// Validate and bind reference indices to their immutable histogram table.
pub fn try_new(indices: UInt32Array, batch: Arc<HistogramBatch>) -> Result<Self> {
if indices.iter().flatten().any(|i| i as usize >= batch.len()) {
return invalid("histogram sample index is out of bounds");
}
Ok(Self { indices, batch })
}
/// Read-only reference indices.
pub fn indices(&self) -> &UInt32Array {
&self.indices
}
/// Shared histogram table.
pub fn batch(&self) -> &Arc<HistogramBatch> {
&self.batch
}
/// Number of time positions.
pub fn len(&self) -> usize {
self.indices.len()
}
/// Whether this contains no positions.
pub fn is_empty(&self) -> bool {
self.indices.is_empty()
}
/// Returns a referenced histogram, or `None` for missing/out-of-range positions.
pub fn view_at(&self, index: usize) -> Option<HistogramView<'_>> {
if index >= self.indices.len() {
return None;
}
if self.indices.is_valid(index) {
self.batch.view(self.indices.value(index) as usize)
} else {
None
}
}
fn slice(&self, offset: usize, len: usize) -> Self {
Self {
indices: self.indices.slice(offset, len),
batch: self.batch.clone(),
}
}
}
/// Float and histogram values with mutually exclusive validity at each position.
#[derive(Clone, Debug)]
pub struct MixedSamples {
floats: Float64Array,
histograms: HistogramSamples,
}
impl MixedSamples {
/// Read-only float values, including validity bitmap.
pub fn floats(&self) -> &Float64Array {
&self.floats
}
/// Read-only histogram references and their shared table.
pub fn histograms(&self) -> &HistogramSamples {
&self.histograms
}
/// Number of positions.
pub fn len(&self) -> usize {
self.floats.len()
}
/// Whether this contains no positions.
pub fn is_empty(&self) -> bool {
self.floats.is_empty()
}
}
#[derive(Clone, Debug)]
enum SampleStorage {
Float(Float64Array),
Histogram(HistogramSamples),
Mixed(MixedSamples),
Chunks(Arc<[Samples]>, usize),
}
/// Typed Arrow-backed values for a time series.
#[derive(Clone, Debug)]
pub struct Samples {
storage: SampleStorage,
}
/// A borrowed dense representation.
pub enum DenseSamplesRef<'a> {
Float(&'a Float64Array),
Histogram(&'a HistogramSamples),
Mixed(&'a MixedSamples),
}
/// A sample value; ordinary NaN is a valid float, distinct from Missing.
pub enum SampleRef<'a> {
Missing,
Float(f64),
Histogram(HistogramView<'a>),
}
impl Samples {
/// Wrap float samples without copying; nulls are missing and NaNs remain values.
pub fn from_float(values: Float64Array) -> Self {
Self {
storage: SampleStorage::Float(values),
}
}
/// Validate histogram indices and retain their referenced table.
pub fn try_from_histogram(indices: UInt32Array, batch: Arc<HistogramBatch>) -> Result<Self> {
Ok(Self {
storage: SampleStorage::Histogram(HistogramSamples::try_new(indices, batch)?),
})
}
/// Validate lengths and ensure no position is simultaneously float and histogram.
pub fn try_from_mixed(floats: Float64Array, histograms: HistogramSamples) -> Result<Self> {
if floats.len() != histograms.len() {
return invalid("mixed sample arrays have unequal lengths");
}
if (0..floats.len()).any(|i| floats.is_valid(i) && histograms.indices.is_valid(i)) {
return invalid("mixed sample position has both float and histogram values");
}
Ok(Self {
storage: SampleStorage::Mixed(MixedSamples { floats, histograms }),
})
}
/// Retain immutable chunks, flattening nested descriptors and checking total length.
pub fn try_from_chunks(chunks: Vec<Samples>) -> Result<Self> {
let mut flat = Vec::new();
for chunk in chunks {
match chunk.storage {
SampleStorage::Chunks(parts, _) => flat.extend(parts.iter().cloned()),
_ => flat.push(chunk),
}
}
let total = flat
.iter()
.try_fold(0usize, |n, s| n.checked_add(s.len()))
.context(InvalidSeriesSnafu {
reason: "sample chunk length overflow",
})?;
flat.retain(|s| !s.is_empty());
if flat.is_empty() {
return Ok(Self::from_float(Float64Array::new_null(0)));
}
if flat.len() == 1 {
return flat.pop().ok_or_else(|| {
InvalidSeriesSnafu {
reason: "single sample chunk disappeared".to_string(),
}
.build()
});
}
Ok(Self {
storage: SampleStorage::Chunks(flat.into(), total),
})
}
/// Number of sample positions.
pub fn len(&self) -> usize {
match &self.storage {
SampleStorage::Float(a) => a.len(),
SampleStorage::Histogram(a) => a.len(),
SampleStorage::Mixed(a) => a.len(),
SampleStorage::Chunks(_, len) => *len,
}
}
/// Whether this has no positions.
pub fn is_empty(&self) -> bool {
self.len() == 0
}
/// Returns a dense typed representation, or `None` for chunks.
pub fn dense(&self) -> Option<DenseSamplesRef<'_>> {
match &self.storage {
SampleStorage::Float(a) => Some(DenseSamplesRef::Float(a)),
SampleStorage::Histogram(a) => Some(DenseSamplesRef::Histogram(a)),
SampleStorage::Mixed(a) => Some(DenseSamplesRef::Mixed(a)),
SampleStorage::Chunks(_, _) => None,
}
}
/// Returns the sample, treating an out-of-range index as missing.
pub fn sample(&self, index: usize) -> SampleRef<'_> {
if index >= self.len() {
return SampleRef::Missing;
}
match &self.storage {
SampleStorage::Float(a) => {
if a.is_valid(index) {
SampleRef::Float(a.value(index))
} else {
SampleRef::Missing
}
}
SampleStorage::Histogram(a) => a
.view_at(index)
.map_or(SampleRef::Missing, SampleRef::Histogram),
SampleStorage::Mixed(a) => {
if a.floats.is_valid(index) {
SampleRef::Float(a.floats.value(index))
} else {
a.histograms
.view_at(index)
.map_or(SampleRef::Missing, SampleRef::Histogram)
}
}
SampleStorage::Chunks(chunks, _) => {
let mut i = index;
for part in chunks.iter() {
if i < part.len() {
return part.sample(i);
}
i -= part.len();
}
SampleRef::Missing
}
}
}
/// Iterate all positions sequentially without materializing per-sample objects.
pub fn iter(&self) -> impl ExactSizeIterator<Item = SampleRef<'_>> + '_ {
let (chunks, remaining) = match &self.storage {
SampleStorage::Chunks(chunks, len) => (Some(chunks.as_ref()), *len),
_ => (None, self.len()),
};
SamplesIter {
samples: self,
chunks,
chunk_index: 0,
local_index: 0,
remaining,
}
}
/// Return a checked zero-copy slice (across chunks only intersected descriptors are retained).
pub fn slice(&self, offset: usize, len: usize) -> Result<Self> {
let end = offset.checked_add(len).context(InvalidSeriesSnafu {
reason: "sample slice bounds overflow",
})?;
if end > self.len() {
return invalid("sample slice is out of bounds");
}
match &self.storage {
SampleStorage::Float(a) => Ok(Self::from_float(a.slice(offset, len))),
SampleStorage::Histogram(a) => Ok(Self {
storage: SampleStorage::Histogram(a.slice(offset, len)),
}),
SampleStorage::Mixed(a) => Ok(Self {
storage: SampleStorage::Mixed(MixedSamples {
floats: a.floats.slice(offset, len),
histograms: a.histograms.slice(offset, len),
}),
}),
SampleStorage::Chunks(chunks, _) => {
let mut skip = offset;
let mut remain = len;
let mut out = Vec::new();
for chunk in chunks.iter() {
if skip >= chunk.len() {
skip -= chunk.len();
continue;
}
let take = remain.min(chunk.len() - skip);
if take != 0 {
out.push(chunk.slice(skip, take)?);
remain -= take;
}
skip = 0;
if remain == 0 {
break;
}
}
Self::try_from_chunks(out)
}
}
}
/// Materialize chunks into one dense array; histogram tables are concatenated per chunk,
/// including repeated shared tables (no registry/deduplication is performed).
pub fn into_dense(self) -> Result<Self> {
let SampleStorage::Chunks(chunks, _) = self.storage else {
return Ok(self);
};
let all_float = chunks
.iter()
.all(|s| matches!(s.storage, SampleStorage::Float(_)));
if all_float {
let arrays: Vec<&dyn Array> = chunks
.iter()
.filter_map(|s| match &s.storage {
SampleStorage::Float(a) => Some(a as &dyn Array),
_ => None,
})
.collect();
let array = arrow_result(concat(&arrays))?;
let values = array
.as_any()
.downcast_ref::<Float64Array>()
.ok_or_else(|| {
InvalidSeriesSnafu {
reason: "float concatenation returned a non-float array",
}
.build()
})?;
return Ok(Self::from_float(values.clone()));
}
let mut tables: Vec<&dyn Array> = Vec::new();
let mut indices = UInt32Builder::new();
let mut floats = arrow::array::Float64Builder::new();
let mut base = 0u32;
for chunk in chunks.iter() {
let (float_values, histogram) = match &chunk.storage {
SampleStorage::Float(values) => (Some(values), None),
SampleStorage::Histogram(histogram) => (None, Some(histogram)),
SampleStorage::Mixed(mixed) => (Some(&mixed.floats), Some(&mixed.histograms)),
SampleStorage::Chunks(_, _) => {
return invalid("nested sample chunks were not flattened");
}
};
if let Some(histogram) = histogram {
tables.push(histogram.batch.as_arrow());
}
for position in 0..chunk.len() {
match float_values {
Some(values) if values.is_valid(position) => {
floats.append_value(values.value(position))
}
_ => floats.append_null(),
}
match histogram {
Some(histogram) if histogram.indices.is_valid(position) => {
indices.append_value(
base.checked_add(histogram.indices.value(position))
.context(InvalidSeriesSnafu {
reason: "histogram remapped index overflow",
})?,
);
}
_ => indices.append_null(),
}
}
if let Some(histogram) = histogram {
base = base
.checked_add(u32::try_from(histogram.batch.len()).map_err(|_| {
InvalidSeriesSnafu {
reason: "histogram table too large",
}
.build()
})?)
.context(InvalidSeriesSnafu {
reason: "histogram table index overflow",
})?;
}
}
let table = arrow_result(concat(&tables))?;
let struct_array = table
.as_any()
.downcast_ref::<arrow::array::StructArray>()
.ok_or_else(|| {
InvalidSeriesSnafu {
reason: "histogram concatenation returned a non-struct array",
}
.build()
})?
.clone();
let batch = Arc::new(HistogramBatch::try_from_arrow(struct_array)?);
let histogram = HistogramSamples::try_new(indices.finish(), batch)?;
let floats = floats.finish();
if chunks.iter().any(|chunk| {
matches!(
chunk.storage,
SampleStorage::Float(_) | SampleStorage::Mixed(_)
)
}) {
Self::try_from_mixed(floats, histogram)
} else {
Ok(Self {
storage: SampleStorage::Histogram(histogram),
})
}
}
}
/// Sequential exact-size iterator over a dense series or its chunk descriptors.
struct SamplesIter<'a> {
samples: &'a Samples,
chunks: Option<&'a [Samples]>,
chunk_index: usize,
local_index: usize,
remaining: usize,
}
impl<'a> Iterator for SamplesIter<'a> {
type Item = SampleRef<'a>;
fn next(&mut self) -> Option<Self::Item> {
if self.remaining == 0 {
return None;
}
self.remaining -= 1;
if let Some(chunks) = self.chunks {
while self.chunk_index < chunks.len()
&& self.local_index == chunks[self.chunk_index].len()
{
self.chunk_index += 1;
self.local_index = 0;
}
let chunk = &chunks[self.chunk_index];
let value = chunk.sample(self.local_index);
self.local_index += 1;
Some(value)
} else {
let index = self.samples.len() - self.remaining - 1;
Some(self.samples.sample(index))
}
}
fn size_hint(&self) -> (usize, Option<usize>) {
(self.remaining, Some(self.remaining))
}
}
impl ExactSizeIterator for SamplesIter<'_> {
fn len(&self) -> usize {
self.remaining
}
}
#[derive(Clone, Debug)]
enum Fields<T> {
One(T),
Many(Box<[T]>),
}
/// Field values using inline storage for a single field.
#[derive(Clone, Debug)]
pub struct SeriesFields {
storage: Fields<Samples>,
}
impl SeriesFields {
/// Create a one-field collection without allocating a field list.
pub fn one(field: Samples) -> Self {
Self {
storage: Fields::One(field),
}
}
/// Validate nonempty, equally sized fields.
pub fn try_from_vec(fields: Vec<Samples>) -> Result<Self> {
if fields.is_empty() {
return invalid("series fields must not be empty");
}
if fields.iter().any(|f| f.len() != fields[0].len()) {
return invalid("series fields have unequal lengths");
}
Ok(if fields.len() == 1 {
let mut fields = fields.into_iter();
let field = fields.next().ok_or_else(|| {
InvalidSeriesSnafu {
reason: "single series field disappeared".to_string(),
}
.build()
})?;
Self::one(field)
} else {
Self {
storage: Fields::Many(fields.into_boxed_slice()),
}
})
}
/// Borrow all fields.
pub fn as_slice(&self) -> &[Samples] {
match &self.storage {
Fields::One(f) => std::slice::from_ref(f),
Fields::Many(f) => f,
}
}
/// Number of fields.
pub fn len(&self) -> usize {
self.as_slice().len()
}
/// Whether this has no fields.
pub fn is_empty(&self) -> bool {
false
}
/// Get a field by index.
pub fn get(&self, index: usize) -> Option<&Samples> {
self.as_slice().get(index)
}
/// Iterate fields.
pub fn iter(&self) -> std::slice::Iter<'_, Samples> {
self.as_slice().iter()
}
}
/// Shared name and position of each output field.
#[derive(Clone, Debug)]
pub struct FieldSpec {
pub name: Arc<str>,
}
/// Shared schema describing series fields.
#[derive(Clone, Debug)]
pub struct SeriesSchema {
pub fields: Arc<[FieldSpec]>,
/// Shared grid for regular series; its `GridSpec` must match each series grid axis. Explicit axes require `None`.
pub eval_grid: Option<Arc<EvalGrid>>,
}
/// A validated metric, time axis, and aligned fields.
#[derive(Clone, Debug)]
pub struct Series {
pub metric: MetricName,
pub time: TimeAxis,
pub fields: SeriesFields,
}
impl Series {
/// Construct after validating schema, field lengths, and strictly increasing timestamps.
pub fn try_new(
schema: &SeriesSchema,
metric: MetricName,
time: TimeAxis,
fields: SeriesFields,
) -> Result<Self> {
if fields.len() != schema.fields.len() {
return invalid("series field count does not match schema");
}
let time_len = checked_time_len(&time)?;
if fields.iter().any(|f| f.len() != time_len) {
return invalid("series field length does not match time axis");
}
match (&schema.eval_grid, &time) {
(Some(expected), TimeAxis::Grid(range)) if expected.spec() == range.grid().spec() => {}
(Some(_), TimeAxis::Grid(_)) => {
return invalid("series grid does not match schema evaluation grid");
}
(Some(_), _) => {
return invalid("explicit series time axis cannot use an evaluation grid");
}
(None, TimeAxis::Grid(_)) => {
return invalid("grid series requires a schema evaluation grid");
}
(None, TimeAxis::Explicit(_) | TimeAxis::ExplicitChunks(_)) => {}
}
Ok(Self {
metric,
time,
fields,
})
}
}
fn checked_time_len(time: &TimeAxis) -> Result<usize> {
match time {
TimeAxis::Grid(range) => Ok(range.len()),
TimeAxis::Explicit(timestamps) => {
if timestamps.windows(2).any(|pair| pair[0] >= pair[1]) {
return invalid("timestamps must be strictly increasing");
}
Ok(timestamps.len())
}
TimeAxis::ExplicitChunks(chunks) => {
let mut total = 0usize;
let mut previous = None;
for chunk in chunks.iter() {
total = total.checked_add(chunk.len()).ok_or_else(|| {
InvalidSeriesSnafu {
reason: "timestamp length overflow".to_string(),
}
.build()
})?;
if chunk.windows(2).any(|pair| pair[0] >= pair[1])
|| chunk
.first()
.is_some_and(|first| previous.is_some_and(|last| last >= *first))
{
return invalid("timestamps must be strictly increasing across chunks");
}
if let Some(last) = chunk.last() {
previous = Some(*last);
}
}
Ok(total)
}
}
}
+842
View File
@@ -0,0 +1,842 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use super::*;
use std::sync::atomic::{AtomicUsize, Ordering};
use arrow::array::{Float64Array, UInt32Array};
use arrow::buffer::{Buffer, ScalarBuffer};
use crate::series::histogram::{BucketSideView, CounterResetHint, HistogramBatchBuilder};
use crate::series::time::{EvalGrid, GridRange, GridSpec, TimeAxis};
fn batch(sum: f64, count: f64, buckets: &[f64]) -> Arc<HistogramBatch> {
let offsets = if buckets.is_empty() { vec![] } else { vec![0] };
let lengths = if buckets.is_empty() {
vec![]
} else {
vec![buckets.len() as i32]
};
let side = BucketSideView {
span_offsets: &offsets,
span_lengths: &lengths,
buckets,
};
let row = HistogramView {
schema: 1,
zero_threshold: 0.0,
sum,
count,
zero_count: 0.0,
reset_hint: CounterResetHint::Unknown,
start_timestamp: None,
custom_values: &[],
positive: side,
negative: BucketSideView {
span_offsets: &[],
span_lengths: &[],
buckets: &[],
},
};
histogram_batch(&[row])
}
fn histogram_batch(rows: &[HistogramView<'_>]) -> Arc<HistogramBatch> {
let mut builder = HistogramBatchBuilder::new();
for row in rows {
builder.push(*row).unwrap();
}
builder.finish().unwrap()
}
struct FloatBufferOwner {
values: Vec<f64>,
drops: Arc<AtomicUsize>,
}
impl Drop for FloatBufferOwner {
fn drop(&mut self) {
self.drops.fetch_add(1, Ordering::SeqCst);
}
}
fn owned_floats(values: Vec<f64>, drops: Arc<AtomicUsize>) -> ScalarBuffer<f64> {
let mut owner = FloatBufferOwner { values, drops };
let pointer = std::ptr::NonNull::new(owner.values.as_mut_ptr().cast()).unwrap();
let len = owner.values.len() * std::mem::size_of::<f64>();
let element_count = owner.values.len();
// SAFETY: the vector allocation remains stable in the Arc owner for the buffer lifetime.
let buffer = unsafe { Buffer::from_custom_allocation(pointer, len, Arc::new(owner)) };
ScalarBuffer::new(buffer, 0, element_count)
}
fn same_histogram(a: HistogramView<'_>, b: HistogramView<'_>) -> bool {
a.schema == b.schema
&& a.zero_threshold.to_bits() == b.zero_threshold.to_bits()
&& a.sum.to_bits() == b.sum.to_bits()
&& a.count.to_bits() == b.count.to_bits()
&& a.zero_count.to_bits() == b.zero_count.to_bits()
&& a.reset_hint == b.reset_hint
&& a.start_timestamp == b.start_timestamp
&& a.custom_values
.iter()
.map(|value| value.to_bits())
.eq(b.custom_values.iter().map(|value| value.to_bits()))
&& a.positive.span_offsets == b.positive.span_offsets
&& a.positive.span_lengths == b.positive.span_lengths
&& a.positive.buckets.iter().map(|value| value.to_bits()).eq(b
.positive
.buckets
.iter()
.map(|value| value.to_bits()))
&& a.negative.span_offsets == b.negative.span_offsets
&& a.negative.span_lengths == b.negative.span_lengths
&& a.negative.buckets.iter().map(|value| value.to_bits()).eq(b
.negative
.buckets
.iter()
.map(|value| value.to_bits()))
}
fn same_sample(a: SampleRef<'_>, b: SampleRef<'_>) -> bool {
match (a, b) {
(SampleRef::Missing, SampleRef::Missing) => true,
(SampleRef::Float(a), SampleRef::Float(b)) => a.to_bits() == b.to_bits(),
(SampleRef::Histogram(a), SampleRef::Histogram(b)) => same_histogram(a, b),
_ => false,
}
}
#[test]
fn float_states_and_slice_share_values_and_bitmap_after_owner_drop() {
let stale = f64::from_bits(0x7ff0_0000_0000_0002);
let owner = Float64Array::from(vec![Some(1.0), None, Some(f64::NAN), Some(stale)]);
let samples = Samples::from_float(owner.clone());
assert!(matches!(samples.sample(1), SampleRef::Missing));
assert!(matches!(samples.sample(2), SampleRef::Float(v) if v.is_nan()));
assert!(matches!(samples.sample(3), SampleRef::Float(v) if v.to_bits() == stale.to_bits()));
let sliced = samples.slice(1, 3).unwrap();
let DenseSamplesRef::Float(slice) = sliced.dense().unwrap() else {
panic!()
};
assert_eq!(
slice.values().as_ptr(),
owner.values().as_ptr().wrapping_add(1)
);
assert_eq!(
slice.nulls().unwrap().buffer().as_ptr(),
owner.nulls().unwrap().buffer().as_ptr()
);
assert!(slice.is_null(0));
drop(samples);
drop(owner);
assert!(matches!(sliced.sample(0), SampleRef::Missing));
assert!(matches!(sliced.sample(1), SampleRef::Float(v) if v.is_nan()));
assert!(matches!(sliced.sample(2), SampleRef::Float(v) if v.to_bits() == stale.to_bits()));
assert!(sliced.slice(usize::MAX, 2).is_err());
}
#[test]
fn histogram_bounds_nulls_and_public_view_bounds_are_safe() {
let owner = batch(2.0, 3.0, &[4.0, 5.0]);
let refs =
HistogramSamples::try_new(UInt32Array::from(vec![None, Some(0)]), owner.clone()).unwrap();
assert!(refs.view_at(usize::MAX).is_none());
assert!(refs.view_at(2).is_none());
assert!(refs.view_at(0).is_none());
assert_eq!(refs.view_at(1).unwrap().sum.to_bits(), 2.0f64.to_bits());
assert!(HistogramSamples::try_new(UInt32Array::from(vec![Some(1)]), owner).is_err());
}
#[test]
fn mixed_validates_length_exclusivity_and_missing_positions() {
let table = batch(2.0, 1.0, &[6.0]);
let refs =
HistogramSamples::try_new(UInt32Array::from(vec![None, Some(0), None]), table).unwrap();
assert!(
Samples::try_from_mixed(Float64Array::from(vec![None, Some(1.0)]), refs.clone()).is_err()
);
assert!(
Samples::try_from_mixed(
Float64Array::from(vec![None, Some(1.0), None]),
refs.clone()
)
.is_err()
);
let mixed = Samples::try_from_mixed(Float64Array::from(vec![None, None, None]), refs).unwrap();
assert!(matches!(mixed.sample(0), SampleRef::Missing));
assert!(matches!(mixed.sample(1), SampleRef::Histogram(_)));
assert!(matches!(mixed.sample(2), SampleRef::Missing));
}
#[test]
fn chunk_iterator_is_sequential_exact_and_handles_many_singletons() {
let inner = Samples::try_from_chunks(vec![
Samples::from_float(Float64Array::new_null(0)),
Samples::from_float(Float64Array::from(vec![2.0])),
Samples::from_float(Float64Array::from(vec![3.0])),
])
.unwrap();
let normalized = Samples::try_from_chunks(vec![
Samples::from_float(Float64Array::from(vec![0.0, 1.0])),
inner,
Samples::from_float(Float64Array::new_null(0)),
Samples::from_float(Float64Array::from(vec![4.0])),
])
.unwrap();
let SampleStorage::Chunks(parts, _) = &normalized.storage else {
panic!()
};
assert_eq!(parts.len(), 4);
assert_eq!(normalized.iter().len(), 5);
for (index, part) in parts.iter().enumerate() {
assert!(matches!(part.dense(), Some(DenseSamplesRef::Float(_))));
assert!(matches!(part.sample(0), SampleRef::Float(v) if v == [0.0, 2.0, 3.0, 4.0][index]));
}
for (index, value) in normalized.iter().enumerate() {
assert!(matches!(value, SampleRef::Float(v) if v == index as f64));
}
let collapsed = Samples::try_from_chunks(vec![
Samples::from_float(Float64Array::new_null(0)),
normalized.slice(2, 1).unwrap(),
Samples::from_float(Float64Array::new_null(0)),
])
.unwrap();
assert!(matches!(collapsed.storage, SampleStorage::Float(_)));
assert!(matches!(collapsed.sample(0), SampleRef::Float(v) if v == 2.0));
let empty = Samples::try_from_chunks(vec![]).unwrap();
assert!(empty.is_empty());
assert!(matches!(empty.dense(), Some(DenseSamplesRef::Float(values)) if values.is_empty()));
let chunks = Samples::try_from_chunks(
(0..256)
.map(|i| Samples::from_float(Float64Array::from(vec![i as f64])))
.collect(),
)
.unwrap();
let mut iter = chunks.iter();
assert_eq!(iter.len(), 256);
for i in 0..256 {
assert!(matches!(iter.next(), Some(SampleRef::Float(v)) if v == i as f64));
assert_eq!(iter.len(), 255 - i);
}
assert!(iter.next().is_none());
}
#[test]
fn densification_preserves_values_nulls_nan_and_remaps_multitable_histograms() {
let stale = f64::from_bits(0x7ff0_0000_0000_0002);
let float = Samples::from_float(Float64Array::from(vec![Some(1.0), None, Some(stale)]));
let first_buckets = [13.0, 14.0];
let first_positive_offsets = [0];
let first_positive_lengths = [2];
let first_negative_offsets = [0];
let first_negative_lengths = [1];
let first_negative_buckets = [15.0];
let first_custom = [16.0, 17.0];
let first_rows = [
HistogramView {
schema: 2,
zero_threshold: 0.25,
sum: f64::from_bits(0x7ff0_0000_0000_0002),
count: 12.5,
zero_count: 1.5,
reset_hint: CounterResetHint::Unrecognized(9),
start_timestamp: Some(123),
custom_values: &first_custom,
positive: BucketSideView {
span_offsets: &first_positive_offsets,
span_lengths: &first_positive_lengths,
buckets: &first_buckets,
},
negative: BucketSideView {
span_offsets: &first_negative_offsets,
span_lengths: &first_negative_lengths,
buckets: &first_negative_buckets,
},
},
HistogramView {
schema: 1,
zero_threshold: 0.0,
sum: -21.0,
count: 22.0,
zero_count: 0.0,
reset_hint: CounterResetHint::Unknown,
start_timestamp: None,
custom_values: &[],
positive: BucketSideView {
span_offsets: &[0],
span_lengths: &[1],
buckets: &[23.0],
},
negative: BucketSideView {
span_offsets: &[],
span_lengths: &[],
buckets: &[],
},
},
];
let first_table = histogram_batch(&first_rows);
let first =
Samples::try_from_histogram(UInt32Array::from(vec![None, Some(1)]), first_table.clone())
.unwrap();
let second_positive_offsets = [0];
let second_positive_lengths = [1];
let second_positive_buckets = [31.0];
let second_rows = [
HistogramView {
schema: 3,
zero_threshold: 0.5,
sum: 32.0,
count: f64::from_bits(0x7ff0_0000_0000_0002),
zero_count: 2.25,
reset_hint: CounterResetHint::CounterReset,
start_timestamp: Some(-456),
custom_values: &[33.0],
positive: BucketSideView {
span_offsets: &second_positive_offsets,
span_lengths: &second_positive_lengths,
buckets: &second_positive_buckets,
},
negative: BucketSideView {
span_offsets: &[0],
span_lengths: &[1],
buckets: &[34.0],
},
},
HistogramView {
schema: 4,
zero_threshold: 0.75,
sum: 35.0,
count: 36.0,
zero_count: 3.5,
reset_hint: CounterResetHint::Gauge,
start_timestamp: None,
custom_values: &[37.0, 38.0],
positive: BucketSideView {
span_offsets: &[0],
span_lengths: &[2],
buckets: &[39.0, 40.0],
},
negative: BucketSideView {
span_offsets: &[],
span_lengths: &[],
buckets: &[],
},
},
HistogramView {
schema: 5,
zero_threshold: 1.0,
sum: 41.0,
count: 42.0,
zero_count: 4.5,
reset_hint: CounterResetHint::NotCounterReset,
start_timestamp: Some(789),
custom_values: &[],
positive: BucketSideView {
span_offsets: &[],
span_lengths: &[],
buckets: &[],
},
negative: BucketSideView {
span_offsets: &[0],
span_lengths: &[2],
buckets: &[43.0, 44.0],
},
},
];
let second_table = histogram_batch(&second_rows);
let second_refs =
HistogramSamples::try_new(UInt32Array::from(vec![Some(2), None]), second_table.clone())
.unwrap();
let mixed =
Samples::try_from_mixed(Float64Array::from(vec![None, Some(f64::NAN)]), second_refs)
.unwrap();
let chunked = Samples::try_from_chunks(vec![float, first.clone(), mixed]).unwrap();
let expected: Vec<_> = chunked.iter().collect();
let dense = chunked.clone().into_dense().unwrap();
assert_eq!(dense.len(), expected.len());
for (i, before) in expected.into_iter().enumerate() {
assert!(same_sample(before, dense.sample(i)), "position {i}");
}
let float_only = Samples::try_from_chunks(vec![
Samples::from_float(Float64Array::from(vec![Some(1.0), None])),
Samples::from_float(Float64Array::from(vec![Some(f64::NAN), Some(stale)])),
])
.unwrap();
let float_expected: Vec<_> = float_only.iter().collect();
let float_dense = float_only.clone().into_dense().unwrap();
for (index, expected) in float_expected.into_iter().enumerate() {
assert!(same_sample(expected, float_dense.sample(index)));
}
let DenseSamplesRef::Mixed(dense_mixed) = dense.dense().unwrap() else {
panic!()
};
assert_eq!(dense_mixed.histograms.indices().value(4), 1);
assert_eq!(dense_mixed.histograms.indices().value(5), 4);
assert_eq!(dense_mixed.histograms.batch().len(), 5);
for index in 0..5 {
let actual = dense_mixed.histograms.batch().view(index).unwrap();
let expected = if index < 2 {
first_table.view(index).unwrap()
} else {
second_table.view(index - 2).unwrap()
};
assert!(same_sample(
SampleRef::Histogram(actual),
SampleRef::Histogram(expected)
));
}
let pure_hist = Samples::try_from_chunks(vec![
first.clone(),
Samples::try_from_histogram(UInt32Array::from(vec![Some(0)]), second_table).unwrap(),
])
.unwrap();
let dense_hist = pure_hist.clone().into_dense().unwrap();
assert!(matches!(
dense_hist.dense(),
Some(DenseSamplesRef::Histogram(_))
));
for index in 0..pure_hist.len() {
assert!(same_sample(
pure_hist.sample(index),
dense_hist.sample(index)
));
}
let SampleStorage::Histogram(first_refs) = &first.storage else {
panic!()
};
let sliced = first_refs.slice(1, 1);
assert!(Arc::ptr_eq(sliced.batch(), &first_table));
assert_eq!(
sliced.indices().values().as_ptr(),
first_refs.indices().values().as_ptr().wrapping_add(1)
);
}
#[test]
fn chunk_slices_keep_indices_tables_and_numeric_buffers_alive() {
let f = Float64Array::from(vec![Some(1.0), None, Some(3.0)]);
let float_samples = Samples::from_float(f.clone());
let hist_table = batch(7.0, 8.0, &[9.0]);
let hist_samples = Samples::try_from_histogram(
UInt32Array::from(vec![None, Some(0), None]),
hist_table.clone(),
)
.unwrap();
let mixed_table = batch(17.0, 18.0, &[19.0]);
let mixed_values = Float64Array::from(vec![Some(20.0), None, None]);
let mixed = Samples::try_from_mixed(
mixed_values.clone(),
HistogramSamples::try_new(
UInt32Array::from(vec![None, None, Some(0)]),
mixed_table.clone(),
)
.unwrap(),
)
.unwrap();
let chunks = Samples::try_from_chunks(vec![
float_samples.clone(),
hist_samples.clone(),
mixed.clone(),
])
.unwrap();
let sliced = chunks.slice(1, 8).unwrap();
for index in 0..8 {
assert!(same_sample(chunks.sample(index + 1), sliced.sample(index)));
}
let SampleStorage::Chunks(parts, _) = &sliced.storage else {
panic!()
};
let SampleStorage::Float(fpart) = &parts[0].storage else {
panic!()
};
assert_eq!(fpart.values().as_ptr(), f.values().as_ptr().wrapping_add(1));
assert_eq!(
fpart.nulls().unwrap().buffer().as_ptr(),
f.nulls().unwrap().buffer().as_ptr()
);
let SampleStorage::Histogram(hpart) = &parts[1].storage else {
panic!()
};
assert!(Arc::ptr_eq(hpart.batch(), &hist_table));
let SampleStorage::Mixed(mpart) = &parts[2].storage else {
panic!()
};
assert!(Arc::ptr_eq(mpart.histograms.batch(), &mixed_table));
assert_eq!(
mpart.floats.values().as_ptr(),
mixed_values.values().as_ptr()
);
drop(chunks);
drop(float_samples);
drop(hist_samples);
drop(mixed);
drop(f);
drop(mixed_values);
drop(hist_table);
drop(mixed_table);
assert!(matches!(sliced.sample(1), SampleRef::Float(v) if v == 3.0));
assert!(matches!(sliced.sample(7), SampleRef::Histogram(_)));
}
#[test]
fn fields_clone_buffers_and_series_validate_schema_alignment_and_time_order() {
let sample = Samples::from_float(Float64Array::from(vec![1.0, 2.0]));
let fields = SeriesFields::try_from_vec(vec![sample.clone(), sample.clone()]).unwrap();
assert!(matches!(fields.storage, Fields::Many(_)));
let cloned = fields.clone();
let DenseSamplesRef::Float(a) = fields.get(0).unwrap().dense().unwrap() else {
panic!()
};
let DenseSamplesRef::Float(b) = cloned.get(0).unwrap().dense().unwrap() else {
panic!()
};
assert_eq!(a.values().as_ptr(), b.values().as_ptr());
assert!(SeriesFields::try_from_vec(vec![]).is_err());
assert!(
SeriesFields::try_from_vec(vec![
sample.clone(),
Samples::from_float(Float64Array::from(vec![1.0]))
])
.is_err()
);
let schema = SeriesSchema {
fields: Arc::from([
FieldSpec {
name: Arc::from("a"),
},
FieldSpec {
name: Arc::from("b"),
},
]),
eval_grid: None,
};
let metric = MetricName::try_new(None, vec![]).unwrap();
let time = TimeAxis::Explicit(ScalarBuffer::from(vec![1, 2]));
assert!(
Series::try_new(
&schema,
metric.clone(),
time.clone(),
SeriesFields::one(sample.clone())
)
.is_err()
);
assert!(
Series::try_new(
&schema,
metric.clone(),
time.clone(),
SeriesFields::try_from_vec(vec![
Samples::from_float(Float64Array::from(vec![1.0])),
Samples::from_float(Float64Array::from(vec![2.0]))
])
.unwrap()
)
.is_err()
);
let unordered = TimeAxis::Explicit(ScalarBuffer::from(vec![2, 1]));
assert!(Series::try_new(&schema, metric.clone(), unordered, fields.clone()).is_err());
assert!(Series::try_new(&schema, metric, time, fields).is_ok());
}
#[test]
fn float_fields_clones_and_series_keep_real_arrow_buffer_owner_alive() {
let drops = Arc::new(AtomicUsize::new(0));
let values = Float64Array::new(owned_floats(vec![7.0, 8.0], drops.clone()), None);
let samples = Samples::from_float(values);
let fields = SeriesFields::one(samples);
let cloned = fields.clone();
let schema = SeriesSchema {
fields: Arc::from([FieldSpec {
name: Arc::from("value"),
}]),
eval_grid: None,
};
let series = Series::try_new(
&schema,
MetricName::try_new(None, vec![]).unwrap(),
TimeAxis::Explicit(ScalarBuffer::from(vec![1_i64, 2])),
fields,
)
.unwrap();
drop(cloned);
assert_eq!(drops.load(Ordering::SeqCst), 0);
drop(series);
assert_eq!(drops.load(Ordering::SeqCst), 1);
}
#[test]
fn series_enforces_grid_policy_and_matches_grid_spec_by_value() {
let spec = GridSpec::try_new(100, 10, 3).unwrap();
let schema_grid = Arc::new(EvalGrid::try_new(spec).unwrap());
let equal_grid = Arc::new(EvalGrid::try_new(spec).unwrap());
let range = GridRange::try_new(equal_grid, 1, 1).unwrap();
let fields = SeriesFields::one(Samples::from_float(Float64Array::from(vec![1.0])));
let metric = MetricName::try_new(None, vec![]).unwrap();
let grid_schema = SeriesSchema {
fields: Arc::from([FieldSpec {
name: Arc::from("value"),
}]),
eval_grid: Some(schema_grid.clone()),
};
assert!(
Series::try_new(
&grid_schema,
metric.clone(),
TimeAxis::Grid(range),
fields.clone()
)
.is_ok()
);
// Timestamp 110 overlaps, but a different grid specification is not interchangeable.
let different_grid =
Arc::new(EvalGrid::try_new(GridSpec::try_new(110, 5, 3).unwrap()).unwrap());
assert!(
Series::try_new(
&grid_schema,
metric.clone(),
TimeAxis::Grid(GridRange::try_new(different_grid, 0, 1).unwrap()),
fields.clone(),
)
.is_err()
);
let explicit_schema = SeriesSchema {
fields: grid_schema.fields.clone(),
eval_grid: None,
};
assert!(
Series::try_new(
&explicit_schema,
metric.clone(),
TimeAxis::Explicit(ScalarBuffer::from(vec![110])),
fields.clone(),
)
.is_ok()
);
assert!(
Series::try_new(
&explicit_schema,
metric.clone(),
TimeAxis::Grid(GridRange::try_new(schema_grid.clone(), 0, 1).unwrap()),
fields.clone(),
)
.is_err()
);
assert!(
Series::try_new(
&grid_schema,
metric.clone(),
TimeAxis::Explicit(ScalarBuffer::from(vec![110])),
fields.clone(),
)
.is_err()
);
for (different_spec, range_start) in [
(GridSpec::try_new(90, 10, 3).unwrap(), 1),
(GridSpec::try_new(100, 5, 3).unwrap(), 2),
(GridSpec::try_new(100, 10, 4).unwrap(), 1),
] {
let other_grid = Arc::new(EvalGrid::try_new(different_spec).unwrap());
assert!(
Series::try_new(
&grid_schema,
metric.clone(),
TimeAxis::Grid(GridRange::try_new(other_grid, range_start, 1).unwrap()),
fields.clone(),
)
.is_err()
);
}
let empty_fields = SeriesFields::one(Samples::from_float(Float64Array::new_null(0)));
assert!(
Series::try_new(
&grid_schema,
metric.clone(),
TimeAxis::Grid(GridRange::try_new(schema_grid, 3, 0).unwrap()),
empty_fields.clone(),
)
.is_ok()
);
assert!(
Series::try_new(
&explicit_schema,
metric.clone(),
TimeAxis::Explicit(ScalarBuffer::from(Vec::<i64>::new())),
empty_fields.clone(),
)
.is_ok()
);
assert!(
Series::try_new(
&grid_schema,
metric.clone(),
TimeAxis::Explicit(ScalarBuffer::from(Vec::<i64>::new())),
empty_fields.clone(),
)
.is_err()
);
assert!(
Series::try_new(
&explicit_schema,
metric,
TimeAxis::Grid(
GridRange::try_new(grid_schema.eval_grid.as_ref().unwrap().clone(), 3, 0).unwrap()
),
empty_fields,
)
.is_err()
);
}
#[test]
fn short_multifield_series_uses_its_grid_range_and_checks_explicit_chunk_order() {
let grid = Arc::new(EvalGrid::try_new(GridSpec::try_new(0, 1, 2017).unwrap()).unwrap());
let float = Samples::from_float(Float64Array::from(
(0..20)
.map(|i| {
if i == 4 {
None
} else if i == 7 {
Some(f64::NAN)
} else {
Some(i as f64)
}
})
.collect::<Vec<_>>(),
));
let histogram_table = batch(2.0, 3.0, &[4.0]);
let histogram = Samples::try_from_histogram(
UInt32Array::from(vec![Some(0); 20]),
histogram_table.clone(),
)
.unwrap();
let fields = SeriesFields::try_from_vec(vec![float, histogram]).unwrap();
let schema = SeriesSchema {
fields: Arc::from([
FieldSpec {
name: Arc::from("float"),
},
FieldSpec {
name: Arc::from("histogram"),
},
]),
eval_grid: Some(grid.clone()),
};
let series = Series::try_new(
&schema,
MetricName::try_new(None, vec![]).unwrap(),
TimeAxis::Grid(GridRange::try_new(grid, 800, 20).unwrap()),
fields.clone(),
)
.unwrap();
assert_eq!(series.time.len(), 20);
assert!(matches!(
series.fields.get(0).unwrap().sample(4),
SampleRef::Missing
));
assert!(matches!(series.fields.get(0).unwrap().sample(7), SampleRef::Float(v) if v.is_nan()));
assert!(matches!(
series.fields.get(1).unwrap().sample(19),
SampleRef::Histogram(_)
));
assert!(Arc::ptr_eq(
match series.fields.get(1).unwrap().dense().unwrap() {
DenseSamplesRef::Histogram(samples) => samples.batch(),
_ => unreachable!(),
},
&histogram_table,
));
let explicit_schema = SeriesSchema {
fields: schema.fields.clone(),
eval_grid: None,
};
let explicit_fields = SeriesFields::try_from_vec(vec![
Samples::from_float(Float64Array::new_null(4)),
Samples::try_from_histogram(UInt32Array::from(vec![None; 4]), histogram_table).unwrap(),
])
.unwrap();
let out_of_order = TimeAxis::ExplicitChunks(Arc::from([
ScalarBuffer::from(vec![1_i64, 3]),
ScalarBuffer::from(Vec::<i64>::new()),
ScalarBuffer::from(vec![2_i64, 4]),
]));
assert!(
Series::try_new(
&explicit_schema,
MetricName::try_new(None, vec![]).unwrap(),
out_of_order,
explicit_fields,
)
.is_err()
);
let ordered_chunks = TimeAxis::ExplicitChunks(Arc::from([
ScalarBuffer::from(vec![1_i64, 3]),
ScalarBuffer::from(Vec::<i64>::new()),
ScalarBuffer::from(vec![5_i64]),
]));
let ordered_fields = SeriesFields::try_from_vec(vec![
Samples::from_float(Float64Array::from(vec![1.0, 2.0, 3.0])),
Samples::from_float(Float64Array::from(vec![4.0, 5.0, 6.0])),
])
.unwrap();
assert!(
Series::try_new(
&explicit_schema,
MetricName::try_new(None, vec![]).unwrap(),
ordered_chunks.clone(),
ordered_fields.clone(),
)
.is_ok()
);
let grid = Arc::new(EvalGrid::try_new(GridSpec::try_new(1, 2, 3).unwrap()).unwrap());
let grid_schema = SeriesSchema {
fields: explicit_schema.fields.clone(),
eval_grid: Some(grid),
};
assert!(
Series::try_new(
&grid_schema,
MetricName::try_new(None, vec![]).unwrap(),
ordered_chunks,
ordered_fields,
)
.is_err()
);
let unordered_single = TimeAxis::Explicit(ScalarBuffer::from(vec![2_i64, 1]));
assert!(
Series::try_new(
&SeriesSchema {
fields: Arc::from([FieldSpec {
name: Arc::from("value")
}]),
eval_grid: None
},
MetricName::try_new(None, vec![]).unwrap(),
unordered_single,
SeriesFields::one(Samples::from_float(Float64Array::from(vec![1.0, 2.0]))),
)
.is_err()
);
}