mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-18 00:34:11 +00:00
feat(datatypes): add shared native series samples and fields
Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
This commit is contained in:
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub mod histogram;
|
||||
pub mod samples;
|
||||
pub mod time;
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -0,0 +1,620 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use arrow::array::{Array, Float64Array, UInt32Array, UInt32Builder};
|
||||
use arrow::compute::concat;
|
||||
use snafu::OptionExt;
|
||||
|
||||
use crate::error::{InvalidSeriesSnafu, Result};
|
||||
use crate::series::MetricName;
|
||||
use crate::series::histogram::{HistogramBatch, HistogramView};
|
||||
use crate::series::time::{EvalGrid, TimeAxis};
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
fn invalid<T>(reason: impl Into<String>) -> Result<T> {
|
||||
Err(InvalidSeriesSnafu {
|
||||
reason: reason.into(),
|
||||
}
|
||||
.build())
|
||||
}
|
||||
fn arrow_result<T>(r: std::result::Result<T, arrow::error::ArrowError>) -> Result<T> {
|
||||
r.map_err(|e| {
|
||||
InvalidSeriesSnafu {
|
||||
reason: e.to_string(),
|
||||
}
|
||||
.build()
|
||||
})
|
||||
}
|
||||
|
||||
/// Histogram references for each time position, backed by one immutable table.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct HistogramSamples {
|
||||
indices: UInt32Array,
|
||||
batch: Arc<HistogramBatch>,
|
||||
}
|
||||
impl HistogramSamples {
|
||||
/// Validate and bind reference indices to their immutable histogram table.
|
||||
pub fn try_new(indices: UInt32Array, batch: Arc<HistogramBatch>) -> Result<Self> {
|
||||
if indices.iter().flatten().any(|i| i as usize >= batch.len()) {
|
||||
return invalid("histogram sample index is out of bounds");
|
||||
}
|
||||
Ok(Self { indices, batch })
|
||||
}
|
||||
/// Read-only reference indices.
|
||||
pub fn indices(&self) -> &UInt32Array {
|
||||
&self.indices
|
||||
}
|
||||
/// Shared histogram table.
|
||||
pub fn batch(&self) -> &Arc<HistogramBatch> {
|
||||
&self.batch
|
||||
}
|
||||
/// Number of time positions.
|
||||
pub fn len(&self) -> usize {
|
||||
self.indices.len()
|
||||
}
|
||||
/// Whether this contains no positions.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.indices.is_empty()
|
||||
}
|
||||
/// Returns a referenced histogram, or `None` for missing/out-of-range positions.
|
||||
pub fn view_at(&self, index: usize) -> Option<HistogramView<'_>> {
|
||||
if index >= self.indices.len() {
|
||||
return None;
|
||||
}
|
||||
if self.indices.is_valid(index) {
|
||||
self.batch.view(self.indices.value(index) as usize)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
fn slice(&self, offset: usize, len: usize) -> Self {
|
||||
Self {
|
||||
indices: self.indices.slice(offset, len),
|
||||
batch: self.batch.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Float and histogram values with mutually exclusive validity at each position.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct MixedSamples {
|
||||
floats: Float64Array,
|
||||
histograms: HistogramSamples,
|
||||
}
|
||||
impl MixedSamples {
|
||||
/// Read-only float values, including validity bitmap.
|
||||
pub fn floats(&self) -> &Float64Array {
|
||||
&self.floats
|
||||
}
|
||||
/// Read-only histogram references and their shared table.
|
||||
pub fn histograms(&self) -> &HistogramSamples {
|
||||
&self.histograms
|
||||
}
|
||||
/// Number of positions.
|
||||
pub fn len(&self) -> usize {
|
||||
self.floats.len()
|
||||
}
|
||||
/// Whether this contains no positions.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.floats.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
enum SampleStorage {
|
||||
Float(Float64Array),
|
||||
Histogram(HistogramSamples),
|
||||
Mixed(MixedSamples),
|
||||
Chunks(Arc<[Samples]>, usize),
|
||||
}
|
||||
|
||||
/// Typed Arrow-backed values for a time series.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Samples {
|
||||
storage: SampleStorage,
|
||||
}
|
||||
|
||||
/// A borrowed dense representation.
|
||||
pub enum DenseSamplesRef<'a> {
|
||||
Float(&'a Float64Array),
|
||||
Histogram(&'a HistogramSamples),
|
||||
Mixed(&'a MixedSamples),
|
||||
}
|
||||
|
||||
/// A sample value; ordinary NaN is a valid float, distinct from Missing.
|
||||
pub enum SampleRef<'a> {
|
||||
Missing,
|
||||
Float(f64),
|
||||
Histogram(HistogramView<'a>),
|
||||
}
|
||||
|
||||
impl Samples {
|
||||
/// Wrap float samples without copying; nulls are missing and NaNs remain values.
|
||||
pub fn from_float(values: Float64Array) -> Self {
|
||||
Self {
|
||||
storage: SampleStorage::Float(values),
|
||||
}
|
||||
}
|
||||
/// Validate histogram indices and retain their referenced table.
|
||||
pub fn try_from_histogram(indices: UInt32Array, batch: Arc<HistogramBatch>) -> Result<Self> {
|
||||
Ok(Self {
|
||||
storage: SampleStorage::Histogram(HistogramSamples::try_new(indices, batch)?),
|
||||
})
|
||||
}
|
||||
/// Validate lengths and ensure no position is simultaneously float and histogram.
|
||||
pub fn try_from_mixed(floats: Float64Array, histograms: HistogramSamples) -> Result<Self> {
|
||||
if floats.len() != histograms.len() {
|
||||
return invalid("mixed sample arrays have unequal lengths");
|
||||
}
|
||||
if (0..floats.len()).any(|i| floats.is_valid(i) && histograms.indices.is_valid(i)) {
|
||||
return invalid("mixed sample position has both float and histogram values");
|
||||
}
|
||||
Ok(Self {
|
||||
storage: SampleStorage::Mixed(MixedSamples { floats, histograms }),
|
||||
})
|
||||
}
|
||||
/// Retain immutable chunks, flattening nested descriptors and checking total length.
|
||||
pub fn try_from_chunks(chunks: Vec<Samples>) -> Result<Self> {
|
||||
let mut flat = Vec::new();
|
||||
for chunk in chunks {
|
||||
match chunk.storage {
|
||||
SampleStorage::Chunks(parts, _) => flat.extend(parts.iter().cloned()),
|
||||
_ => flat.push(chunk),
|
||||
}
|
||||
}
|
||||
let total = flat
|
||||
.iter()
|
||||
.try_fold(0usize, |n, s| n.checked_add(s.len()))
|
||||
.context(InvalidSeriesSnafu {
|
||||
reason: "sample chunk length overflow",
|
||||
})?;
|
||||
flat.retain(|s| !s.is_empty());
|
||||
if flat.is_empty() {
|
||||
return Ok(Self::from_float(Float64Array::new_null(0)));
|
||||
}
|
||||
if flat.len() == 1 {
|
||||
return flat.pop().ok_or_else(|| {
|
||||
InvalidSeriesSnafu {
|
||||
reason: "single sample chunk disappeared".to_string(),
|
||||
}
|
||||
.build()
|
||||
});
|
||||
}
|
||||
Ok(Self {
|
||||
storage: SampleStorage::Chunks(flat.into(), total),
|
||||
})
|
||||
}
|
||||
/// Number of sample positions.
|
||||
pub fn len(&self) -> usize {
|
||||
match &self.storage {
|
||||
SampleStorage::Float(a) => a.len(),
|
||||
SampleStorage::Histogram(a) => a.len(),
|
||||
SampleStorage::Mixed(a) => a.len(),
|
||||
SampleStorage::Chunks(_, len) => *len,
|
||||
}
|
||||
}
|
||||
/// Whether this has no positions.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
/// Returns a dense typed representation, or `None` for chunks.
|
||||
pub fn dense(&self) -> Option<DenseSamplesRef<'_>> {
|
||||
match &self.storage {
|
||||
SampleStorage::Float(a) => Some(DenseSamplesRef::Float(a)),
|
||||
SampleStorage::Histogram(a) => Some(DenseSamplesRef::Histogram(a)),
|
||||
SampleStorage::Mixed(a) => Some(DenseSamplesRef::Mixed(a)),
|
||||
SampleStorage::Chunks(_, _) => None,
|
||||
}
|
||||
}
|
||||
/// Returns the sample, treating an out-of-range index as missing.
|
||||
pub fn sample(&self, index: usize) -> SampleRef<'_> {
|
||||
if index >= self.len() {
|
||||
return SampleRef::Missing;
|
||||
}
|
||||
match &self.storage {
|
||||
SampleStorage::Float(a) => {
|
||||
if a.is_valid(index) {
|
||||
SampleRef::Float(a.value(index))
|
||||
} else {
|
||||
SampleRef::Missing
|
||||
}
|
||||
}
|
||||
SampleStorage::Histogram(a) => a
|
||||
.view_at(index)
|
||||
.map_or(SampleRef::Missing, SampleRef::Histogram),
|
||||
SampleStorage::Mixed(a) => {
|
||||
if a.floats.is_valid(index) {
|
||||
SampleRef::Float(a.floats.value(index))
|
||||
} else {
|
||||
a.histograms
|
||||
.view_at(index)
|
||||
.map_or(SampleRef::Missing, SampleRef::Histogram)
|
||||
}
|
||||
}
|
||||
SampleStorage::Chunks(chunks, _) => {
|
||||
let mut i = index;
|
||||
for part in chunks.iter() {
|
||||
if i < part.len() {
|
||||
return part.sample(i);
|
||||
}
|
||||
i -= part.len();
|
||||
}
|
||||
SampleRef::Missing
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Iterate all positions sequentially without materializing per-sample objects.
|
||||
pub fn iter(&self) -> impl ExactSizeIterator<Item = SampleRef<'_>> + '_ {
|
||||
let (chunks, remaining) = match &self.storage {
|
||||
SampleStorage::Chunks(chunks, len) => (Some(chunks.as_ref()), *len),
|
||||
_ => (None, self.len()),
|
||||
};
|
||||
SamplesIter {
|
||||
samples: self,
|
||||
chunks,
|
||||
chunk_index: 0,
|
||||
local_index: 0,
|
||||
remaining,
|
||||
}
|
||||
}
|
||||
/// Return a checked zero-copy slice (across chunks only intersected descriptors are retained).
|
||||
pub fn slice(&self, offset: usize, len: usize) -> Result<Self> {
|
||||
let end = offset.checked_add(len).context(InvalidSeriesSnafu {
|
||||
reason: "sample slice bounds overflow",
|
||||
})?;
|
||||
if end > self.len() {
|
||||
return invalid("sample slice is out of bounds");
|
||||
}
|
||||
match &self.storage {
|
||||
SampleStorage::Float(a) => Ok(Self::from_float(a.slice(offset, len))),
|
||||
SampleStorage::Histogram(a) => Ok(Self {
|
||||
storage: SampleStorage::Histogram(a.slice(offset, len)),
|
||||
}),
|
||||
SampleStorage::Mixed(a) => Ok(Self {
|
||||
storage: SampleStorage::Mixed(MixedSamples {
|
||||
floats: a.floats.slice(offset, len),
|
||||
histograms: a.histograms.slice(offset, len),
|
||||
}),
|
||||
}),
|
||||
SampleStorage::Chunks(chunks, _) => {
|
||||
let mut skip = offset;
|
||||
let mut remain = len;
|
||||
let mut out = Vec::new();
|
||||
for chunk in chunks.iter() {
|
||||
if skip >= chunk.len() {
|
||||
skip -= chunk.len();
|
||||
continue;
|
||||
}
|
||||
let take = remain.min(chunk.len() - skip);
|
||||
if take != 0 {
|
||||
out.push(chunk.slice(skip, take)?);
|
||||
remain -= take;
|
||||
}
|
||||
skip = 0;
|
||||
if remain == 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Self::try_from_chunks(out)
|
||||
}
|
||||
}
|
||||
}
|
||||
/// Materialize chunks into one dense array; histogram tables are concatenated per chunk,
|
||||
/// including repeated shared tables (no registry/deduplication is performed).
|
||||
pub fn into_dense(self) -> Result<Self> {
|
||||
let SampleStorage::Chunks(chunks, _) = self.storage else {
|
||||
return Ok(self);
|
||||
};
|
||||
let all_float = chunks
|
||||
.iter()
|
||||
.all(|s| matches!(s.storage, SampleStorage::Float(_)));
|
||||
if all_float {
|
||||
let arrays: Vec<&dyn Array> = chunks
|
||||
.iter()
|
||||
.filter_map(|s| match &s.storage {
|
||||
SampleStorage::Float(a) => Some(a as &dyn Array),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
let array = arrow_result(concat(&arrays))?;
|
||||
let values = array
|
||||
.as_any()
|
||||
.downcast_ref::<Float64Array>()
|
||||
.ok_or_else(|| {
|
||||
InvalidSeriesSnafu {
|
||||
reason: "float concatenation returned a non-float array",
|
||||
}
|
||||
.build()
|
||||
})?;
|
||||
return Ok(Self::from_float(values.clone()));
|
||||
}
|
||||
|
||||
let mut tables: Vec<&dyn Array> = Vec::new();
|
||||
let mut indices = UInt32Builder::new();
|
||||
let mut floats = arrow::array::Float64Builder::new();
|
||||
let mut base = 0u32;
|
||||
for chunk in chunks.iter() {
|
||||
let (float_values, histogram) = match &chunk.storage {
|
||||
SampleStorage::Float(values) => (Some(values), None),
|
||||
SampleStorage::Histogram(histogram) => (None, Some(histogram)),
|
||||
SampleStorage::Mixed(mixed) => (Some(&mixed.floats), Some(&mixed.histograms)),
|
||||
SampleStorage::Chunks(_, _) => {
|
||||
return invalid("nested sample chunks were not flattened");
|
||||
}
|
||||
};
|
||||
if let Some(histogram) = histogram {
|
||||
tables.push(histogram.batch.as_arrow());
|
||||
}
|
||||
for position in 0..chunk.len() {
|
||||
match float_values {
|
||||
Some(values) if values.is_valid(position) => {
|
||||
floats.append_value(values.value(position))
|
||||
}
|
||||
_ => floats.append_null(),
|
||||
}
|
||||
match histogram {
|
||||
Some(histogram) if histogram.indices.is_valid(position) => {
|
||||
indices.append_value(
|
||||
base.checked_add(histogram.indices.value(position))
|
||||
.context(InvalidSeriesSnafu {
|
||||
reason: "histogram remapped index overflow",
|
||||
})?,
|
||||
);
|
||||
}
|
||||
_ => indices.append_null(),
|
||||
}
|
||||
}
|
||||
if let Some(histogram) = histogram {
|
||||
base = base
|
||||
.checked_add(u32::try_from(histogram.batch.len()).map_err(|_| {
|
||||
InvalidSeriesSnafu {
|
||||
reason: "histogram table too large",
|
||||
}
|
||||
.build()
|
||||
})?)
|
||||
.context(InvalidSeriesSnafu {
|
||||
reason: "histogram table index overflow",
|
||||
})?;
|
||||
}
|
||||
}
|
||||
let table = arrow_result(concat(&tables))?;
|
||||
let struct_array = table
|
||||
.as_any()
|
||||
.downcast_ref::<arrow::array::StructArray>()
|
||||
.ok_or_else(|| {
|
||||
InvalidSeriesSnafu {
|
||||
reason: "histogram concatenation returned a non-struct array",
|
||||
}
|
||||
.build()
|
||||
})?
|
||||
.clone();
|
||||
let batch = Arc::new(HistogramBatch::try_from_arrow(struct_array)?);
|
||||
let histogram = HistogramSamples::try_new(indices.finish(), batch)?;
|
||||
let floats = floats.finish();
|
||||
if chunks.iter().any(|chunk| {
|
||||
matches!(
|
||||
chunk.storage,
|
||||
SampleStorage::Float(_) | SampleStorage::Mixed(_)
|
||||
)
|
||||
}) {
|
||||
Self::try_from_mixed(floats, histogram)
|
||||
} else {
|
||||
Ok(Self {
|
||||
storage: SampleStorage::Histogram(histogram),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sequential exact-size iterator over a dense series or its chunk descriptors.
|
||||
struct SamplesIter<'a> {
|
||||
samples: &'a Samples,
|
||||
chunks: Option<&'a [Samples]>,
|
||||
chunk_index: usize,
|
||||
local_index: usize,
|
||||
remaining: usize,
|
||||
}
|
||||
impl<'a> Iterator for SamplesIter<'a> {
|
||||
type Item = SampleRef<'a>;
|
||||
|
||||
fn next(&mut self) -> Option<Self::Item> {
|
||||
if self.remaining == 0 {
|
||||
return None;
|
||||
}
|
||||
self.remaining -= 1;
|
||||
if let Some(chunks) = self.chunks {
|
||||
while self.chunk_index < chunks.len()
|
||||
&& self.local_index == chunks[self.chunk_index].len()
|
||||
{
|
||||
self.chunk_index += 1;
|
||||
self.local_index = 0;
|
||||
}
|
||||
let chunk = &chunks[self.chunk_index];
|
||||
let value = chunk.sample(self.local_index);
|
||||
self.local_index += 1;
|
||||
Some(value)
|
||||
} else {
|
||||
let index = self.samples.len() - self.remaining - 1;
|
||||
Some(self.samples.sample(index))
|
||||
}
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
(self.remaining, Some(self.remaining))
|
||||
}
|
||||
}
|
||||
impl ExactSizeIterator for SamplesIter<'_> {
|
||||
fn len(&self) -> usize {
|
||||
self.remaining
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
enum Fields<T> {
|
||||
One(T),
|
||||
Many(Box<[T]>),
|
||||
}
|
||||
/// Field values using inline storage for a single field.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SeriesFields {
|
||||
storage: Fields<Samples>,
|
||||
}
|
||||
impl SeriesFields {
|
||||
/// Create a one-field collection without allocating a field list.
|
||||
pub fn one(field: Samples) -> Self {
|
||||
Self {
|
||||
storage: Fields::One(field),
|
||||
}
|
||||
}
|
||||
/// Validate nonempty, equally sized fields.
|
||||
pub fn try_from_vec(fields: Vec<Samples>) -> Result<Self> {
|
||||
if fields.is_empty() {
|
||||
return invalid("series fields must not be empty");
|
||||
}
|
||||
if fields.iter().any(|f| f.len() != fields[0].len()) {
|
||||
return invalid("series fields have unequal lengths");
|
||||
}
|
||||
Ok(if fields.len() == 1 {
|
||||
let mut fields = fields.into_iter();
|
||||
let field = fields.next().ok_or_else(|| {
|
||||
InvalidSeriesSnafu {
|
||||
reason: "single series field disappeared".to_string(),
|
||||
}
|
||||
.build()
|
||||
})?;
|
||||
Self::one(field)
|
||||
} else {
|
||||
Self {
|
||||
storage: Fields::Many(fields.into_boxed_slice()),
|
||||
}
|
||||
})
|
||||
}
|
||||
/// Borrow all fields.
|
||||
pub fn as_slice(&self) -> &[Samples] {
|
||||
match &self.storage {
|
||||
Fields::One(f) => std::slice::from_ref(f),
|
||||
Fields::Many(f) => f,
|
||||
}
|
||||
}
|
||||
/// Number of fields.
|
||||
pub fn len(&self) -> usize {
|
||||
self.as_slice().len()
|
||||
}
|
||||
/// Whether this has no fields.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
false
|
||||
}
|
||||
/// Get a field by index.
|
||||
pub fn get(&self, index: usize) -> Option<&Samples> {
|
||||
self.as_slice().get(index)
|
||||
}
|
||||
/// Iterate fields.
|
||||
pub fn iter(&self) -> std::slice::Iter<'_, Samples> {
|
||||
self.as_slice().iter()
|
||||
}
|
||||
}
|
||||
|
||||
/// Shared name and position of each output field.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct FieldSpec {
|
||||
pub name: Arc<str>,
|
||||
}
|
||||
/// Shared schema describing series fields.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SeriesSchema {
|
||||
pub fields: Arc<[FieldSpec]>,
|
||||
/// Shared grid for regular series; its `GridSpec` must match each series grid axis. Explicit axes require `None`.
|
||||
pub eval_grid: Option<Arc<EvalGrid>>,
|
||||
}
|
||||
/// A validated metric, time axis, and aligned fields.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Series {
|
||||
pub metric: MetricName,
|
||||
pub time: TimeAxis,
|
||||
pub fields: SeriesFields,
|
||||
}
|
||||
impl Series {
|
||||
/// Construct after validating schema, field lengths, and strictly increasing timestamps.
|
||||
pub fn try_new(
|
||||
schema: &SeriesSchema,
|
||||
metric: MetricName,
|
||||
time: TimeAxis,
|
||||
fields: SeriesFields,
|
||||
) -> Result<Self> {
|
||||
if fields.len() != schema.fields.len() {
|
||||
return invalid("series field count does not match schema");
|
||||
}
|
||||
let time_len = checked_time_len(&time)?;
|
||||
if fields.iter().any(|f| f.len() != time_len) {
|
||||
return invalid("series field length does not match time axis");
|
||||
}
|
||||
match (&schema.eval_grid, &time) {
|
||||
(Some(expected), TimeAxis::Grid(range)) if expected.spec() == range.grid().spec() => {}
|
||||
(Some(_), TimeAxis::Grid(_)) => {
|
||||
return invalid("series grid does not match schema evaluation grid");
|
||||
}
|
||||
(Some(_), _) => {
|
||||
return invalid("explicit series time axis cannot use an evaluation grid");
|
||||
}
|
||||
(None, TimeAxis::Grid(_)) => {
|
||||
return invalid("grid series requires a schema evaluation grid");
|
||||
}
|
||||
(None, TimeAxis::Explicit(_) | TimeAxis::ExplicitChunks(_)) => {}
|
||||
}
|
||||
Ok(Self {
|
||||
metric,
|
||||
time,
|
||||
fields,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn checked_time_len(time: &TimeAxis) -> Result<usize> {
|
||||
match time {
|
||||
TimeAxis::Grid(range) => Ok(range.len()),
|
||||
TimeAxis::Explicit(timestamps) => {
|
||||
if timestamps.windows(2).any(|pair| pair[0] >= pair[1]) {
|
||||
return invalid("timestamps must be strictly increasing");
|
||||
}
|
||||
Ok(timestamps.len())
|
||||
}
|
||||
TimeAxis::ExplicitChunks(chunks) => {
|
||||
let mut total = 0usize;
|
||||
let mut previous = None;
|
||||
for chunk in chunks.iter() {
|
||||
total = total.checked_add(chunk.len()).ok_or_else(|| {
|
||||
InvalidSeriesSnafu {
|
||||
reason: "timestamp length overflow".to_string(),
|
||||
}
|
||||
.build()
|
||||
})?;
|
||||
if chunk.windows(2).any(|pair| pair[0] >= pair[1])
|
||||
|| chunk
|
||||
.first()
|
||||
.is_some_and(|first| previous.is_some_and(|last| last >= *first))
|
||||
{
|
||||
return invalid("timestamps must be strictly increasing across chunks");
|
||||
}
|
||||
if let Some(last) = chunk.last() {
|
||||
previous = Some(*last);
|
||||
}
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,842 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use arrow::array::{Float64Array, UInt32Array};
|
||||
use arrow::buffer::{Buffer, ScalarBuffer};
|
||||
|
||||
use crate::series::histogram::{BucketSideView, CounterResetHint, HistogramBatchBuilder};
|
||||
|
||||
use crate::series::time::{EvalGrid, GridRange, GridSpec, TimeAxis};
|
||||
|
||||
fn batch(sum: f64, count: f64, buckets: &[f64]) -> Arc<HistogramBatch> {
|
||||
let offsets = if buckets.is_empty() { vec![] } else { vec![0] };
|
||||
let lengths = if buckets.is_empty() {
|
||||
vec![]
|
||||
} else {
|
||||
vec![buckets.len() as i32]
|
||||
};
|
||||
let side = BucketSideView {
|
||||
span_offsets: &offsets,
|
||||
span_lengths: &lengths,
|
||||
buckets,
|
||||
};
|
||||
let row = HistogramView {
|
||||
schema: 1,
|
||||
zero_threshold: 0.0,
|
||||
sum,
|
||||
count,
|
||||
zero_count: 0.0,
|
||||
reset_hint: CounterResetHint::Unknown,
|
||||
start_timestamp: None,
|
||||
custom_values: &[],
|
||||
positive: side,
|
||||
negative: BucketSideView {
|
||||
span_offsets: &[],
|
||||
span_lengths: &[],
|
||||
buckets: &[],
|
||||
},
|
||||
};
|
||||
histogram_batch(&[row])
|
||||
}
|
||||
|
||||
fn histogram_batch(rows: &[HistogramView<'_>]) -> Arc<HistogramBatch> {
|
||||
let mut builder = HistogramBatchBuilder::new();
|
||||
for row in rows {
|
||||
builder.push(*row).unwrap();
|
||||
}
|
||||
builder.finish().unwrap()
|
||||
}
|
||||
|
||||
struct FloatBufferOwner {
|
||||
values: Vec<f64>,
|
||||
drops: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl Drop for FloatBufferOwner {
|
||||
fn drop(&mut self) {
|
||||
self.drops.fetch_add(1, Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
fn owned_floats(values: Vec<f64>, drops: Arc<AtomicUsize>) -> ScalarBuffer<f64> {
|
||||
let mut owner = FloatBufferOwner { values, drops };
|
||||
let pointer = std::ptr::NonNull::new(owner.values.as_mut_ptr().cast()).unwrap();
|
||||
let len = owner.values.len() * std::mem::size_of::<f64>();
|
||||
let element_count = owner.values.len();
|
||||
// SAFETY: the vector allocation remains stable in the Arc owner for the buffer lifetime.
|
||||
let buffer = unsafe { Buffer::from_custom_allocation(pointer, len, Arc::new(owner)) };
|
||||
ScalarBuffer::new(buffer, 0, element_count)
|
||||
}
|
||||
|
||||
fn same_histogram(a: HistogramView<'_>, b: HistogramView<'_>) -> bool {
|
||||
a.schema == b.schema
|
||||
&& a.zero_threshold.to_bits() == b.zero_threshold.to_bits()
|
||||
&& a.sum.to_bits() == b.sum.to_bits()
|
||||
&& a.count.to_bits() == b.count.to_bits()
|
||||
&& a.zero_count.to_bits() == b.zero_count.to_bits()
|
||||
&& a.reset_hint == b.reset_hint
|
||||
&& a.start_timestamp == b.start_timestamp
|
||||
&& a.custom_values
|
||||
.iter()
|
||||
.map(|value| value.to_bits())
|
||||
.eq(b.custom_values.iter().map(|value| value.to_bits()))
|
||||
&& a.positive.span_offsets == b.positive.span_offsets
|
||||
&& a.positive.span_lengths == b.positive.span_lengths
|
||||
&& a.positive.buckets.iter().map(|value| value.to_bits()).eq(b
|
||||
.positive
|
||||
.buckets
|
||||
.iter()
|
||||
.map(|value| value.to_bits()))
|
||||
&& a.negative.span_offsets == b.negative.span_offsets
|
||||
&& a.negative.span_lengths == b.negative.span_lengths
|
||||
&& a.negative.buckets.iter().map(|value| value.to_bits()).eq(b
|
||||
.negative
|
||||
.buckets
|
||||
.iter()
|
||||
.map(|value| value.to_bits()))
|
||||
}
|
||||
fn same_sample(a: SampleRef<'_>, b: SampleRef<'_>) -> bool {
|
||||
match (a, b) {
|
||||
(SampleRef::Missing, SampleRef::Missing) => true,
|
||||
(SampleRef::Float(a), SampleRef::Float(b)) => a.to_bits() == b.to_bits(),
|
||||
(SampleRef::Histogram(a), SampleRef::Histogram(b)) => same_histogram(a, b),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn float_states_and_slice_share_values_and_bitmap_after_owner_drop() {
|
||||
let stale = f64::from_bits(0x7ff0_0000_0000_0002);
|
||||
let owner = Float64Array::from(vec![Some(1.0), None, Some(f64::NAN), Some(stale)]);
|
||||
let samples = Samples::from_float(owner.clone());
|
||||
assert!(matches!(samples.sample(1), SampleRef::Missing));
|
||||
assert!(matches!(samples.sample(2), SampleRef::Float(v) if v.is_nan()));
|
||||
assert!(matches!(samples.sample(3), SampleRef::Float(v) if v.to_bits() == stale.to_bits()));
|
||||
let sliced = samples.slice(1, 3).unwrap();
|
||||
let DenseSamplesRef::Float(slice) = sliced.dense().unwrap() else {
|
||||
panic!()
|
||||
};
|
||||
assert_eq!(
|
||||
slice.values().as_ptr(),
|
||||
owner.values().as_ptr().wrapping_add(1)
|
||||
);
|
||||
assert_eq!(
|
||||
slice.nulls().unwrap().buffer().as_ptr(),
|
||||
owner.nulls().unwrap().buffer().as_ptr()
|
||||
);
|
||||
assert!(slice.is_null(0));
|
||||
drop(samples);
|
||||
drop(owner);
|
||||
assert!(matches!(sliced.sample(0), SampleRef::Missing));
|
||||
assert!(matches!(sliced.sample(1), SampleRef::Float(v) if v.is_nan()));
|
||||
assert!(matches!(sliced.sample(2), SampleRef::Float(v) if v.to_bits() == stale.to_bits()));
|
||||
assert!(sliced.slice(usize::MAX, 2).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn histogram_bounds_nulls_and_public_view_bounds_are_safe() {
|
||||
let owner = batch(2.0, 3.0, &[4.0, 5.0]);
|
||||
let refs =
|
||||
HistogramSamples::try_new(UInt32Array::from(vec![None, Some(0)]), owner.clone()).unwrap();
|
||||
assert!(refs.view_at(usize::MAX).is_none());
|
||||
assert!(refs.view_at(2).is_none());
|
||||
assert!(refs.view_at(0).is_none());
|
||||
assert_eq!(refs.view_at(1).unwrap().sum.to_bits(), 2.0f64.to_bits());
|
||||
assert!(HistogramSamples::try_new(UInt32Array::from(vec![Some(1)]), owner).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_validates_length_exclusivity_and_missing_positions() {
|
||||
let table = batch(2.0, 1.0, &[6.0]);
|
||||
let refs =
|
||||
HistogramSamples::try_new(UInt32Array::from(vec![None, Some(0), None]), table).unwrap();
|
||||
assert!(
|
||||
Samples::try_from_mixed(Float64Array::from(vec![None, Some(1.0)]), refs.clone()).is_err()
|
||||
);
|
||||
assert!(
|
||||
Samples::try_from_mixed(
|
||||
Float64Array::from(vec![None, Some(1.0), None]),
|
||||
refs.clone()
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let mixed = Samples::try_from_mixed(Float64Array::from(vec![None, None, None]), refs).unwrap();
|
||||
assert!(matches!(mixed.sample(0), SampleRef::Missing));
|
||||
assert!(matches!(mixed.sample(1), SampleRef::Histogram(_)));
|
||||
assert!(matches!(mixed.sample(2), SampleRef::Missing));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunk_iterator_is_sequential_exact_and_handles_many_singletons() {
|
||||
let inner = Samples::try_from_chunks(vec![
|
||||
Samples::from_float(Float64Array::new_null(0)),
|
||||
Samples::from_float(Float64Array::from(vec![2.0])),
|
||||
Samples::from_float(Float64Array::from(vec![3.0])),
|
||||
])
|
||||
.unwrap();
|
||||
let normalized = Samples::try_from_chunks(vec![
|
||||
Samples::from_float(Float64Array::from(vec![0.0, 1.0])),
|
||||
inner,
|
||||
Samples::from_float(Float64Array::new_null(0)),
|
||||
Samples::from_float(Float64Array::from(vec![4.0])),
|
||||
])
|
||||
.unwrap();
|
||||
let SampleStorage::Chunks(parts, _) = &normalized.storage else {
|
||||
panic!()
|
||||
};
|
||||
assert_eq!(parts.len(), 4);
|
||||
assert_eq!(normalized.iter().len(), 5);
|
||||
for (index, part) in parts.iter().enumerate() {
|
||||
assert!(matches!(part.dense(), Some(DenseSamplesRef::Float(_))));
|
||||
assert!(matches!(part.sample(0), SampleRef::Float(v) if v == [0.0, 2.0, 3.0, 4.0][index]));
|
||||
}
|
||||
for (index, value) in normalized.iter().enumerate() {
|
||||
assert!(matches!(value, SampleRef::Float(v) if v == index as f64));
|
||||
}
|
||||
|
||||
let collapsed = Samples::try_from_chunks(vec![
|
||||
Samples::from_float(Float64Array::new_null(0)),
|
||||
normalized.slice(2, 1).unwrap(),
|
||||
Samples::from_float(Float64Array::new_null(0)),
|
||||
])
|
||||
.unwrap();
|
||||
assert!(matches!(collapsed.storage, SampleStorage::Float(_)));
|
||||
assert!(matches!(collapsed.sample(0), SampleRef::Float(v) if v == 2.0));
|
||||
let empty = Samples::try_from_chunks(vec![]).unwrap();
|
||||
assert!(empty.is_empty());
|
||||
assert!(matches!(empty.dense(), Some(DenseSamplesRef::Float(values)) if values.is_empty()));
|
||||
|
||||
let chunks = Samples::try_from_chunks(
|
||||
(0..256)
|
||||
.map(|i| Samples::from_float(Float64Array::from(vec![i as f64])))
|
||||
.collect(),
|
||||
)
|
||||
.unwrap();
|
||||
let mut iter = chunks.iter();
|
||||
assert_eq!(iter.len(), 256);
|
||||
for i in 0..256 {
|
||||
assert!(matches!(iter.next(), Some(SampleRef::Float(v)) if v == i as f64));
|
||||
assert_eq!(iter.len(), 255 - i);
|
||||
}
|
||||
assert!(iter.next().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn densification_preserves_values_nulls_nan_and_remaps_multitable_histograms() {
|
||||
let stale = f64::from_bits(0x7ff0_0000_0000_0002);
|
||||
let float = Samples::from_float(Float64Array::from(vec![Some(1.0), None, Some(stale)]));
|
||||
let first_buckets = [13.0, 14.0];
|
||||
let first_positive_offsets = [0];
|
||||
let first_positive_lengths = [2];
|
||||
let first_negative_offsets = [0];
|
||||
let first_negative_lengths = [1];
|
||||
let first_negative_buckets = [15.0];
|
||||
let first_custom = [16.0, 17.0];
|
||||
let first_rows = [
|
||||
HistogramView {
|
||||
schema: 2,
|
||||
zero_threshold: 0.25,
|
||||
sum: f64::from_bits(0x7ff0_0000_0000_0002),
|
||||
count: 12.5,
|
||||
zero_count: 1.5,
|
||||
reset_hint: CounterResetHint::Unrecognized(9),
|
||||
start_timestamp: Some(123),
|
||||
custom_values: &first_custom,
|
||||
positive: BucketSideView {
|
||||
span_offsets: &first_positive_offsets,
|
||||
span_lengths: &first_positive_lengths,
|
||||
buckets: &first_buckets,
|
||||
},
|
||||
negative: BucketSideView {
|
||||
span_offsets: &first_negative_offsets,
|
||||
span_lengths: &first_negative_lengths,
|
||||
buckets: &first_negative_buckets,
|
||||
},
|
||||
},
|
||||
HistogramView {
|
||||
schema: 1,
|
||||
zero_threshold: 0.0,
|
||||
sum: -21.0,
|
||||
count: 22.0,
|
||||
zero_count: 0.0,
|
||||
reset_hint: CounterResetHint::Unknown,
|
||||
start_timestamp: None,
|
||||
custom_values: &[],
|
||||
positive: BucketSideView {
|
||||
span_offsets: &[0],
|
||||
span_lengths: &[1],
|
||||
buckets: &[23.0],
|
||||
},
|
||||
negative: BucketSideView {
|
||||
span_offsets: &[],
|
||||
span_lengths: &[],
|
||||
buckets: &[],
|
||||
},
|
||||
},
|
||||
];
|
||||
let first_table = histogram_batch(&first_rows);
|
||||
let first =
|
||||
Samples::try_from_histogram(UInt32Array::from(vec![None, Some(1)]), first_table.clone())
|
||||
.unwrap();
|
||||
let second_positive_offsets = [0];
|
||||
let second_positive_lengths = [1];
|
||||
let second_positive_buckets = [31.0];
|
||||
let second_rows = [
|
||||
HistogramView {
|
||||
schema: 3,
|
||||
zero_threshold: 0.5,
|
||||
sum: 32.0,
|
||||
count: f64::from_bits(0x7ff0_0000_0000_0002),
|
||||
zero_count: 2.25,
|
||||
reset_hint: CounterResetHint::CounterReset,
|
||||
start_timestamp: Some(-456),
|
||||
custom_values: &[33.0],
|
||||
positive: BucketSideView {
|
||||
span_offsets: &second_positive_offsets,
|
||||
span_lengths: &second_positive_lengths,
|
||||
buckets: &second_positive_buckets,
|
||||
},
|
||||
negative: BucketSideView {
|
||||
span_offsets: &[0],
|
||||
span_lengths: &[1],
|
||||
buckets: &[34.0],
|
||||
},
|
||||
},
|
||||
HistogramView {
|
||||
schema: 4,
|
||||
zero_threshold: 0.75,
|
||||
sum: 35.0,
|
||||
count: 36.0,
|
||||
zero_count: 3.5,
|
||||
reset_hint: CounterResetHint::Gauge,
|
||||
start_timestamp: None,
|
||||
custom_values: &[37.0, 38.0],
|
||||
positive: BucketSideView {
|
||||
span_offsets: &[0],
|
||||
span_lengths: &[2],
|
||||
buckets: &[39.0, 40.0],
|
||||
},
|
||||
negative: BucketSideView {
|
||||
span_offsets: &[],
|
||||
span_lengths: &[],
|
||||
buckets: &[],
|
||||
},
|
||||
},
|
||||
HistogramView {
|
||||
schema: 5,
|
||||
zero_threshold: 1.0,
|
||||
sum: 41.0,
|
||||
count: 42.0,
|
||||
zero_count: 4.5,
|
||||
reset_hint: CounterResetHint::NotCounterReset,
|
||||
start_timestamp: Some(789),
|
||||
custom_values: &[],
|
||||
positive: BucketSideView {
|
||||
span_offsets: &[],
|
||||
span_lengths: &[],
|
||||
buckets: &[],
|
||||
},
|
||||
negative: BucketSideView {
|
||||
span_offsets: &[0],
|
||||
span_lengths: &[2],
|
||||
buckets: &[43.0, 44.0],
|
||||
},
|
||||
},
|
||||
];
|
||||
let second_table = histogram_batch(&second_rows);
|
||||
let second_refs =
|
||||
HistogramSamples::try_new(UInt32Array::from(vec![Some(2), None]), second_table.clone())
|
||||
.unwrap();
|
||||
let mixed =
|
||||
Samples::try_from_mixed(Float64Array::from(vec![None, Some(f64::NAN)]), second_refs)
|
||||
.unwrap();
|
||||
let chunked = Samples::try_from_chunks(vec![float, first.clone(), mixed]).unwrap();
|
||||
let expected: Vec<_> = chunked.iter().collect();
|
||||
let dense = chunked.clone().into_dense().unwrap();
|
||||
assert_eq!(dense.len(), expected.len());
|
||||
for (i, before) in expected.into_iter().enumerate() {
|
||||
assert!(same_sample(before, dense.sample(i)), "position {i}");
|
||||
}
|
||||
|
||||
let float_only = Samples::try_from_chunks(vec![
|
||||
Samples::from_float(Float64Array::from(vec![Some(1.0), None])),
|
||||
Samples::from_float(Float64Array::from(vec![Some(f64::NAN), Some(stale)])),
|
||||
])
|
||||
.unwrap();
|
||||
let float_expected: Vec<_> = float_only.iter().collect();
|
||||
let float_dense = float_only.clone().into_dense().unwrap();
|
||||
for (index, expected) in float_expected.into_iter().enumerate() {
|
||||
assert!(same_sample(expected, float_dense.sample(index)));
|
||||
}
|
||||
let DenseSamplesRef::Mixed(dense_mixed) = dense.dense().unwrap() else {
|
||||
panic!()
|
||||
};
|
||||
assert_eq!(dense_mixed.histograms.indices().value(4), 1);
|
||||
assert_eq!(dense_mixed.histograms.indices().value(5), 4);
|
||||
assert_eq!(dense_mixed.histograms.batch().len(), 5);
|
||||
for index in 0..5 {
|
||||
let actual = dense_mixed.histograms.batch().view(index).unwrap();
|
||||
let expected = if index < 2 {
|
||||
first_table.view(index).unwrap()
|
||||
} else {
|
||||
second_table.view(index - 2).unwrap()
|
||||
};
|
||||
assert!(same_sample(
|
||||
SampleRef::Histogram(actual),
|
||||
SampleRef::Histogram(expected)
|
||||
));
|
||||
}
|
||||
|
||||
let pure_hist = Samples::try_from_chunks(vec![
|
||||
first.clone(),
|
||||
Samples::try_from_histogram(UInt32Array::from(vec![Some(0)]), second_table).unwrap(),
|
||||
])
|
||||
.unwrap();
|
||||
let dense_hist = pure_hist.clone().into_dense().unwrap();
|
||||
assert!(matches!(
|
||||
dense_hist.dense(),
|
||||
Some(DenseSamplesRef::Histogram(_))
|
||||
));
|
||||
for index in 0..pure_hist.len() {
|
||||
assert!(same_sample(
|
||||
pure_hist.sample(index),
|
||||
dense_hist.sample(index)
|
||||
));
|
||||
}
|
||||
|
||||
let SampleStorage::Histogram(first_refs) = &first.storage else {
|
||||
panic!()
|
||||
};
|
||||
let sliced = first_refs.slice(1, 1);
|
||||
assert!(Arc::ptr_eq(sliced.batch(), &first_table));
|
||||
assert_eq!(
|
||||
sliced.indices().values().as_ptr(),
|
||||
first_refs.indices().values().as_ptr().wrapping_add(1)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunk_slices_keep_indices_tables_and_numeric_buffers_alive() {
|
||||
let f = Float64Array::from(vec![Some(1.0), None, Some(3.0)]);
|
||||
let float_samples = Samples::from_float(f.clone());
|
||||
let hist_table = batch(7.0, 8.0, &[9.0]);
|
||||
let hist_samples = Samples::try_from_histogram(
|
||||
UInt32Array::from(vec![None, Some(0), None]),
|
||||
hist_table.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
let mixed_table = batch(17.0, 18.0, &[19.0]);
|
||||
let mixed_values = Float64Array::from(vec![Some(20.0), None, None]);
|
||||
let mixed = Samples::try_from_mixed(
|
||||
mixed_values.clone(),
|
||||
HistogramSamples::try_new(
|
||||
UInt32Array::from(vec![None, None, Some(0)]),
|
||||
mixed_table.clone(),
|
||||
)
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
let chunks = Samples::try_from_chunks(vec![
|
||||
float_samples.clone(),
|
||||
hist_samples.clone(),
|
||||
mixed.clone(),
|
||||
])
|
||||
.unwrap();
|
||||
let sliced = chunks.slice(1, 8).unwrap();
|
||||
for index in 0..8 {
|
||||
assert!(same_sample(chunks.sample(index + 1), sliced.sample(index)));
|
||||
}
|
||||
let SampleStorage::Chunks(parts, _) = &sliced.storage else {
|
||||
panic!()
|
||||
};
|
||||
let SampleStorage::Float(fpart) = &parts[0].storage else {
|
||||
panic!()
|
||||
};
|
||||
assert_eq!(fpart.values().as_ptr(), f.values().as_ptr().wrapping_add(1));
|
||||
assert_eq!(
|
||||
fpart.nulls().unwrap().buffer().as_ptr(),
|
||||
f.nulls().unwrap().buffer().as_ptr()
|
||||
);
|
||||
let SampleStorage::Histogram(hpart) = &parts[1].storage else {
|
||||
panic!()
|
||||
};
|
||||
assert!(Arc::ptr_eq(hpart.batch(), &hist_table));
|
||||
let SampleStorage::Mixed(mpart) = &parts[2].storage else {
|
||||
panic!()
|
||||
};
|
||||
assert!(Arc::ptr_eq(mpart.histograms.batch(), &mixed_table));
|
||||
assert_eq!(
|
||||
mpart.floats.values().as_ptr(),
|
||||
mixed_values.values().as_ptr()
|
||||
);
|
||||
drop(chunks);
|
||||
drop(float_samples);
|
||||
drop(hist_samples);
|
||||
drop(mixed);
|
||||
drop(f);
|
||||
drop(mixed_values);
|
||||
drop(hist_table);
|
||||
drop(mixed_table);
|
||||
assert!(matches!(sliced.sample(1), SampleRef::Float(v) if v == 3.0));
|
||||
assert!(matches!(sliced.sample(7), SampleRef::Histogram(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fields_clone_buffers_and_series_validate_schema_alignment_and_time_order() {
|
||||
let sample = Samples::from_float(Float64Array::from(vec![1.0, 2.0]));
|
||||
let fields = SeriesFields::try_from_vec(vec![sample.clone(), sample.clone()]).unwrap();
|
||||
assert!(matches!(fields.storage, Fields::Many(_)));
|
||||
let cloned = fields.clone();
|
||||
let DenseSamplesRef::Float(a) = fields.get(0).unwrap().dense().unwrap() else {
|
||||
panic!()
|
||||
};
|
||||
let DenseSamplesRef::Float(b) = cloned.get(0).unwrap().dense().unwrap() else {
|
||||
panic!()
|
||||
};
|
||||
assert_eq!(a.values().as_ptr(), b.values().as_ptr());
|
||||
assert!(SeriesFields::try_from_vec(vec![]).is_err());
|
||||
assert!(
|
||||
SeriesFields::try_from_vec(vec![
|
||||
sample.clone(),
|
||||
Samples::from_float(Float64Array::from(vec![1.0]))
|
||||
])
|
||||
.is_err()
|
||||
);
|
||||
let schema = SeriesSchema {
|
||||
fields: Arc::from([
|
||||
FieldSpec {
|
||||
name: Arc::from("a"),
|
||||
},
|
||||
FieldSpec {
|
||||
name: Arc::from("b"),
|
||||
},
|
||||
]),
|
||||
eval_grid: None,
|
||||
};
|
||||
let metric = MetricName::try_new(None, vec![]).unwrap();
|
||||
let time = TimeAxis::Explicit(ScalarBuffer::from(vec![1, 2]));
|
||||
assert!(
|
||||
Series::try_new(
|
||||
&schema,
|
||||
metric.clone(),
|
||||
time.clone(),
|
||||
SeriesFields::one(sample.clone())
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
Series::try_new(
|
||||
&schema,
|
||||
metric.clone(),
|
||||
time.clone(),
|
||||
SeriesFields::try_from_vec(vec![
|
||||
Samples::from_float(Float64Array::from(vec![1.0])),
|
||||
Samples::from_float(Float64Array::from(vec![2.0]))
|
||||
])
|
||||
.unwrap()
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let unordered = TimeAxis::Explicit(ScalarBuffer::from(vec![2, 1]));
|
||||
assert!(Series::try_new(&schema, metric.clone(), unordered, fields.clone()).is_err());
|
||||
assert!(Series::try_new(&schema, metric, time, fields).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn float_fields_clones_and_series_keep_real_arrow_buffer_owner_alive() {
|
||||
let drops = Arc::new(AtomicUsize::new(0));
|
||||
let values = Float64Array::new(owned_floats(vec![7.0, 8.0], drops.clone()), None);
|
||||
let samples = Samples::from_float(values);
|
||||
let fields = SeriesFields::one(samples);
|
||||
let cloned = fields.clone();
|
||||
let schema = SeriesSchema {
|
||||
fields: Arc::from([FieldSpec {
|
||||
name: Arc::from("value"),
|
||||
}]),
|
||||
eval_grid: None,
|
||||
};
|
||||
let series = Series::try_new(
|
||||
&schema,
|
||||
MetricName::try_new(None, vec![]).unwrap(),
|
||||
TimeAxis::Explicit(ScalarBuffer::from(vec![1_i64, 2])),
|
||||
fields,
|
||||
)
|
||||
.unwrap();
|
||||
drop(cloned);
|
||||
assert_eq!(drops.load(Ordering::SeqCst), 0);
|
||||
drop(series);
|
||||
assert_eq!(drops.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn series_enforces_grid_policy_and_matches_grid_spec_by_value() {
|
||||
let spec = GridSpec::try_new(100, 10, 3).unwrap();
|
||||
let schema_grid = Arc::new(EvalGrid::try_new(spec).unwrap());
|
||||
let equal_grid = Arc::new(EvalGrid::try_new(spec).unwrap());
|
||||
let range = GridRange::try_new(equal_grid, 1, 1).unwrap();
|
||||
let fields = SeriesFields::one(Samples::from_float(Float64Array::from(vec![1.0])));
|
||||
let metric = MetricName::try_new(None, vec![]).unwrap();
|
||||
let grid_schema = SeriesSchema {
|
||||
fields: Arc::from([FieldSpec {
|
||||
name: Arc::from("value"),
|
||||
}]),
|
||||
eval_grid: Some(schema_grid.clone()),
|
||||
};
|
||||
|
||||
assert!(
|
||||
Series::try_new(
|
||||
&grid_schema,
|
||||
metric.clone(),
|
||||
TimeAxis::Grid(range),
|
||||
fields.clone()
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
// Timestamp 110 overlaps, but a different grid specification is not interchangeable.
|
||||
let different_grid =
|
||||
Arc::new(EvalGrid::try_new(GridSpec::try_new(110, 5, 3).unwrap()).unwrap());
|
||||
assert!(
|
||||
Series::try_new(
|
||||
&grid_schema,
|
||||
metric.clone(),
|
||||
TimeAxis::Grid(GridRange::try_new(different_grid, 0, 1).unwrap()),
|
||||
fields.clone(),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
|
||||
let explicit_schema = SeriesSchema {
|
||||
fields: grid_schema.fields.clone(),
|
||||
eval_grid: None,
|
||||
};
|
||||
assert!(
|
||||
Series::try_new(
|
||||
&explicit_schema,
|
||||
metric.clone(),
|
||||
TimeAxis::Explicit(ScalarBuffer::from(vec![110])),
|
||||
fields.clone(),
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
assert!(
|
||||
Series::try_new(
|
||||
&explicit_schema,
|
||||
metric.clone(),
|
||||
TimeAxis::Grid(GridRange::try_new(schema_grid.clone(), 0, 1).unwrap()),
|
||||
fields.clone(),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
Series::try_new(
|
||||
&grid_schema,
|
||||
metric.clone(),
|
||||
TimeAxis::Explicit(ScalarBuffer::from(vec![110])),
|
||||
fields.clone(),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
for (different_spec, range_start) in [
|
||||
(GridSpec::try_new(90, 10, 3).unwrap(), 1),
|
||||
(GridSpec::try_new(100, 5, 3).unwrap(), 2),
|
||||
(GridSpec::try_new(100, 10, 4).unwrap(), 1),
|
||||
] {
|
||||
let other_grid = Arc::new(EvalGrid::try_new(different_spec).unwrap());
|
||||
assert!(
|
||||
Series::try_new(
|
||||
&grid_schema,
|
||||
metric.clone(),
|
||||
TimeAxis::Grid(GridRange::try_new(other_grid, range_start, 1).unwrap()),
|
||||
fields.clone(),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
let empty_fields = SeriesFields::one(Samples::from_float(Float64Array::new_null(0)));
|
||||
assert!(
|
||||
Series::try_new(
|
||||
&grid_schema,
|
||||
metric.clone(),
|
||||
TimeAxis::Grid(GridRange::try_new(schema_grid, 3, 0).unwrap()),
|
||||
empty_fields.clone(),
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
assert!(
|
||||
Series::try_new(
|
||||
&explicit_schema,
|
||||
metric.clone(),
|
||||
TimeAxis::Explicit(ScalarBuffer::from(Vec::<i64>::new())),
|
||||
empty_fields.clone(),
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
assert!(
|
||||
Series::try_new(
|
||||
&grid_schema,
|
||||
metric.clone(),
|
||||
TimeAxis::Explicit(ScalarBuffer::from(Vec::<i64>::new())),
|
||||
empty_fields.clone(),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
Series::try_new(
|
||||
&explicit_schema,
|
||||
metric,
|
||||
TimeAxis::Grid(
|
||||
GridRange::try_new(grid_schema.eval_grid.as_ref().unwrap().clone(), 3, 0).unwrap()
|
||||
),
|
||||
empty_fields,
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn short_multifield_series_uses_its_grid_range_and_checks_explicit_chunk_order() {
|
||||
let grid = Arc::new(EvalGrid::try_new(GridSpec::try_new(0, 1, 2017).unwrap()).unwrap());
|
||||
let float = Samples::from_float(Float64Array::from(
|
||||
(0..20)
|
||||
.map(|i| {
|
||||
if i == 4 {
|
||||
None
|
||||
} else if i == 7 {
|
||||
Some(f64::NAN)
|
||||
} else {
|
||||
Some(i as f64)
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>(),
|
||||
));
|
||||
let histogram_table = batch(2.0, 3.0, &[4.0]);
|
||||
let histogram = Samples::try_from_histogram(
|
||||
UInt32Array::from(vec![Some(0); 20]),
|
||||
histogram_table.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
let fields = SeriesFields::try_from_vec(vec![float, histogram]).unwrap();
|
||||
let schema = SeriesSchema {
|
||||
fields: Arc::from([
|
||||
FieldSpec {
|
||||
name: Arc::from("float"),
|
||||
},
|
||||
FieldSpec {
|
||||
name: Arc::from("histogram"),
|
||||
},
|
||||
]),
|
||||
eval_grid: Some(grid.clone()),
|
||||
};
|
||||
let series = Series::try_new(
|
||||
&schema,
|
||||
MetricName::try_new(None, vec![]).unwrap(),
|
||||
TimeAxis::Grid(GridRange::try_new(grid, 800, 20).unwrap()),
|
||||
fields.clone(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(series.time.len(), 20);
|
||||
assert!(matches!(
|
||||
series.fields.get(0).unwrap().sample(4),
|
||||
SampleRef::Missing
|
||||
));
|
||||
assert!(matches!(series.fields.get(0).unwrap().sample(7), SampleRef::Float(v) if v.is_nan()));
|
||||
assert!(matches!(
|
||||
series.fields.get(1).unwrap().sample(19),
|
||||
SampleRef::Histogram(_)
|
||||
));
|
||||
assert!(Arc::ptr_eq(
|
||||
match series.fields.get(1).unwrap().dense().unwrap() {
|
||||
DenseSamplesRef::Histogram(samples) => samples.batch(),
|
||||
_ => unreachable!(),
|
||||
},
|
||||
&histogram_table,
|
||||
));
|
||||
|
||||
let explicit_schema = SeriesSchema {
|
||||
fields: schema.fields.clone(),
|
||||
eval_grid: None,
|
||||
};
|
||||
let explicit_fields = SeriesFields::try_from_vec(vec![
|
||||
Samples::from_float(Float64Array::new_null(4)),
|
||||
Samples::try_from_histogram(UInt32Array::from(vec![None; 4]), histogram_table).unwrap(),
|
||||
])
|
||||
.unwrap();
|
||||
let out_of_order = TimeAxis::ExplicitChunks(Arc::from([
|
||||
ScalarBuffer::from(vec![1_i64, 3]),
|
||||
ScalarBuffer::from(Vec::<i64>::new()),
|
||||
ScalarBuffer::from(vec![2_i64, 4]),
|
||||
]));
|
||||
assert!(
|
||||
Series::try_new(
|
||||
&explicit_schema,
|
||||
MetricName::try_new(None, vec![]).unwrap(),
|
||||
out_of_order,
|
||||
explicit_fields,
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
let ordered_chunks = TimeAxis::ExplicitChunks(Arc::from([
|
||||
ScalarBuffer::from(vec![1_i64, 3]),
|
||||
ScalarBuffer::from(Vec::<i64>::new()),
|
||||
ScalarBuffer::from(vec![5_i64]),
|
||||
]));
|
||||
let ordered_fields = SeriesFields::try_from_vec(vec![
|
||||
Samples::from_float(Float64Array::from(vec![1.0, 2.0, 3.0])),
|
||||
Samples::from_float(Float64Array::from(vec![4.0, 5.0, 6.0])),
|
||||
])
|
||||
.unwrap();
|
||||
assert!(
|
||||
Series::try_new(
|
||||
&explicit_schema,
|
||||
MetricName::try_new(None, vec![]).unwrap(),
|
||||
ordered_chunks.clone(),
|
||||
ordered_fields.clone(),
|
||||
)
|
||||
.is_ok()
|
||||
);
|
||||
let grid = Arc::new(EvalGrid::try_new(GridSpec::try_new(1, 2, 3).unwrap()).unwrap());
|
||||
let grid_schema = SeriesSchema {
|
||||
fields: explicit_schema.fields.clone(),
|
||||
eval_grid: Some(grid),
|
||||
};
|
||||
assert!(
|
||||
Series::try_new(
|
||||
&grid_schema,
|
||||
MetricName::try_new(None, vec![]).unwrap(),
|
||||
ordered_chunks,
|
||||
ordered_fields,
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
|
||||
let unordered_single = TimeAxis::Explicit(ScalarBuffer::from(vec![2_i64, 1]));
|
||||
assert!(
|
||||
Series::try_new(
|
||||
&SeriesSchema {
|
||||
fields: Arc::from([FieldSpec {
|
||||
name: Arc::from("value")
|
||||
}]),
|
||||
eval_grid: None
|
||||
},
|
||||
MetricName::try_new(None, vec![]).unwrap(),
|
||||
unordered_single,
|
||||
SeriesFields::one(Samples::from_float(Float64Array::from(vec![1.0, 2.0]))),
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user