fix(mito2): read raw series when columnar indexes are not ready

Keep native queries available for unflushed memtables and SSTs with incomplete columnar series or samples coverage by selecting the ordinary raw-series reader. Preserve structural validation and propagate artifact read errors instead of retrying them through legacy execution.

Cover all four incomplete-readiness states in public PromQL queries, both storage scan modes and ordering, complete prepared-source execution, and artifact error propagation.

Signed-off-by: discord9 <discord9@163.com>
This commit is contained in:
discord9
2026-10-24 23:01:49 +08:00
parent 5ba2d76f5a
commit 3b0587aa8d
4 changed files with 482 additions and 109 deletions
+62 -29
View File
@@ -256,10 +256,41 @@ impl SeriesScan {
.is_some_and(|context| context.format == crate::config::SeriesIndexFormat::Columnar)
}
/// Returns whether raw series come from the columnar prepared source.
///
/// Derived only from the pinned series index version and every configured partition range, so
/// preflight, the ordering guarantee, the partition reader and scanner-wide preparation agree.
/// `false` means the stored sources are read by the ordinary scan.
fn uses_prepared_columnar(&self) -> Result<bool, BoxedError> {
if !self.uses_columnar() {
return Ok(false);
}
let ranges = self
.properties
.partitions
.iter()
.flatten()
.copied()
.collect::<Vec<_>>();
prepared::validate_inputs(&self.stream_ctx, &ranges).map_err(BoxedError::new)
}
/// Returns the raw ordering guarantee for a scan whose columnar readiness is already known.
fn raw_ordering(&self, prepared_columnar: bool) -> SeriesOrdering {
if prepared_columnar {
return SeriesOrdering::KeyThenTime;
}
match self.mode {
SeriesScanMode::Legacy => SeriesOrdering::KeyThenTime,
SeriesScanMode::TwoPhase => SeriesOrdering::Contiguous,
}
}
/// Prepares the raw scan and reports whether its data comes from the columnar prepared source.
fn prepare_raw_scan(
&self,
label_source: &RawLabelSource,
) -> Result<Option<RawScanPreparation>, BoxedError> {
) -> Result<Option<(RawScanPreparation, bool)>, BoxedError> {
let input = &self.stream_ctx.input;
let metadata = input.mapper.metadata();
match label_source {
@@ -386,25 +417,19 @@ impl SeriesScan {
);
return Ok(None);
}
if self.uses_columnar() {
let ranges = self
.properties
.partitions
.iter()
.flatten()
.copied()
.collect::<Vec<_>>();
prepared::validate_inputs(&self.stream_ctx, &ranges).map_err(BoxedError::new)?;
}
// Structural eligibility errors still reject the raw path; ordinary scans cover sources
// without columnar artifacts, so missing coverage is not an error here.
let prepared_columnar = self.uses_prepared_columnar()?;
let schema = Arc::new(SeriesSchema {
fields: specs.into(),
eval_grid: None,
});
let typed_schema = input.mapper.input_arrow_schema(false);
let mut predicates = Vec::new();
let predicate = if self.uses_columnar() {
let predicate = if prepared_columnar {
input.predicate_group().predicate()
} else {
// The ordinary scan applies region predicates before this residual filtering.
input.predicate_group().predicate_without_region()
};
if let Some(predicate) = predicate {
@@ -452,11 +477,14 @@ impl SeriesScan {
predicates.push(physical);
}
}
Ok(Some(RawScanPreparation {
schema,
fields,
predicates,
}))
Ok(Some((
RawScanPreparation {
schema,
fields,
predicates,
},
prepared_columnar,
)))
}
fn supports_two_phase(input: &ScanInput) -> bool {
@@ -844,12 +872,11 @@ impl RegionScanner for SeriesScan {
}
fn raw_series_ordering(&self) -> SeriesOrdering {
if self.uses_columnar() {
return SeriesOrdering::KeyThenTime;
}
match self.mode {
SeriesScanMode::Legacy => SeriesOrdering::KeyThenTime,
SeriesScanMode::TwoPhase => SeriesOrdering::Contiguous,
// Readiness is fallible here, but every fallible entry point rejects the same structural
// errors before execution, so the `false` fallback below is never observed in a query.
match self.uses_prepared_columnar() {
Ok(prepared_columnar) => self.raw_ordering(prepared_columnar),
Err(_) => self.raw_ordering(false),
}
}
@@ -859,7 +886,7 @@ impl RegionScanner for SeriesScan {
) -> Result<Option<Arc<SeriesSchema>>, BoxedError> {
Ok(self
.prepare_raw_scan(label_source)?
.map(|preparation| preparation.schema))
.map(|(preparation, _)| preparation.schema))
}
fn scan_series_partition(
@@ -869,10 +896,10 @@ impl RegionScanner for SeriesScan {
partition: usize,
label_source: &RawLabelSource,
) -> Result<Option<SendableRawSeriesStream>, BoxedError> {
let Some(preparation) = self.prepare_raw_scan(label_source)? else {
let Some((preparation, prepared_columnar)) = self.prepare_raw_scan(label_source)? else {
return Ok(None);
};
if self.uses_columnar() {
if prepared_columnar {
let partitions = self.properties.num_partitions().max(1);
if partition >= partitions {
return Err(BoxedError::new(
@@ -897,10 +924,12 @@ impl RegionScanner for SeriesScan {
return Ok(Some(raw::from_stream(
schema,
labels,
SeriesOrdering::KeyThenTime,
self.raw_ordering(prepared_columnar),
Box::pin(stream),
)));
}
// Stored sources without columnar artifacts (memtables, unindexed SSTs) are read by the
// ordinary scan, so the raw path stays available.
let metrics = new_partition_metrics(
&self.stream_ctx,
ctx.explain_verbose,
@@ -911,7 +940,7 @@ impl RegionScanner for SeriesScan {
let batches = self
.scan_batch_in_partition(ctx, partition, metrics.clone(), metrics_set)
.map_err(BoxedError::new)?;
let ordering = self.raw_series_ordering();
let ordering = self.raw_ordering(prepared_columnar);
Ok(Some(raw::make_stream(
batches,
preparation.schema,
@@ -957,9 +986,13 @@ impl RegionScanner for SeriesScan {
&self.metrics_list,
);
Box::pin(async move {
let Some(preparation) = preparation? else {
let Some((preparation, prepared_columnar)) = preparation? else {
return Ok(None);
};
if !prepared_columnar {
// Known unsupported coverage: the caller keeps the ordinary scan instead.
return Ok(None);
}
state
.prepare(stream_ctx, ranges, labels, preparation, metrics)
.await
+29 -18
View File
@@ -137,20 +137,30 @@ fn same_labels(left: &RawLabelSource, right: &RawLabelSource) -> bool {
}
/// Validates the pinned assignments and coverage without opening artifacts.
pub(crate) fn validate_inputs(context: &StreamContext, ranges: &[PartitionRange]) -> Result<()> {
collect_inputs(context, ranges).map(|_| ())
///
/// Returns whether every assigned source can be read from columnar artifacts. `false` is not an
/// error: the caller scans those sources with the ordinary reader instead.
pub(crate) fn validate_inputs(context: &StreamContext, ranges: &[PartitionRange]) -> Result<bool> {
collect_inputs(context, ranges).map(|inputs| inputs.is_some())
}
/// Collects the assigned SSTs and their covering columnar series indexes.
///
/// `None` means the assignments are structurally valid but not ready for columnar reads: an
/// assigned memtable, an SST without columnar series coverage, or an SST without columnar
/// samples. Structural corruption is still an error, and every assignment is validated before
/// either answer is returned, so a later invalid assignment is never masked by missing artifacts.
fn collect_inputs(
context: &StreamContext,
ranges: &[PartitionRange],
) -> Result<(Vec<usize>, HashSet<FileId>)> {
) -> Result<Option<(Vec<usize>, HashSet<FileId>)>> {
let input = &context.input;
let metadata = input.region_metadata();
let index = input
.series_index
.as_ref()
.ok_or_else(|| invalid("missing columnar index context"))?;
let mut ready = true;
let mut files = BTreeMap::<usize, HashSet<i64>>::new();
for range in ranges {
let range = context
@@ -159,7 +169,9 @@ fn collect_inputs(
.ok_or_else(|| invalid("invalid prepared scan range"))?;
for row_group in &range.row_group_indices {
if context.is_mem_range_index(*row_group) {
return Err(invalid("columnar prepared reads require flushed memtables"));
// Artifacts exist for flushed SSTs only.
ready = false;
continue;
}
if !context.is_file_range_index(*row_group) {
return Err(invalid(
@@ -203,24 +215,18 @@ fn collect_inputs(
}
}
if !covered {
return Err(invalid(format!(
"SST {} has no columnar series coverage; run build_series_index()",
file.file_id()
)));
ready = false;
}
index
if index
.version
.samples
.get(&file.file_id().file_id())
.filter(|handle| handle.entry().artifact.format == ArtifactFormat::ColumnarSamples)
.ok_or_else(|| {
invalid(format!(
"SST {} has no columnar samples; run build_series_index()",
file.file_id()
))
})?;
.is_none_or(|handle| handle.entry().artifact.format != ArtifactFormat::ColumnarSamples)
{
ready = false;
}
}
Ok((file_indices, indexes))
Ok(ready.then_some((file_indices, indexes)))
}
struct Candidate {
@@ -254,7 +260,12 @@ impl ColumnarPreparedSource {
.series_index
.as_ref()
.ok_or_else(|| invalid("missing columnar index context"))?;
let (files, indexes) = collect_inputs(&context, &ranges)?;
let Some((files, indexes)) = collect_inputs(&context, &ranges)? else {
// Readiness was checked before preparation, so this is a real capability change.
return Err(invalid(
"columnar series coverage changed after series preparation",
));
};
let mut samples = Vec::<Arc<dyn SamplesFormatReader>>::new();
for file_idx in files {
let file = &input.files[file_idx];
+218 -62
View File
@@ -25,13 +25,14 @@ use datafusion::execution::TaskContext;
use datafusion::physical_plan::{ExecutionPlan, metrics::ExecutionPlanMetricsSet};
use datafusion_expr::{col, lit};
use datatypes::prelude::ConcreteDataType;
use datatypes::series::SeriesOrdering;
use datatypes::series::raw::RawSeriesChunk;
use datatypes::series::samples::SampleRef;
use object_store::ObjectStore;
use object_store::layers::mock::{self, MockLayerBuilder, oio};
use store_api::codec::PrimaryKeyEncoding;
use store_api::region_engine::{
BoundTag, PrepareRequest, QueryScanContext, RegionEngine, RegionScanner,
BoundTag, PartitionRange, PrepareRequest, QueryScanContext, RegionEngine, RegionScanner,
};
use store_api::region_request::{
RegionBuildIndexRequest, RegionDeleteRequest, RegionPutRequest, RegionRequest,
@@ -51,6 +52,10 @@ use crate::test_util::sst_util::{new_sparse_primary_key, sst_region_metadata_wit
use crate::test_util::{TestEnv, flush_region};
async fn fixture() -> (TestEnv, MitoEngine, MitoRegionRef) {
fixture_with(false).await
}
async fn fixture_with(experimental_series_scan_v2: bool) -> (TestEnv, MitoEngine, MitoRegionRef) {
let mut env = TestEnv::with_prefix("prepared-columnar").await;
let mut metadata = sst_region_metadata_with_encoding(PrimaryKeyEncoding::Sparse);
metadata
@@ -66,6 +71,7 @@ async fn fixture() -> (TestEnv, MitoEngine, MitoRegionRef) {
MitoConfig {
experimental_enable_series_index: true,
experimental_series_index_format: SelectedFormat::Columnar,
experimental_series_scan_v2,
experimental_series_index_maintenance_interval: Duration::from_secs(3600),
min_compaction_interval: Duration::from_secs(3600),
..Default::default()
@@ -192,6 +198,41 @@ impl mock::Read for ReadRecorder {
}
}
/// Fails reads of the selected artifact paths so tests can prove that object-store failures
/// surface as errors instead of being classified as not-ready fallbacks.
struct FailingReader {
inner: oio::Reader,
fail: bool,
}
impl mock::Read for FailingReader {
async fn read(
&self,
range: mock::BytesRange,
) -> object_store::Result<(mock::RpRead, object_store::Buffer)> {
if self.fail {
return Err(injected_failure());
}
self.inner.read(range).await
}
async fn open(
&self,
range: mock::BytesRange,
) -> object_store::Result<(mock::RpRead, Box<dyn oio::ReadStreamDyn>)> {
if self.fail {
return Err(injected_failure());
}
self.inner.open(range).await
}
}
fn injected_failure() -> mock::Error {
mock::Error::new(
mock::ErrorKind::Unexpected,
"injected series manifest read failure",
)
}
fn scanner(region: &MitoRegionRef, store: ObjectStore) -> SeriesScan {
scanner_with_format(region, store, SelectedFormat::Columnar)
}
@@ -269,6 +310,35 @@ fn rows(chunks: &[RawSeriesChunk]) -> Vec<(Vec<u8>, i64, f64)> {
.collect()
}
fn samples(chunks: &[RawSeriesChunk]) -> Vec<(i64, f64)> {
rows(chunks)
.into_iter()
.map(|(_, time, value)| (time, value))
.collect()
}
/// Reads the raw path of every partition, checks each stream declares `ordering`, and returns the
/// samples ordered by time so a check does not depend on how range grouping spread a memtable.
async fn all_raw_samples(
scanner: &dyn RegionScanner,
ctx: &QueryScanContext,
metrics: &ExecutionPlanMetricsSet,
binding: &RawLabelSource,
ordering: SeriesOrdering,
) -> Vec<(i64, f64)> {
let mut collected = Vec::new();
for partition in 0..scanner.properties().num_partitions().max(1) {
let stream = scanner
.scan_series_partition(ctx, metrics, partition, binding)
.unwrap()
.unwrap();
assert_eq!(ordering, stream.ordering());
collected.extend(samples(&stream.try_collect::<Vec<_>>().await.unwrap()));
}
collected.sort_unstable_by_key(|(time, _)| *time);
collected
}
#[tokio::test]
async fn prepared_sources_share_discovery_and_read_independently() {
let (_env, engine, region) = fixture().await;
@@ -386,89 +456,133 @@ async fn prepared_sources_share_discovery_and_read_independently() {
engine.stop().await.unwrap();
}
#[rstest::rstest]
#[case::legacy(false)]
#[case::two_phase(true)]
#[tokio::test]
async fn native_scan_reports_incomplete_coverage_and_uses_prepared_source() {
let (_env, engine, region) = fixture().await;
async fn native_scan_falls_back_to_raw_sources_until_columnar_coverage(#[case] v2: bool) {
let (_env, engine, region) = fixture_with(v2).await;
let request = ScanRequest {
distribution: Some(TimeSeriesDistribution::PerSeries),
..Default::default()
};
// Three files are below the automatic index threshold.
let ctx = QueryScanContext::default();
let metrics = ExecutionPlanMetricsSet::new();
// The three flushed SSTs only have columnar artifacts after `build`.
let fallback_ordering = if v2 {
SeriesOrdering::Contiguous
} else {
SeriesOrdering::KeyThenTime
};
let raw_scanner = engine
.handle_query(region.region_id, request.clone())
.await
.unwrap();
let binding = labels(raw_scanner.metadata());
assert!(raw_scanner.raw_series_schema(&binding).is_err());
assert!(
raw_scanner
.scan_series_partition(
&QueryScanContext::default(),
&ExecutionPlanMetricsSet::new(),
0,
&binding,
)
.is_err()
assert_eq!(fallback_ordering, raw_scanner.raw_series_ordering());
assert!(raw_scanner.raw_series_schema(&binding).unwrap().is_some());
assert_eq!(
vec![(1000, 1.0), (2000, 1.0)],
all_raw_samples(
raw_scanner.as_ref(),
&ctx,
&metrics,
&binding,
fallback_ordering
)
.await
);
// Known unready coverage selects the ordinary scan instead of a prepared source.
assert!(
raw_scanner
.prepare_series(
&QueryScanContext::default(),
&ExecutionPlanMetricsSet::new(),
&binding
)
.prepare_series(&ctx, &metrics, &binding)
.await
.is_err()
.unwrap()
.is_none()
);
build(&engine, &region).await;
let raw_scanner = engine
let mut raw_scanner = engine
.handle_query(region.region_id, request.clone())
.await
.unwrap();
// Move every assigned range into a new last partition: readiness, the ordering guarantee and
// the reader must all follow the new assignment, not the pre-prepare partition list.
let assigned = raw_scanner
.properties()
.partitions
.iter()
.flatten()
.copied()
.collect::<Vec<_>>();
raw_scanner
.prepare(PrepareRequest::default().with_ranges(vec![vec![], assigned]))
.unwrap();
assert_eq!(2, raw_scanner.properties().num_partitions());
let binding = labels(raw_scanner.metadata());
assert_eq!(
SeriesOrdering::KeyThenTime,
raw_scanner.raw_series_ordering()
);
let source = raw_scanner
.prepare_series(&ctx, &metrics, &binding)
.await
.unwrap()
.unwrap();
assert_eq!(1, source.num_series());
// The first partition owns no range and the second owns all of them.
assert_eq!(
vec![(1000, 1.0), (2000, 1.0)],
all_raw_samples(
raw_scanner.as_ref(),
&ctx,
&metrics,
&binding,
SeriesOrdering::KeyThenTime
)
.await
);
let plan = Arc::new(RegionScanExec::new(raw_scanner, request.clone(), None).unwrap())
.try_to_series_scan(binding.clone())
.unwrap()
.unwrap();
let frame = plan.execute(0, Arc::new(TaskContext::default())).unwrap();
let chunks = unpack_raw_series(frame, Some(plan.series_schema().clone()))
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
assert_eq!(SeriesOrdering::KeyThenTime, plan.raw_series_ordering());
let frame = plan.execute(1, Arc::new(TaskContext::default())).unwrap();
let unpacked = unpack_raw_series(frame, Some(plan.series_schema().clone())).unwrap();
// The packed frame declares the same guarantee the plan advertises.
assert_eq!(plan.raw_series_ordering(), unpacked.ordering());
assert_eq!(
vec![1000, 2000],
rows(&chunks).iter().map(|(_, t, _)| *t).collect::<Vec<_>>()
vec![(1000, 1.0), (2000, 1.0)],
samples(&unpacked.try_collect::<Vec<_>>().await.unwrap())
);
// An assigned memtable is known unready, so the raw path keeps the ordinary scan and must
// still return the new row.
write(&engine, &region, 0, 3000, Some(4.0)).await;
let raw_scanner = engine
.handle_query(region.region_id, request)
.await
.unwrap();
let binding = labels(raw_scanner.metadata());
assert!(raw_scanner.raw_series_schema(&binding).is_err());
assert_eq!(fallback_ordering, raw_scanner.raw_series_ordering());
assert!(raw_scanner.raw_series_schema(&binding).unwrap().is_some());
assert!(
raw_scanner
.scan_series_partition(
&QueryScanContext::default(),
&ExecutionPlanMetricsSet::new(),
0,
&binding,
)
.is_err()
);
assert!(
raw_scanner
.prepare_series(
&QueryScanContext::default(),
&ExecutionPlanMetricsSet::new(),
&binding
)
.prepare_series(&ctx, &metrics, &binding)
.await
.err()
.unwrap()
.output_msg()
.contains("memtables")
.is_none()
);
assert_eq!(
vec![(1000, 1.0), (2000, 1.0), (3000, 4.0)],
all_raw_samples(
raw_scanner.as_ref(),
&ctx,
&metrics,
&binding,
fallback_ordering
)
.await
);
engine.stop().await.unwrap();
}
@@ -1046,7 +1160,8 @@ async fn preflight_rejects_unsupported_assignments_without_artifact_io() {
};
let mut context = make_context();
let ranges = context.partition_ranges();
validate_inputs(&context, &ranges).unwrap();
// Complete columnar coverage is the only ready answer.
assert!(validate_inputs(&context, &ranges).unwrap());
context.ranges[ranges[0].identifier].row_group_indices = vec![RowGroupIndex {
index: context.input.num_files(),
row_group_index: -1,
@@ -1083,30 +1198,30 @@ async fn preflight_rejects_unsupported_assignments_without_artifact_io() {
.contains("foreign SST")
);
// Missing columnar samples are a not-ready answer, not an error.
let mut context = make_context();
let index = context.input.series_index.as_mut().unwrap();
index.version = Arc::new(SeriesIndexVersion {
series_indexes: index.version.series_indexes.clone(),
..Default::default()
});
assert!(!validate_inputs(&context, &ranges).unwrap());
// Invalid assignments remain errors even when sample coverage is incomplete.
let mut invalid = ranges.clone();
invalid.push(PartitionRange {
identifier: ranges[0].identifier + ranges.len(),
..ranges[0]
});
assert!(
validate_inputs(&context, &ranges)
validate_inputs(&context, &invalid)
.unwrap_err()
.to_string()
.contains("no columnar samples")
.contains("invalid prepared scan range")
);
let scan = SeriesScan::new(context.input, false);
let binding = labels(scan.metadata());
assert!(scan.raw_series_schema(&binding).is_err());
assert!(
scan.scan_series_partition(
&QueryScanContext::default(),
&ExecutionPlanMetricsSet::new(),
0,
&binding
)
.is_err()
);
// The raw path stays available for the ordinary scan and preparation is skipped.
assert!(scan.raw_series_schema(&binding).unwrap().is_some());
assert!(
scan.prepare_series(
&QueryScanContext::default(),
@@ -1114,8 +1229,49 @@ async fn preflight_rejects_unsupported_assignments_without_artifact_io() {
&binding
)
.await
.is_err()
.unwrap()
.is_none()
);
assert!(reads.lock().unwrap().is_empty());
engine.stop().await.unwrap();
}
#[tokio::test]
async fn series_artifact_read_failure_propagates() {
let (_env, engine, region) = fixture().await;
build(&engine, &region).await;
// The published catalog is complete; the injected manifest read failure must still propagate.
let store = region.series_index_store.clone().unwrap().layer(
MockLayerBuilder::default()
.reader_factory(Arc::new(|path, _, inner| {
Box::new(FailingReader {
inner,
fail: path.contains("/series/") && path.ends_with("/manifest.json"),
})
}))
.build()
.unwrap(),
);
let context = StreamContext::seq_scan_ctx(scan_input(
&region,
store,
SelectedFormat::Columnar,
(0..region.version().metadata.column_metadatas.len()).collect(),
));
let ranges = context.partition_ranges();
assert!(validate_inputs(&context, &ranges).unwrap());
let scan = SeriesScan::new(context.input, false);
let binding = labels(scan.metadata());
assert!(scan.raw_series_schema(&binding).unwrap().is_some());
let error = scan
.prepare_series(
&QueryScanContext::default(),
&ExecutionPlanMetricsSet::new(),
&binding,
)
.await
.err()
.expect("a failed artifact read must not report a not-ready source");
assert!(error.output_msg().contains("injected"), "{error}");
engine.stop().await.unwrap();
}
+173
View File
@@ -2586,3 +2586,176 @@ async fn native_promql_columnar_fused_series_matches_legacy() {
assert_eq!(native_rows, legacy_rows, "same rows on both routes");
assert!(observer.fused().is_none(), "the legacy route must not fuse");
}
/// No published series index: the columnar `Float64` selector must answer from the raw memtable
/// and from an unindexed SST instead of failing to plan.
#[rstest]
#[case::memtable("memtable", false)]
#[case::unindexed_sst("unindexed_sst", true)]
#[tokio::test(flavor = "multi_thread")]
async fn native_promql_columnar_unindexed_float64_matches_legacy(
#[case] stage: &str,
#[case] flush: bool,
) {
common_telemetry::init_default_ut_logging();
const PHYSICAL: &str = "native_columnar_unindexed_phy";
const TABLE: &str = "native_columnar_unindexed";
let mito_config = mito2::config::MitoConfig {
experimental_enable_series_index: true,
experimental_series_index_format: mito2::config::SeriesIndexFormat::Columnar,
// No periodic build, so neither case ever gets an index.
experimental_series_index_maintenance_interval: Duration::from_secs(3600),
..Default::default()
};
let (standalone, observer) = native_test_instance(
GreptimeDbStandaloneBuilder::new(&format!("native_promql_columnar_{stage}"))
.with_mito_config(mito_config),
)
.await;
let instance = standalone.fe_instance().clone();
execute_all(
&instance,
&format!(
"CREATE TABLE {PHYSICAL} (ts TIMESTAMP(3) TIME INDEX, val DOUBLE) ENGINE=metric \
WITH ('physical_metric_table' = 'true', 'primary_key_encoding' = 'sparse'); \
CREATE TABLE {TABLE} (host STRING, ts TIMESTAMP(3) TIME INDEX, val DOUBLE, PRIMARY KEY (host)) \
ENGINE=metric WITH ('on_physical_table' = '{PHYSICAL}'); \
INSERT INTO {TABLE} (host, ts, val) VALUES ('a', 60000, 10), ('a', 120000, 30), \
('b', 60000, 20), ('b', 120000, 60);"
),
QueryContext::arc(),
)
.await;
if flush {
execute_all(
&instance,
&format!("ADMIN FLUSH_TABLE('{PHYSICAL}');"),
QueryContext::arc(),
)
.await;
}
// Both samples of each series fall in the 2m window: 10 + 30 and 20 + 60.
let expected = vec![
(120_000, "a".to_string(), 40.0),
(120_000, "b".to_string(), 80.0),
];
let sql = format!("TQL EVAL (120, 120, '1m') sum_over_time({TABLE}[2m])");
// Legacy off first, so a red native query is about the fused route.
observer.clear();
let legacy_rows = native_result_rows(native_tql_output(&instance, &sql, false).await).await;
assert_eq!(legacy_rows, expected, "{stage}: existing plan must answer");
assert!(observer.fused().is_none(), "{stage}: legacy must not fuse");
observer.clear();
let native_rows = native_result_rows(native_tql_output(&instance, &sql, true).await).await;
// Drained first, so the counter below describes execution, not planning; `take` requires the
// fused node to have run.
let plan = observer.take();
assert_eq!(native_rows, expected, "{stage}: fused route must answer");
let output_values = plan
.metrics()
.expect("fused plan metrics")
.iter()
.filter_map(|metric| match metric.value() {
datafusion::physical_plan::metrics::MetricValue::Count { name, count }
if name.as_ref() == "native.root.output_values" =>
{
Some(count.value())
}
_ => None,
})
.sum::<usize>();
assert_eq!(output_values, 2, "{stage}: two values, one per series");
assert_eq!(
native_rows, legacy_rows,
"{stage}: same rows on both routes"
);
}
/// An index over the first SST only: the newer rows still in the memtable, then flushed into a
/// second unindexed SST, must answer the same sums.
#[rstest]
#[case::partial_memtable("partial_memtable", false)]
#[case::partial_sst("partial_sst", true)]
#[tokio::test(flavor = "multi_thread")]
async fn native_promql_columnar_partial_index_float64_matches_legacy(
#[case] stage: &str,
#[case] flush: bool,
) {
common_telemetry::init_default_ut_logging();
const PHYSICAL: &str = "native_columnar_partial_phy";
const TABLE: &str = "native_columnar_partial";
let mito_config = mito2::config::MitoConfig {
experimental_enable_series_index: true,
experimental_series_index_format: mito2::config::SeriesIndexFormat::Columnar,
// Slow the periodic pass, so only the manual build below publishes an index.
experimental_series_index_maintenance_interval: Duration::from_secs(3600),
..Default::default()
};
let (standalone, observer) = native_test_instance(
GreptimeDbStandaloneBuilder::new(&format!("native_promql_columnar_{stage}"))
.with_mito_config(mito_config),
)
.await;
let instance = standalone.fe_instance().clone();
// The first samples are flushed and indexed before the second ones are inserted, so the index
// covers one SST while the later rows start out unflushed.
execute_all(
&instance,
&format!(
"CREATE TABLE {PHYSICAL} (ts TIMESTAMP(3) TIME INDEX, val DOUBLE) ENGINE=metric \
WITH ('physical_metric_table' = 'true', 'primary_key_encoding' = 'sparse'); \
CREATE TABLE {TABLE} (host STRING, ts TIMESTAMP(3) TIME INDEX, val DOUBLE, PRIMARY KEY (host)) \
ENGINE=metric WITH ('on_physical_table' = '{PHYSICAL}'); \
INSERT INTO {TABLE} (host, ts, val) VALUES ('a', 60000, 10), ('b', 60000, 20); \
ADMIN FLUSH_TABLE('{PHYSICAL}'); \
ADMIN BUILD_SERIES_INDEX('{PHYSICAL}'); \
INSERT INTO {TABLE} (host, ts, val) VALUES ('a', 120000, 30), ('b', 120000, 60);"
),
QueryContext::arc(),
)
.await;
if flush {
// The new SST stays unindexed: the build above is not repeated.
execute_all(
&instance,
&format!("ADMIN FLUSH_TABLE('{PHYSICAL}');"),
QueryContext::arc(),
)
.await;
}
// Both samples of each series fall in the 2m window: 10 + 30 and 20 + 60.
let expected = vec![
(120_000, "a".to_string(), 40.0),
(120_000, "b".to_string(), 80.0),
];
let sql = format!("TQL EVAL (120, 120, '1m') sum_over_time({TABLE}[2m])");
// Legacy off first, so a red native query is about the fused route.
observer.clear();
let legacy_rows = native_result_rows(native_tql_output(&instance, &sql, false).await).await;
assert_eq!(legacy_rows, expected, "{stage}: existing plan must answer");
assert!(observer.fused().is_none(), "{stage}: legacy must not fuse");
observer.clear();
let native_rows = native_result_rows(native_tql_output(&instance, &sql, true).await).await;
// Drained first, so the counter below describes execution, not planning; `take` requires the
// fused node to have run.
let plan = observer.take();
assert_eq!(native_rows, expected, "{stage}: fused route must answer");
let output_values = plan
.metrics()
.expect("fused plan metrics")
.iter()
.filter_map(|metric| match metric.value() {
datafusion::physical_plan::metrics::MetricValue::Count { name, count }
if name.as_ref() == "native.root.output_values" =>
{
Some(count.value())
}
_ => None,
})
.sum::<usize>();
assert_eq!(output_values, 2, "{stage}: two values, one per series");
assert_eq!(
native_rows, legacy_rows,
"{stage}: same rows on both routes"
);
}