diff --git a/.github/scripts/query-regression-run.py b/.github/scripts/query-regression-run.py index d293aa3e0c..268cb0bae6 100644 --- a/.github/scripts/query-regression-run.py +++ b/.github/scripts/query-regression-run.py @@ -38,6 +38,7 @@ DEFAULT_CASES = [ "tests/perf/query_cases/prom_remote_write_mixed_every/case.toml", "tests/perf/query_cases/prom_remote_write_integer_counter/case.toml", "tests/perf/query_cases/promql_range_boundary/case.toml", + "tests/perf/query_cases/promql_instant_last_row_9034/case.toml", ] HEAVY_CASES = [ diff --git a/src/cmd/src/bin/query_perf_fixture/direct_sst.rs b/src/cmd/src/bin/query_perf_fixture/direct_sst.rs index 41b70a9690..746ecbd76e 100644 --- a/src/cmd/src/bin/query_perf_fixture/direct_sst.rs +++ b/src/cmd/src/bin/query_perf_fixture/direct_sst.rs @@ -32,6 +32,7 @@ use mito_codec::row_converter::{DensePrimaryKeyCodec, PrimaryKeyCodecExt, SortFi use mito2::access_layer::{FilePathProvider, Metrics, WriteType}; use mito2::config::IndexConfig; use mito2::manifest::action::{RegionCheckpoint, RegionManifest, RemovedFilesRecord}; +use mito2::memtable::sort_primary_key_record_batch; use mito2::read::FlatSource; use mito2::sst::file::{FileMeta, RegionFileId}; use mito2::sst::index::{Indexer, IndexerBuilder}; @@ -282,8 +283,10 @@ fn generate_record_batch( columns.push(Arc::new(pk_builder.finish())); columns.push(Arc::new(UInt64Array::from_value(sequence, rows))); columns.push(Arc::new(UInt8Array::from_value(OpType::Put as u8, rows))); - RecordBatch::try_new(flat_schema, columns) - .expect("generated fixture columns should match flat SST Arrow schema") + let batch = RecordBatch::try_new(flat_schema, columns) + .expect("generated fixture columns should match flat SST Arrow schema"); + sort_primary_key_record_batch(&batch) + .expect("generated fixture batch should sort by primary key, timestamp, and sequence") } fn file_meta_from_sst_info( @@ -532,3 +535,204 @@ pub(super) async fn run_direct_sst(args: DirectArgs) { fs::write(out_dir.join("summary.json"), serde_json::to_vec_pretty(&serde_json::json!({ "case": case_name, "seed": seed, "table_index": table_index, "table": table.name, "database": table.database, "region_id": region_id.as_u64(), "table_dir": table_dir, "region_dir": region_dir, "sst_format": format!("{format:?}"), "sst_count": scenario.layout.sst_count, "rows_per_sst": scenario.layout.rows_per_sst, "row_group_size": scenario.layout.row_group_size, "total_rows": scenario.layout.sst_count * scenario.layout.rows_per_sst, "checkpoint_path": checkpoint_path, "files_jsonl_path": files_jsonl_path, "readback_validated": false, "metadata_source": "synthetic" })).expect("failed to serialize fixture summary")).expect("failed to write fixture summary.json"); println!("Done. wrote {} SST file entries", manifest.files.len()); } + +#[cfg(test)] +mod tests { + use std::num::NonZeroUsize; + + use datatypes::arrow::array::{BinaryArray, DictionaryArray, StringArray}; + + use super::*; + + type LogicalRow = (Vec, i64, u64, String, String, f64); + + fn string_values(batch: &RecordBatch, column: usize) -> Vec { + let dictionary = batch + .column(column) + .as_any() + .downcast_ref::>() + .expect("fixture tag column should be a string dictionary"); + let values = dictionary + .values() + .as_any() + .downcast_ref::() + .expect("fixture tag dictionary should contain strings"); + dictionary + .keys() + .values() + .iter() + .map(|key| values.value(*key as usize).to_string()) + .collect() + } + + fn encoded_primary_keys(batch: &RecordBatch) -> Vec> { + let dictionary = batch + .column(batch.num_columns() - 3) + .as_any() + .downcast_ref::>() + .expect("fixture primary key should be a binary dictionary"); + let values = dictionary + .values() + .as_any() + .downcast_ref::() + .expect("fixture primary key dictionary should contain binary keys"); + dictionary + .keys() + .values() + .iter() + .map(|key| values.value(*key as usize).to_vec()) + .collect() + } + + #[test] + fn direct_sst_generator_sorts_all_layouts_and_timestamp_units() { + for (series_layout, timestamp_type) in [ + ("timestamp_major", "TIMESTAMP(9)"), + ("timestamp_major", "TIMESTAMP(3)"), + ("round_robin", "TIMESTAMP(9)"), + ("round_robin", "TIMESTAMP(3)"), + ("per_sst", "TIMESTAMP(9)"), + ("per_sst", "TIMESTAMP(3)"), + ] { + let mut case: CaseFile = toml::from_str(include_str!( + "../../../../../tests/perf/query_cases/promql_instant_last_row_9034/case.toml" + )) + .expect("existing query perf case should parse"); + let scenario = match &mut case.scenario { + Scenario::DirectReadableSst(scenario) => scenario, + _ => unreachable!("included case is a direct SST case"), + }; + scenario.layout.series_layout = series_layout.to_string(); + scenario.layout.series_count = NonZeroUsize::new(12).expect("12 is nonzero"); + scenario.layout.rows_per_sst = 36; + scenario.layout.sst_count = 1; + scenario.layout.row_group_size = 12; + let table = &mut scenario.tables[0]; + let time_index = table.time_index.clone(); + table + .columns + .iter_mut() + .find(|column| column.name == time_index) + .expect("fixture case has time index column") + .ty = timestamp_type.to_string(); + + let sst_idx = 10; + let sequence = 1010; + let metadata = Arc::new(build_region_metadata(table, RegionId::from(42))); + let batch = + generate_record_batch(table, &metadata, &scenario.layout, sst_idx, sequence); + assert_eq!(36, batch.num_rows(), "{series_layout}/{timestamp_type}"); + + let base_row = sst_idx * scenario.layout.rows_per_sst; + let host = table + .columns + .iter() + .find(|column| column.name == "host") + .expect("fixture case has host tag"); + let instance = table + .columns + .iter() + .find(|column| column.name == "instance") + .expect("fixture case has instance tag"); + let value = table + .columns + .iter() + .find(|column| column.name == "value") + .expect("fixture case has value field"); + let (min, max) = match value.distribution.as_ref() { + Some(Distribution::DeterministicWave { min, max }) => (*min, *max), + _ => unreachable!("included case has deterministic wave values"), + }; + let mut expected = (0..scenario.layout.rows_per_sst) + .map(|row| { + let series = series_for_row(&scenario.layout, sst_idx, base_row, row); + let mut tags = HashMap::new(); + tags.insert(host.name.clone(), tag_value(host, series)); + tags.insert(instance.name.clone(), tag_value(instance, series)); + let timestamp = timestamp_for_row(&scenario.layout, base_row, row); + ( + encode_dense_primary_key(table, &tags), + if timestamp_type == "TIMESTAMP(3)" { + timestamp / 1_000_000 + } else { + timestamp + }, + sequence, + tag_value(host, series), + tag_value(instance, series), + wave_value(min, max, base_row + row), + ) + }) + .collect::>(); + expected.sort_by(|left, right| { + left.0 + .cmp(&right.0) + .then_with(|| left.1.cmp(&right.1)) + .then_with(|| right.2.cmp(&left.2)) + }); + + let timestamps = if timestamp_type == "TIMESTAMP(9)" { + batch + .column(batch.num_columns() - 4) + .as_any() + .downcast_ref::() + .expect("nanosecond fixture timestamp") + .values() + .to_vec() + } else { + batch + .column(batch.num_columns() - 4) + .as_any() + .downcast_ref::() + .expect("millisecond fixture timestamp") + .values() + .to_vec() + }; + let sequences = batch + .column(batch.num_columns() - 2) + .as_any() + .downcast_ref::() + .expect("fixture sequence column") + .values() + .to_vec(); + let hosts = string_values(&batch, 0); + let instances = string_values(&batch, 1); + let values = batch + .column(2) + .as_any() + .downcast_ref::() + .expect("fixture value column") + .values() + .to_vec(); + let actual = encoded_primary_keys(&batch) + .into_iter() + .zip(timestamps) + .zip(sequences) + .zip(hosts) + .zip(instances) + .zip(values) + .map( + |(((((primary_key, timestamp), sequence), host), instance), value)| { + (primary_key, timestamp, sequence, host, instance, value) + }, + ) + .collect::>(); + + assert_eq!(expected, actual, "{series_layout}/{timestamp_type}"); + for pair in actual.windows(2) { + assert!( + pair[0].0 < pair[1].0 + || (pair[0].0 == pair[1].0 && pair[0].1 < pair[1].1) + || (pair[0].0 == pair[1].0 + && pair[0].1 == pair[1].1 + && pair[0].2 >= pair[1].2), + "rows must be encoded-PK ascending, timestamp ascending, sequence descending: {series_layout}/{timestamp_type}" + ); + } + if series_layout != "per_sst" { + assert!(actual.iter().any(|row| row.3 == "host2")); + assert!(actual.iter().any(|row| row.3 == "host10")); + } + } + } +} diff --git a/src/cmd/src/datanode/scanbench.rs b/src/cmd/src/datanode/scanbench.rs index 61d34e9e13..82a3460e4b 100644 --- a/src/cmd/src/datanode/scanbench.rs +++ b/src/cmd/src/datanode/scanbench.rs @@ -389,7 +389,7 @@ fn resolve_series_row_selector( scan_config: &ScanConfig, ) -> error::Result> { match scan_config.series_row_selector.as_deref() { - Some("last_row") => Ok(Some(TimeSeriesRowSelector::LastRow)), + Some("last_row") => Ok(Some(TimeSeriesRowSelector::LastRow { after_merge: false })), Some(other) => Err(error::IllegalConfigSnafu { msg: format!("Unknown series_row_selector '{other}'"), } diff --git a/src/metric-engine/src/engine/read.rs b/src/metric-engine/src/engine/read.rs index ee086184c2..d1afce5e22 100644 --- a/src/metric-engine/src/engine/read.rs +++ b/src/metric-engine/src/engine/read.rs @@ -318,7 +318,7 @@ mod test { use mito2::config::MitoConfig; use store_api::region_engine::{PrepareRequest, QueryScanContext}; use store_api::region_request::{RegionFlushRequest, RegionPutRequest, RegionRequest}; - use store_api::storage::TimeSeriesDistribution; + use store_api::storage::{TimeSeriesDistribution, TimeSeriesRowSelector}; use super::*; use crate::config::EngineConfig; @@ -429,6 +429,7 @@ mod test { let scan_req = ScanRequest { projection, filters: vec![], + series_row_selector: Some(TimeSeriesRowSelector::LastRow { after_merge: true }), ..Default::default() }; @@ -444,6 +445,10 @@ mod test { &[11, 10, 9, 8, 0, 1, 4] ); assert_eq!(scan_req.filters.len(), 1); + assert_eq!( + scan_req.series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: true }) + ); assert_eq!( scan_req.filters[0], logical_expr::col(DATA_SCHEMA_TABLE_ID_COLUMN_NAME) diff --git a/src/mito2/src/cache.rs b/src/mito2/src/cache.rs index 52a40ebbe1..2e98ef3345 100644 --- a/src/mito2/src/cache.rs +++ b/src/mito2/src/cache.rs @@ -2621,7 +2621,7 @@ mod tests { let key = SelectorResultKey { file_id, row_group_idx: 0, - selector: TimeSeriesRowSelector::LastRow, + selector: TimeSeriesRowSelector::LastRow { after_merge: false }, }; assert!(cache.get_selector_result(&key).is_none()); let result = Arc::new(SelectorResultValue::new( diff --git a/src/mito2/src/engine/row_selector_test.rs b/src/mito2/src/engine/row_selector_test.rs index 7c66a5562f..7fd937e85d 100644 --- a/src/mito2/src/engine/row_selector_test.rs +++ b/src/mito2/src/engine/row_selector_test.rs @@ -12,20 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -use api::v1::Rows; +use api::v1::value::ValueData; +use api::v1::{Rows, Value}; use common_base::readable_size::ReadableSize; +use common_query::prometheus::PROMETHEUS_STALE_NAN_BITS; use common_recordbatch::RecordBatches; use datafusion_common::ScalarValue; use datafusion_expr::{col, lit}; +use datatypes::arrow::array::{Float64Array, StringArray, TimestampMillisecondArray}; use store_api::region_engine::RegionEngine; use store_api::region_request::RegionRequest; use store_api::storage::{RegionId, ScanRequest, TimeSeriesRowSelector}; use crate::config::MitoConfig; use crate::engine::MitoEngine; +use crate::read::scan_region::Scanner; use crate::test_util::batch_util::sort_batches_and_print; use crate::test_util::{ - CreateRequestBuilder, TestEnv, build_rows_for_key, flush_region, put_rows, rows_schema, + CreateRequestBuilder, TestEnv, build_delete_rows_for_key, build_rows_for_key, delete_rows, + flush_region, put_rows, rows_schema, }; async fn test_last_row(append_mode: bool, flat_format: bool) { @@ -98,7 +103,7 @@ async fn test_last_row(append_mode: bool, flat_format: bool) { .scanner( region_id, ScanRequest { - series_row_selector: Some(TimeSeriesRowSelector::LastRow), + series_row_selector: Some(TimeSeriesRowSelector::LastRow { after_merge: false }), ..Default::default() }, ) @@ -121,7 +126,7 @@ async fn scan_last_row( region_id, ScanRequest { filters, - series_row_selector: Some(TimeSeriesRowSelector::LastRow), + series_row_selector: Some(TimeSeriesRowSelector::LastRow { after_merge: false }), ..Default::default() }, ) @@ -193,6 +198,276 @@ const LAST_ROW_AT_TEN: &str = "\ | a | 10.0 | 1970-01-01T00:00:10 | +-------+---------+---------------------+"; +async fn new_merge_last_row_engine( + flat_format: bool, +) -> ( + TestEnv, + MitoEngine, + RegionId, + Vec, + Vec, +) { + let mut env = TestEnv::new().await; + let engine = env.create_engine(MitoConfig::default()).await; + let region_id = RegionId::new(1, 1); + env.get_schema_metadata_manager() + .register_region_table_info( + region_id.table_id(), + "test_table", + "test_catalog", + "test_schema", + None, + env.get_kv_backend(), + ) + .await; + let sst_format = if flat_format { "flat" } else { "primary_key" }; + let request = CreateRequestBuilder::new() + .insert_option("sst_format", sst_format) + .build(); + let schema = rows_schema(&request); + let delete_schema = crate::test_util::delete_rows_schema(&request); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + (env, engine, region_id, schema, delete_schema) +} + +fn value_row(key: &str, value: f64, timestamp: i64) -> api::v1::Row { + api::v1::Row { + values: vec![ + Value { + value_data: Some(ValueData::StringValue(key.to_string())), + }, + Value { + value_data: Some(ValueData::F64Value(value)), + }, + Value { + value_data: Some(ValueData::TimestampMillisecondValue(timestamp)), + }, + ], + } +} + +async fn last_row_scanner(engine: &MitoEngine, region_id: RegionId) -> Scanner { + engine + .scanner( + region_id, + ScanRequest { + series_row_selector: Some(TimeSeriesRowSelector::LastRow { after_merge: false }), + ..Default::default() + }, + ) + .await + .unwrap() +} + +async fn scan_last_row_batches(scanner: &Scanner) -> RecordBatches { + RecordBatches::try_collect(scanner.scan().await.unwrap()) + .await + .unwrap() +} + +fn last_row_values(batches: &RecordBatches) -> Vec<(String, f64, i64)> { + let mut rows = Vec::new(); + for batch in batches.iter() { + let batch = batch.df_record_batch(); + let tags = batch + .column_by_name("tag_0") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let fields = batch + .column_by_name("field_0") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let timestamps = batch + .column_by_name("ts") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + for index in 0..batch.num_rows() { + rows.push(( + tags.value(index).to_string(), + fields.value(index), + timestamps.value(index), + )); + } + } + rows.sort_by(|left, right| left.0.cmp(&right.0)); + rows +} + +#[tokio::test] +async fn test_last_row_merge_deduplicates_same_timestamp_across_ssts() { + for flat_format in [false, true] { + let (_env, engine, region_id, schema, _delete_schema) = + new_merge_last_row_engine(flat_format).await; + + put_rows( + &engine, + region_id, + Rows { + schema: schema.clone(), + rows: vec![value_row("a", 1.0, 1000)], + }, + ) + .await; + flush_region(&engine, region_id, None).await; + put_rows( + &engine, + region_id, + Rows { + schema, + rows: vec![value_row("a", 2.0, 1000)], + }, + ) + .await; + flush_region(&engine, region_id, None).await; + + let scanner = last_row_scanner(&engine, region_id).await; + assert_eq!(2, scanner.num_files()); + assert_eq!(0, scanner.num_memtables()); + assert_eq!( + vec![("a".to_string(), 2.0, 1000)], + last_row_values(&scan_last_row_batches(&scanner).await) + ); + } +} + +#[tokio::test] +async fn test_last_row_returns_stale_marker_and_preserves_ordinary_nan() { + for flat_format in [false, true] { + let (_env, engine, region_id, schema, _delete_schema) = + new_merge_last_row_engine(flat_format).await; + put_rows( + &engine, + region_id, + Rows { + schema: schema.clone(), + rows: vec![value_row("a", 1.0, 1000), value_row("b", f64::NAN, 1000)], + }, + ) + .await; + flush_region(&engine, region_id, None).await; + put_rows( + &engine, + region_id, + Rows { + schema, + rows: vec![value_row( + "a", + f64::from_bits(PROMETHEUS_STALE_NAN_BITS), + 1000, + )], + }, + ) + .await; + flush_region(&engine, region_id, None).await; + + let scanner = last_row_scanner(&engine, region_id).await; + assert_eq!(2, scanner.num_files()); + assert_eq!(0, scanner.num_memtables()); + let values = last_row_values(&scan_last_row_batches(&scanner).await); + assert_eq!(2, values.len(), "unexpected LastRow values: {values:?}"); + assert_eq!("a", values[0].0); + assert_eq!(PROMETHEUS_STALE_NAN_BITS, values[0].1.to_bits()); + assert_eq!(1000, values[0].2); + assert_eq!("b", values[1].0); + assert!(values[1].1.is_nan()); + assert_ne!(PROMETHEUS_STALE_NAN_BITS, values[1].1.to_bits()); + } +} + +#[tokio::test] +async fn test_last_row_delete_wins_across_ssts() { + for flat_format in [false, true] { + let (_env, engine, region_id, schema, delete_schema) = + new_merge_last_row_engine(flat_format).await; + put_rows( + &engine, + region_id, + Rows { + schema, + rows: vec![value_row("a", 1.0, 1000)], + }, + ) + .await; + flush_region(&engine, region_id, None).await; + delete_rows( + &engine, + region_id, + Rows { + schema: delete_schema, + rows: build_delete_rows_for_key("a", 1, 2), + }, + ) + .await; + flush_region(&engine, region_id, None).await; + + let scanner = last_row_scanner(&engine, region_id).await; + assert_eq!(2, scanner.num_files()); + assert_eq!(0, scanner.num_memtables()); + assert!(last_row_values(&scan_last_row_batches(&scanner).await).is_empty()); + } +} + +#[tokio::test] +async fn test_last_row_returns_older_put_after_newest_delete_across_ssts() { + for flat_format in [false, true] { + let (_env, engine, region_id, schema, delete_schema) = + new_merge_last_row_engine(flat_format).await; + put_rows( + &engine, + region_id, + Rows { + schema, + rows: vec![value_row("a", 1.0, 1000), value_row("a", 2.0, 2000)], + }, + ) + .await; + flush_region(&engine, region_id, None).await; + delete_rows( + &engine, + region_id, + Rows { + schema: delete_schema, + rows: build_delete_rows_for_key("a", 2, 3), + }, + ) + .await; + flush_region(&engine, region_id, None).await; + + let scanner = engine + .scanner( + region_id, + ScanRequest { + series_row_selector: Some(TimeSeriesRowSelector::LastRow { after_merge: true }), + ..Default::default() + }, + ) + .await + .unwrap(); + assert_eq!(2, scanner.num_files()); + assert_eq!(0, scanner.num_memtables()); + assert_eq!( + vec![("a".to_string(), 1.0, 1000)], + last_row_values(&scan_last_row_batches(&scanner).await) + ); + } +} + +#[tokio::test] +async fn test_last_row_empty_region_returns_empty() { + let (_env, engine, region_id, _schema, _delete_schema) = new_merge_last_row_engine(false).await; + let scanner = last_row_scanner(&engine, region_id).await; + assert!(last_row_values(&scan_last_row_batches(&scanner).await).is_empty()); +} + #[tokio::test] async fn test_last_row_append_mode_disabled() { test_last_row(false, false).await; diff --git a/src/mito2/src/engine/scan_test.rs b/src/mito2/src/engine/scan_test.rs index df5035cdc5..a9d138367a 100644 --- a/src/mito2/src/engine/scan_test.rs +++ b/src/mito2/src/engine/scan_test.rs @@ -2197,7 +2197,7 @@ async fn test_exact_sequence_read_with_last_row_selector_keeps_in_range_rows() { memtable_min_sequence: Some(0), memtable_max_sequence: Some(1), exact_sequence_range: true, - series_row_selector: Some(TimeSeriesRowSelector::LastRow), + series_row_selector: Some(TimeSeriesRowSelector::LastRow { after_merge: false }), ..Default::default() }) .await @@ -2213,7 +2213,7 @@ async fn test_exact_sequence_read_with_last_row_selector_keeps_in_range_rows() { | series | 2.0 | 1970-01-01T00:00:02 | +--------+---------+---------------------+", scan(ScanRequest { - series_row_selector: Some(TimeSeriesRowSelector::LastRow), + series_row_selector: Some(TimeSeriesRowSelector::LastRow { after_merge: false }), ..Default::default() }) .await @@ -2583,7 +2583,9 @@ async fn test_non_preserve_compaction_sequence_collision_with_format(flat_format .scan_to_stream( region_id, ScanRequest { - series_row_selector: Some(TimeSeriesRowSelector::LastRow), + series_row_selector: Some(TimeSeriesRowSelector::LastRow { + after_merge: false, + }), ..Default::default() }, ) diff --git a/src/mito2/src/read/last_row.rs b/src/mito2/src/read/last_row.rs index c2316525f7..c526bfcaa8 100644 --- a/src/mito2/src/read/last_row.rs +++ b/src/mito2/src/read/last_row.rs @@ -144,7 +144,7 @@ impl FlatRowGroupLastRowCachedReader { let key = SelectorResultKey { file_id, row_group_idx, - selector: TimeSeriesRowSelector::LastRow, + selector: TimeSeriesRowSelector::LastRow { after_merge: false }, }; if let Some(value) = cache_strategy.get_selector_result(&key) { diff --git a/src/mito2/src/read/range_cache.rs b/src/mito2/src/read/range_cache.rs index 16994b53f7..ddf8c3cc35 100644 --- a/src/mito2/src/read/range_cache.rs +++ b/src/mito2/src/read/range_cache.rs @@ -1441,7 +1441,27 @@ mod tests { } #[test] - fn normalizes_and_clears_time_filters() { + fn selector_after_merge_changes_fingerprint() { + let ordinary = test_scan_fingerprint( + vec!["k0 = 'foo'".to_string()], + vec![], + Some(TimeSeriesRowSelector::LastRow { after_merge: false }), + true, + 0, + ); + let after_merge = test_scan_fingerprint( + vec!["k0 = 'foo'".to_string()], + vec![], + Some(TimeSeriesRowSelector::LastRow { after_merge: true }), + true, + 0, + ); + + assert_ne!(ordinary, after_merge); + } + + #[test] + fn true_selector_after_merge_is_preserved_by_fingerprint_transforms() { let normalized = test_scan_fingerprint(vec!["k0 = 'foo'".to_string()], vec![], None, true, 0); @@ -1450,18 +1470,32 @@ mod tests { let fingerprint = test_scan_fingerprint( vec!["k0 = 'foo'".to_string()], vec!["ts >= 1000".to_string()], - Some(TimeSeriesRowSelector::LastRow), + Some(TimeSeriesRowSelector::LastRow { after_merge: true }), true, 7, ); let reset = fingerprint.without_time_filters(); + let candidate = fingerprint.for_candidate_series(); + let series_data = fingerprint.for_series_data(SeriesRange::new(0, 1).unwrap()); assert_eq!(reset.read_columns(), fingerprint.read_columns()); assert_eq!(reset.read_column_types(), fingerprint.read_column_types()); assert_eq!(reset.filters(), fingerprint.filters()); assert!(reset.time_filters().is_empty()); assert_eq!(reset.series_row_selector, fingerprint.series_row_selector); + assert_eq!( + fingerprint.series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: true }) + ); + assert_eq!( + candidate.series_row_selector, + fingerprint.series_row_selector + ); + assert_eq!( + series_data.series_row_selector, + fingerprint.series_row_selector + ); assert_eq!(reset.append_mode, fingerprint.append_mode); assert_eq!(reset.filter_deleted, fingerprint.filter_deleted); assert_eq!(reset.merge_mode, fingerprint.merge_mode); diff --git a/src/mito2/src/read/scan_region.rs b/src/mito2/src/read/scan_region.rs index c8ddf20aa7..56c53698e7 100644 --- a/src/mito2/src/read/scan_region.rs +++ b/src/mito2/src/read/scan_region.rs @@ -2503,7 +2503,7 @@ mod tests { ) .await .with_distribution(Some(TimeSeriesDistribution::PerSeries)) - .with_series_row_selector(Some(TimeSeriesRowSelector::LastRow)) + .with_series_row_selector(Some(TimeSeriesRowSelector::LastRow { after_merge: true })) .with_merge_mode(MergeMode::LastNonNull) .with_filter_deleted(false) .build(); @@ -2528,7 +2528,7 @@ mod tests { col("v0").gt(lit(1)).to_string(), ], time_filters: vec![col("ts").gt_eq(ts_lit(1000)).to_string()], - series_row_selector: Some(TimeSeriesRowSelector::LastRow), + series_row_selector: Some(TimeSeriesRowSelector::LastRow { after_merge: true }), append_mode: false, filter_deleted: false, merge_mode: MergeMode::LastNonNull, @@ -2537,6 +2537,10 @@ mod tests { } .build(); assert_eq!(&expected, fingerprint); + assert_eq!( + input.series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: true }) + ); } #[tokio::test] diff --git a/src/mito2/src/read/scan_util.rs b/src/mito2/src/read/scan_util.rs index afd3de10d9..a67dfae158 100644 --- a/src/mito2/src/read/scan_util.rs +++ b/src/mito2/src/read/scan_util.rs @@ -30,7 +30,7 @@ use futures::Stream; use prometheus::IntGauge; use smallvec::SmallVec; use snafu::ResultExt; -use store_api::storage::{RegionId, SequenceRange}; +use store_api::storage::{RegionId, SequenceRange, TimeSeriesRowSelector}; use crate::error::{ComputeArrowSnafu, Result}; use crate::memtable::MemScanMetrics; @@ -1571,18 +1571,13 @@ pub fn build_flat_file_range_scan_stream( let build_reader_start = Instant::now(); let Some(mut reader) = range .flat_reader( - // In exact `sequence_range` mode the row-group-level LastRow - // shortcut would reduce each row group to its last-timestamp - // row *before* the row-level sequence filter runs, silently - // dropping in-range rows (a series with seq 1 at t1 and seq 2 - // at t2 under `(0, 1]` keeps only seq 2 and then filters it - // out). Bypass the shortcut so the final per-row selector - // (`FlatLastRowReader`, applied after source merging and the - // sequence filter) selects on the filtered rows instead. - if stream_ctx.input.sequence_range.is_some() { - None - } else { - stream_ctx.input.series_row_selector + match stream_ctx.input.series_row_selector { + Some(TimeSeriesRowSelector::LastRow { after_merge: false }) + if stream_ctx.input.sequence_range.is_none() => + { + stream_ctx.input.series_row_selector + } + _ => None, }, fetch_metrics.as_deref(), ) diff --git a/src/mito2/src/read/seq_scan.rs b/src/mito2/src/read/seq_scan.rs index 7253ba532f..5d59ddfce1 100644 --- a/src/mito2/src/read/seq_scan.rs +++ b/src/mito2/src/read/seq_scan.rs @@ -263,7 +263,7 @@ impl SeqScan { }; let reader = match &stream_ctx.input.series_row_selector { - Some(TimeSeriesRowSelector::LastRow) => { + Some(TimeSeriesRowSelector::LastRow { .. }) => { Box::pin(FlatLastRowReader::new(reader).into_stream()) as _ } None => reader, diff --git a/src/mito2/src/read/series_scan.rs b/src/mito2/src/read/series_scan.rs index 1a1369830f..12b301a823 100644 --- a/src/mito2/src/read/series_scan.rs +++ b/src/mito2/src/read/series_scan.rs @@ -180,7 +180,7 @@ impl SeriesScan { } fn supports_two_phase(input: &ScanInput) -> bool { - if !is_sparse_metric_metadata(input.region_metadata()) { + if input.sequence_range.is_some() || !is_sparse_metric_metadata(input.region_metadata()) { return false; } #[cfg(feature = "enterprise")] @@ -1214,6 +1214,39 @@ mod tests { use datatypes::arrow::record_batch::RecordBatch; use super::*; + use crate::read::flat_projection::FlatProjectionMapper; + use crate::read::scan_region::PredicateGroup; + use crate::test_util::scheduler_util::SchedulerEnv; + use crate::test_util::sst_util::sst_region_metadata_with_encoding; + + #[tokio::test] + async fn two_phase_eligibility_rejects_exact_sequence_range() { + let env = SchedulerEnv::new().await; + let metadata = Arc::new(sst_region_metadata_with_encoding( + store_api::codec::PrimaryKeyEncoding::Sparse, + )); + let predicate = PredicateGroup::default(); + + let eligible = ScanInput::builder( + env.access_layer.clone(), + FlatProjectionMapper::new(&metadata, [0]).unwrap(), + ) + .with_predicate(predicate.clone()) + .build(); + assert!(SeriesScan::supports_two_phase(&eligible)); + + let exact_sequence = ScanInput::builder( + env.access_layer.clone(), + FlatProjectionMapper::new(&metadata, [0]).unwrap(), + ) + .with_predicate(predicate) + .with_sequence_range(Some(store_api::storage::SequenceRange::GtLtEq { + min: 1, + max: 2, + })) + .build(); + assert!(!SeriesScan::supports_two_phase(&exact_sequence)); + } #[test] fn candidate_distributor_stops_after_all_receivers_close() { diff --git a/src/mito2/src/sst/parquet/file_range.rs b/src/mito2/src/sst/parquet/file_range.rs index 20eaa81180..f55aa52416 100644 --- a/src/mito2/src/sst/parquet/file_range.rs +++ b/src/mito2/src/sst/parquet/file_range.rs @@ -202,10 +202,10 @@ impl FileRange { )) .await?; - let use_last_row_reader = if selector - .map(|s| s == TimeSeriesRowSelector::LastRow) - .unwrap_or(false) - { + let use_last_row_reader = if matches!( + selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: false }) + ) { // Only use LastRowReader if row group does not contain DELETE, all // rows are selected, and filters that still run after this reader // cannot change which row is last. Tag filters are safe because a diff --git a/src/promql/src/extension_plan/instant_manipulate.rs b/src/promql/src/extension_plan/instant_manipulate.rs index 4851916a01..25d62e6bee 100644 --- a/src/promql/src/extension_plan/instant_manipulate.rs +++ b/src/promql/src/extension_plan/instant_manipulate.rs @@ -231,6 +231,11 @@ impl InstantManipulate { "InstantManipulate" } + /// Returns whether this node evaluates a single timestamp rather than a range. + pub fn is_single_evaluation(&self) -> bool { + self.start == self.end + } + fn staleness_field_columns(&self) -> impl Iterator { let [field, companion] = mixed_sample_fields(self.field_column.as_deref()); [ diff --git a/src/promql/src/extension_plan/normalize.rs b/src/promql/src/extension_plan/normalize.rs index 573fa5fb0e..e3410be926 100644 --- a/src/promql/src/extension_plan/normalize.rs +++ b/src/promql/src/extension_plan/normalize.rs @@ -200,6 +200,11 @@ impl SeriesNormalize { "SeriesNormalize" } + /// Returns whether this plan removes Prometheus stale markers. + pub const fn filter_stale_markers(&self) -> bool { + self.filter_stale_markers + } + pub fn to_execution_plan(&self, exec_input: Arc) -> Arc { Arc::new(SeriesNormalizeExec { offset: self.offset, diff --git a/src/query/src/dummy_catalog.rs b/src/query/src/dummy_catalog.rs index e38813b3e6..3d1dfe5f37 100644 --- a/src/query/src/dummy_catalog.rs +++ b/src/query/src/dummy_catalog.rs @@ -286,6 +286,17 @@ impl DummyTableProvider { self.scan_request.lock().unwrap().series_row_selector = Some(selector); } + /// Clones this provider for one logical table-scan use-site. + /// + /// The optimizer may attach different hints to scans that share the same + /// catalog provider, so the per-scan request must not share its mutex. + pub fn clone_for_scan(&self) -> Self { + Self { + scan_request: Arc::new(Mutex::new(self.scan_request.lock().unwrap().clone())), + ..self.clone() + } + } + pub fn with_vector_search_hint(&self, hint: VectorSearchRequest) { self.scan_request.lock().unwrap().vector_search = Some(hint); } diff --git a/src/query/src/optimizer/scan_hint.rs b/src/query/src/optimizer/scan_hint.rs index 89e3afaed0..ce1009f881 100644 --- a/src/query/src/optimizer/scan_hint.rs +++ b/src/query/src/optimizer/scan_hint.rs @@ -18,12 +18,16 @@ use api::v1::SemanticType; use arrow_schema::SortOptions; use common_function::aggrs::aggr_wrapper::aggr_state_func_name; use common_recordbatch::OrderOption; +use common_recordbatch::filter::SimpleFilterEvaluator; +use common_time::timestamp::TimeUnit; use datafusion::datasource::DefaultTableSource; -use datafusion_common::tree_node::{Transformed, TreeNode, TreeNodeRecursion, TreeNodeVisitor}; +use datafusion_common::tree_node::{Transformed, TreeNodeRewriter}; use datafusion_common::{Column, Result}; use datafusion_expr::expr::Sort; use datafusion_expr::{Expr, LogicalPlan, utils}; use datafusion_optimizer::{OptimizerConfig, OptimizerRule}; +use datatypes::arrow::datatypes::{DataType, TimeUnit as ArrowTimeUnit}; +use promql::extension_plan::{InstantManipulate, SeriesDivide, SeriesNormalize}; use store_api::metric_engine_consts::DATA_SCHEMA_TSID_COLUMN_NAME; use store_api::storage::{TimeSeriesDistribution, TimeSeriesRowSelector}; @@ -59,65 +63,145 @@ impl OptimizerRule for ScanHintRule { impl ScanHintRule { fn optimize(plan: LogicalPlan) -> Result> { - let mut visitor = ScanHintVisitor::default(); - let _ = plan.visit(&mut visitor)?; - - if visitor.need_rewrite() { - plan.transform_down(&mut |plan| Self::set_hints(plan, &mut visitor)) - } else { - Ok(Transformed::no(plan)) - } + let mut rewriter = ScanHintRewriter::default(); + // The extension's input is included by DataFusion's normal TreeNode + // rewrite, so this is one scoped recursive walk (subquery expressions + // are included by the dedicated API as well). + plan.rewrite_with_subqueries(&mut rewriter) } fn set_hints( plan: LogicalPlan, - visitor: &mut ScanHintVisitor, + rewriter: &mut ScanHintRewriter, ) -> Result> { - match &plan { - LogicalPlan::TableScan(table_scan) => { - let mut transformed = false; - if let Some(source) = table_scan - .source - .as_any() - .downcast_ref::() - { - // The provider in the region server is [DummyTableProvider]. - if let Some(adapter) = source - .table_provider - .as_any() - .downcast_ref::() - { - // set order_hint - if let Some(order_expr) = &visitor.order_expr { - Self::set_order_hint(adapter, order_expr); - } + let LogicalPlan::TableScan(mut table_scan) = plan else { + return Ok(Transformed::no(plan)); + }; + let Some(source) = table_scan + .source + .as_any() + .downcast_ref::() + else { + return Ok(Transformed::no(LogicalPlan::TableScan(table_scan))); + }; + // The provider in the region server is [DummyTableProvider]. + let Some(original) = source + .table_provider + .as_any() + .downcast_ref::() + else { + return Ok(Transformed::no(LogicalPlan::TableScan(table_scan))); + }; - // set time series selector hint - if let Some((group_by_cols, order_by_col)) = &visitor.ts_row_selector { - Self::set_time_series_row_selector_hint( - adapter, - group_by_cols, - order_by_col, - ); - } + // Attached scan filters are checked below; residual Filter nodes are + // rejected by the single-evaluation path allowlist. + let filters_preserve_last_row = if rewriter.inside_single_evaluation { + Self::filters_preserve_last_row(&table_scan, original) + } else { + true + }; + let use_last_row = rewriter.inside_single_evaluation && filters_preserve_last_row; - #[cfg(feature = "vector_index")] - if let Some(vector_request) = visitor - .vector_search - .take_vector_request_from_dummy(adapter, &table_scan.table_name) - { - adapter.with_vector_search_hint(vector_request); - } - transformed = true; - } - } - if transformed { - Ok(Transformed::yes(plan)) - } else { - Ok(Transformed::no(plan)) - } + #[cfg(feature = "vector_index")] + let has_vector_hint = rewriter.vector_search.need_rewrite(); + #[cfg(not(feature = "vector_index"))] + let has_vector_hint = false; + let has_hint = rewriter.order_expr.is_some() + || rewriter.ts_row_selector.is_some() + || use_last_row + || has_vector_hint; + if !has_hint { + return Ok(Transformed::no(LogicalPlan::TableScan(table_scan))); + } + + // A provider can be used by several TableScan nodes. Fork its request + // for every hinted use-site before applying hints, rather than mutating + // the shared catalog provider. This keeps order/vector/legacy hints + // local as well as the new LastRow hint. + let adapter = original.clone_for_scan(); + Self::apply_hints(&adapter, rewriter, &table_scan, use_last_row); + if use_last_row { + // Apply the instant-derived hint after the aggregate hint. Both + // select LastRow today, and this ordering preserves the existing + // aggregate selector when the instant guard rejects a scan. + adapter.with_time_series_selector_hint(TimeSeriesRowSelector::LastRow { + after_merge: true, + }); + } + table_scan.source = + std::sync::Arc::new(DefaultTableSource::new(std::sync::Arc::new(adapter))); + Ok(Transformed::yes(LogicalPlan::TableScan(table_scan))) + } + + /// Checks whether attached scan predicates permit instant-derived LastRow selection. + /// + /// Selecting the newest row too early can discard an older matching sample if a + /// predicate later rejects that row. Only recognized tag/time predicates are + /// allowed: tags select whole series, and supported time predicates constrain + /// the scan window before row selection. Field or unrecognized predicates are + /// conservatively rejected. Finer-than-millisecond timestamps are also excluded + /// because instant evaluation can conflate distinct samples at that precision. + /// + /// This checks only attached predicates; the path allowlist separately rejects + /// residual Filter nodes between InstantManipulate and the scan. + fn filters_preserve_last_row( + table_scan: &datafusion_expr::logical_plan::TableScan, + provider: &DummyTableProvider, + ) -> bool { + let metadata = provider.region_metadata(); + // Instant evaluation is millisecond-based, so finer time units can + // conflate timestamps and must not use the LastRow hint. + if !matches!( + metadata.time_index_type().unit(), + TimeUnit::Second | TimeUnit::Millisecond + ) { + return false; + } + for filter in &table_scan.filters { + let Some(filter) = SimpleFilterEvaluator::try_new(filter) else { + return false; + }; + let Some(column_metadata) = metadata.column_by_name(filter.column_name()) else { + return false; + }; + if !matches!( + column_metadata.semantic_type, + SemanticType::Tag | SemanticType::Timestamp + ) { + return false; } - _ => Ok(Transformed::no(plan)), + } + true + } + + fn apply_hints( + adapter: &DummyTableProvider, + rewriter: &mut ScanHintRewriter, + table_scan: &datafusion_expr::logical_plan::TableScan, + use_last_row: bool, + ) { + #[cfg(not(feature = "vector_index"))] + let _ = (table_scan, use_last_row); + if let Some(order_expr) = &rewriter.order_expr { + Self::set_order_hint(adapter, order_expr); + } + if let Some((group_by_cols, order_by_col)) = &rewriter.ts_row_selector { + Self::set_time_series_row_selector_hint(adapter, group_by_cols, order_by_col); + } + #[cfg(feature = "vector_index")] + if use_last_row { + // LastRow and vector search are mutually exclusive for one scan: + // vector search would bypass the ordinary sort/limit path needed by + // the single-evaluation semantics. Still consume the queued hint so + // it cannot be applied to a later scan of the same table. + let _ = rewriter + .vector_search + .take_vector_request_from_dummy(adapter, &table_scan.table_name); + } else if let Some(vector_request) = rewriter + .vector_search + .take_vector_request_from_dummy(adapter, &table_scan.table_name) + { + adapter.with_vector_search_hint(vector_request); } } @@ -221,186 +305,222 @@ impl ScanHintRule { } if should_set_selector_hint { - adapter.with_time_series_selector_hint(TimeSeriesRowSelector::LastRow); + adapter.with_time_series_selector_hint(TimeSeriesRowSelector::LastRow { + after_merge: false, + }); } } } -/// Traverse and fetch hints. +/// Traverse and apply hints with state scoped to the current logical-plan path. +/// +/// Rewriting the scan while walking down the tree is important: the state then +/// describes the actual parent path of that scan, and a shared provider is forked +/// at that exact use-site. No traversal-order identity is involved. #[derive(Default)] -struct ScanHintVisitor { - /// The closest order requirement to the leaf node. +struct ScanHintRewriter { order_expr: Option>, - /// Row selection on time series distribution. - /// This field stores saved `group_by` columns when all aggregate functions are `last_value` - /// and the `order_by` column which should be time index. + order_stack: Vec>>, ts_row_selector: Option<(HashSet, Column)>, + ts_stack: Vec, Column)>>, + inside_single_evaluation: bool, + single_evaluation_stack: Vec, #[cfg(feature = "vector_index")] vector_search: VectorSearchState, } -impl TreeNodeVisitor<'_> for ScanHintVisitor { +impl TreeNodeRewriter for ScanHintRewriter { type Node = LogicalPlan; - fn f_down(&mut self, node: &Self::Node) -> Result { - #[cfg(feature = "vector_index")] - if let LogicalPlan::Limit(limit) = node { - // Track LIMIT so vector hint k can be derived within the same input chain. - self.vector_search.on_limit_enter(limit); - } + fn f_down(&mut self, node: LogicalPlan) -> Result> { + self.order_stack.push(self.order_expr.clone()); + self.ts_stack.push(self.ts_row_selector.clone()); + self.single_evaluation_stack + .push(self.inside_single_evaluation); - // Get order requirement from sort plan - if let LogicalPlan::Sort(sort) = node { + if let LogicalPlan::Sort(sort) = &node { self.order_expr = Some(sort.expr.clone()); - - #[cfg(feature = "vector_index")] - { - // Capture vector ORDER BY and TopK hints from sort nodes. - self.vector_search.on_sort_enter(sort); - } + } + if let LogicalPlan::Extension(extension) = &node + && let Some(instant) = extension.node.as_any().downcast_ref::() + { + self.inside_single_evaluation = instant.is_single_evaluation(); + } else if self.inside_single_evaluation { + // This allowlist is coupled to the controlled PromQL planner. It + // permits only nodes known to preserve the newest row per series; + // every other node is a sticky boundary until a nested instant + // extension establishes a new evaluation scope. + self.inside_single_evaluation = single_evaluation_node_allowed(&node); + } + if let LogicalPlan::Aggregate(aggregate) = &node { + self.ts_row_selector = Self::extract_last_value_selector(aggregate); } - // Get time series row selector from aggr plan - if let LogicalPlan::Aggregate(aggregate) = node { - let mut is_all_last_value = !aggregate.aggr_expr.is_empty(); - let mut order_by_expr = None; - for expr in &aggregate.aggr_expr { - // check function name - let Expr::AggregateFunction(func) = expr else { - is_all_last_value = false; - break; - }; - if (func.func.name() != "last_value" - && func.func.name() != aggr_state_func_name("last_value")) - || func.params.filter.is_some() - || func.params.distinct - { - is_all_last_value = false; - break; - } - // check order by requirement - let order_by = &func.params.order_by; - if let Some(first_order_by) = order_by.first() - && order_by.len() == 1 - { - if let Some(existing_order_by) = &order_by_expr { - if existing_order_by != first_order_by { - is_all_last_value = false; - break; - } - } else { - // only allow `order by xxx [ASC]`, xxx is a bare column reference so `last_value()` is the max - // value of the column. - if !first_order_by.asc || !matches!(&first_order_by.expr, Expr::Column(_)) { - is_all_last_value = false; - break; - } - order_by_expr = Some(first_order_by.clone()); - } - } - } - is_all_last_value &= order_by_expr.is_some(); - if is_all_last_value { - // make sure all the exprs are DIRECT `col` and collect them - let mut group_by_cols = HashSet::with_capacity(aggregate.group_expr.len()); - for expr in &aggregate.group_expr { - if let Expr::Column(col) = expr { - group_by_cols.insert(col.clone()); - } else { - is_all_last_value = false; - break; - } - } - // Safety: checked in the above loop - let order_by_expr = order_by_expr.unwrap(); - let Expr::Column(order_by_col) = order_by_expr.expr else { - unreachable!() - }; - if is_all_last_value { - self.ts_row_selector = Some((group_by_cols, order_by_col)); - } - } - } - - // Avoid carrying vector hints across branching inputs (join/subquery) to prevent - // pruning results before global ordering is applied. Only treat a subquery as a - // barrier when it contains non-inlineable operators. - let is_branching_for_ts = matches!( + let is_branching = matches!( node, LogicalPlan::Subquery(_) | LogicalPlan::SubqueryAlias(_) ) || node.inputs().len() > 1; - if is_branching_for_ts && self.ts_row_selector.is_some() { - // clean previous time series selector hint when encounter subqueries or join + if is_branching { self.ts_row_selector = None; } - #[cfg(feature = "vector_index")] - if is_branching_for_vector(node) { - self.vector_search.on_branching_enter(); - } - - if let LogicalPlan::Filter(filter) = node + if let LogicalPlan::Filter(filter) = &node && let Some(group_by_exprs) = &self.ts_row_selector { - let mut filter_referenced_cols = HashSet::default(); - utils::expr_to_columns(&filter.predicate, &mut filter_referenced_cols)?; - // ensure only group_by columns are used in filter - if !filter_referenced_cols.is_subset(&group_by_exprs.0) { + let mut referenced = HashSet::default(); + utils::expr_to_columns(&filter.predicate, &mut referenced)?; + if !referenced.is_subset(&group_by_exprs.0) { self.ts_row_selector = None; } } #[cfg(feature = "vector_index")] - if let LogicalPlan::Filter(filter) = node { - self.vector_search.on_filter_enter(&filter.predicate); + { + if let LogicalPlan::Limit(limit) = &node { + self.vector_search.on_limit_enter(limit); + } + if let LogicalPlan::Sort(sort) = &node { + self.vector_search.on_sort_enter(sort); + } + if is_branching_for_vector(&node) { + self.vector_search.on_branching_enter(); + } + if let LogicalPlan::Filter(filter) = &node { + self.vector_search.on_filter_enter(&filter.predicate); + } + if let LogicalPlan::TableScan(table_scan) = &node { + self.vector_search.on_table_scan(table_scan); + } } - #[cfg(feature = "vector_index")] - if let LogicalPlan::TableScan(table_scan) = node { - // Record vector hints at leaf scans after scope checks. - self.vector_search.on_table_scan(table_scan); - } - - Ok(TreeNodeRecursion::Continue) + ScanHintRule::set_hints(node, self) } - fn f_up(&mut self, _node: &Self::Node) -> Result { + fn f_up(&mut self, node: LogicalPlan) -> Result> { #[cfg(feature = "vector_index")] - match _node { - LogicalPlan::Limit(_) => { - self.vector_search.on_limit_exit(); + { + match &node { + LogicalPlan::Limit(_) => self.vector_search.on_limit_exit(), + LogicalPlan::Sort(_) => self.vector_search.on_sort_exit(), + LogicalPlan::Filter(_) => self.vector_search.on_filter_exit(), + LogicalPlan::Subquery(_) | LogicalPlan::SubqueryAlias(_) + if is_branching_for_vector(&node) => + { + self.vector_search.on_branching_exit() + } + _ if node.inputs().len() > 1 => self.vector_search.on_branching_exit(), + _ => {} } - LogicalPlan::Sort(_) => { - self.vector_search.on_sort_exit(); - } - LogicalPlan::Filter(_) => { - self.vector_search.on_filter_exit(); - } - LogicalPlan::Subquery(_) | LogicalPlan::SubqueryAlias(_) - if is_branching_for_vector(_node) => - { - self.vector_search.on_branching_exit(); - } - _ if _node.inputs().len() > 1 => { - self.vector_search.on_branching_exit(); - } - _ => {} } - - Ok(TreeNodeRecursion::Continue) + if let Some(previous) = self.order_stack.pop() { + self.order_expr = previous; + } + if let Some(previous) = self.ts_stack.pop() { + self.ts_row_selector = previous; + } + if let Some(previous) = self.single_evaluation_stack.pop() { + self.inside_single_evaluation = previous; + } + Ok(Transformed::no(node)) } } -impl ScanHintVisitor { - fn need_rewrite(&self) -> bool { - let base = self.order_expr.is_some() || self.ts_row_selector.is_some(); - #[cfg(feature = "vector_index")] - { - base || self.vector_search.need_rewrite() +/// Returns whether a plan node can occur on the scan path of a controlled +/// PromQL single evaluation without changing which row is newest per series. +fn single_evaluation_node_allowed(node: &LogicalPlan) -> bool { + match node { + LogicalPlan::TableScan(_) | LogicalPlan::SubqueryAlias(_) => true, + LogicalPlan::Sort(sort) => sort.fetch.is_none(), + LogicalPlan::Projection(projection) => projection + .expr + .iter() + .all(|expr| single_evaluation_projection_expr_allowed(expr, projection)), + LogicalPlan::Extension(extension) => { + let extension = extension.node.as_any(); + extension.is::() + || extension + .downcast_ref::() + .is_some_and(|normalize| !normalize.filter_stale_markers()) } - #[cfg(not(feature = "vector_index"))] - { - base + _ => false, + } +} + +/// This whitelist assumes the planner preserves time-index and series identity; +/// it is not a proof that an arbitrary plan does so. +fn single_evaluation_projection_expr_allowed( + expr: &Expr, + projection: &datafusion_expr::logical_plan::Projection, +) -> bool { + match expr { + Expr::Column(_) => true, + Expr::Alias(alias) => match alias.expr.as_ref() { + Expr::Column(column) => alias.name == column.name, + Expr::Cast(cast) => { + let Expr::Column(column) = cast.expr.as_ref() else { + return false; + }; + alias.name == column.name + && matches!( + cast.data_type, + DataType::Timestamp(ArrowTimeUnit::Millisecond, None) + ) + && matches!( + projection + .input + .schema() + .qualified_field_from_column(column), + Ok((_, field)) + if matches!(field.data_type(), DataType::Timestamp(ArrowTimeUnit::Second | ArrowTimeUnit::Millisecond, None)) + ) + } + _ => false, + }, + _ => false, + } +} + +impl ScanHintRewriter { + fn extract_last_value_selector( + aggregate: &datafusion_expr::logical_plan::Aggregate, + ) -> Option<(HashSet, Column)> { + let mut order_by_expr = None; + if aggregate.aggr_expr.is_empty() { + return None; } + for expr in &aggregate.aggr_expr { + let Expr::AggregateFunction(func) = expr else { + return None; + }; + if (func.func.name() != "last_value" + && func.func.name() != aggr_state_func_name("last_value")) + || func.params.filter.is_some() + || func.params.distinct + { + return None; + } + let order_by = &func.params.order_by; + if order_by.len() != 1 || !order_by[0].asc { + return None; + } + if let Some(existing) = &order_by_expr { + if existing != &order_by[0] { + return None; + } + } else { + order_by_expr = Some(order_by[0].clone()); + } + } + let Expr::Column(order_by_col) = order_by_expr?.expr else { + return None; + }; + let mut group_by_cols = HashSet::with_capacity(aggregate.group_expr.len()); + for expr in &aggregate.group_expr { + let Expr::Column(col) = expr else { + return None; + }; + group_by_cols.insert(col.clone()); + } + Some((group_by_cols, order_by_col)) } } @@ -443,54 +563,126 @@ fn has_non_inlineable_ops(plan: &LogicalPlan) -> bool { #[cfg(test)] mod test { + use std::collections::HashMap; use std::sync::Arc; use datafusion::functions_aggregate::first_last::last_value_udaf; - use datafusion_expr::expr::{AggregateFunction, AggregateFunctionParams}; - use datafusion_expr::{LogicalPlanBuilder, col}; + use datafusion::functions_aggregate::min_max::max_udaf; + use datafusion::functions_window::row_number::RowNumber; + use datafusion::logical_expr::expr::WindowFunction; + use datafusion::logical_expr::{WindowFrame, WindowFunctionDefinition}; + use datafusion::prelude::JoinType; + use datafusion_common::tree_node::TreeNodeRecursion; + use datafusion_expr::expr::{ + AggregateFunction, AggregateFunctionParams, Cast, WindowFunctionParams, + }; + use datafusion_expr::expr_fn::scalar_subquery; + use datafusion_expr::{Extension, LogicalPlan, LogicalPlanBuilder, col, lit}; use datafusion_optimizer::OptimizerContext; + use datatypes::arrow::datatypes::DataType; + use datatypes::data_type::ConcreteDataType; + use datatypes::schema::ColumnSchema; + use promql::extension_plan::RangeManipulate; + use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder}; use store_api::metric_engine_consts::DATA_SCHEMA_TSID_COLUMN_NAME; - use store_api::storage::RegionId; + use store_api::storage::{RegionId, TimeSeriesRowSelector}; use super::*; - use crate::optimizer::test_util::{mock_table_provider, mock_table_provider_with_tsid}; - #[test] - fn set_order_hint() { - let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); - let table_source = Arc::new(DefaultTableSource::new(provider.clone())); - let plan = LogicalPlanBuilder::scan("t", table_source, None) - .unwrap() - .sort(vec![col("ts").sort(true, false)]) - .unwrap() - .sort(vec![col("ts").sort(false, true)]) - .unwrap() - .build() - .unwrap(); - - let context = OptimizerContext::default(); - ScanHintRule.rewrite(plan, &context).unwrap(); - - // should read the first (with `.sort(true, false)`) sort option - let scan_req = provider.scan_request(); - assert_eq!( - OrderOption { - name: "ts".to_string(), - options: SortOptions { - descending: false, - nulls_first: false - } - }, - scan_req.output_ordering.as_ref().unwrap()[0] - ); + fn scan_requests(plan: &LogicalPlan) -> Vec { + scan_requests_with_names(plan) + .into_iter() + .map(|(_, request)| request) + .collect() } - #[test] - fn set_time_series_row_selector_hint() { - let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); - let table_source = Arc::new(DefaultTableSource::new(provider.clone())); - let plan = LogicalPlanBuilder::scan("t", table_source, None) - .unwrap() + fn scan_requests_with_names( + plan: &LogicalPlan, + ) -> Vec<(String, store_api::storage::ScanRequest)> { + let mut requests = Vec::new(); + plan.apply_with_subqueries(|node| { + if let LogicalPlan::TableScan(scan) = node + && let Some(source) = scan.source.as_any().downcast_ref::() + && let Some(provider) = source + .table_provider + .as_any() + .downcast_ref::() + { + requests.push((scan.table_name.to_string(), provider.scan_request())); + } + Ok(TreeNodeRecursion::Continue) + }) + .unwrap(); + requests + } + + fn instant_plan(provider: Arc, end: i64) -> LogicalPlan { + instant_plan_named(provider, "t", end) + } + + fn instant_plan_with_filters( + provider: Arc, + filters: Vec, + ) -> LogicalPlan { + let scan = scan_plan(provider, "t"); + let LogicalPlan::TableScan(mut scan) = scan else { + unreachable!(); + }; + scan.filters = filters; + LogicalPlan::Extension(Extension { + node: Arc::new(InstantManipulate::new( + 1000, + 1000, + 1000, + 1000, + "ts".to_string(), + vec![], + Some("v0".to_string()), + LogicalPlan::TableScan(scan), + )), + }) + } + + fn scan_plan(provider: Arc, table_name: &str) -> LogicalPlan { + LogicalPlanBuilder::scan( + table_name, + Arc::new(DefaultTableSource::new(provider)), + None, + ) + .unwrap() + .build() + .unwrap() + } + + fn mock_table_provider_with_timestamp( + region_id: RegionId, + timestamp_type: ConcreteDataType, + ) -> DummyTableProvider { + let mut builder = RegionMetadataBuilder::new(region_id); + builder + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new("k0", ConcreteDataType::string_datatype(), true), + semantic_type: SemanticType::Tag, + column_id: 1, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new("ts", timestamp_type, false), + semantic_type: SemanticType::Timestamp, + column_id: 2, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new("v0", ConcreteDataType::float64_datatype(), false), + semantic_type: SemanticType::Field, + column_id: 3, + }) + .primary_key(vec![1]); + let metadata = Arc::new(builder.build().unwrap()); + let engine = Arc::new(MetaRegionEngine::with_metadata(metadata.clone())); + DummyTableProvider::new(region_id, engine, metadata) + } + + fn last_value_aggregate(input: LogicalPlan) -> LogicalPlan { + LogicalPlanBuilder::from(input) .aggregate( vec![col("k0")], vec![Expr::AggregateFunction(AggregateFunction { @@ -510,13 +702,827 @@ mod test { ) .unwrap() .build() + .unwrap() + } + + fn single_evaluation(input: LogicalPlan) -> LogicalPlan { + LogicalPlan::Extension(Extension { + node: Arc::new(InstantManipulate::new( + 1000, + 1000, + 1000, + 1000, + "ts".to_string(), + vec![], + Some("v0".to_string()), + input, + )), + }) + } + + fn instant_with_expression_subquery( + outer_provider: Arc, + inner_plan: LogicalPlan, + outer_end: i64, + ) -> LogicalPlan { + let outer_scan = scan_plan(outer_provider, "outer"); + let input = LogicalPlanBuilder::from(outer_scan) + .project(vec![ + col("ts"), + col("v0"), + scalar_subquery(Arc::new(inner_plan)), + ]) + .unwrap() + .build() + .unwrap(); + LogicalPlan::Extension(Extension { + node: Arc::new(InstantManipulate::new( + 1000, + outer_end, + 1000, + 1000, + "ts".to_string(), + vec![], + Some("v0".to_string()), + input, + )), + }) + } + + fn instant_plan_named( + provider: Arc, + table_name: &str, + end: i64, + ) -> LogicalPlan { + let input = LogicalPlanBuilder::scan( + table_name, + Arc::new(DefaultTableSource::new(provider)), + None, + ) + .unwrap() + .build() + .unwrap(); + LogicalPlan::Extension(Extension { + node: Arc::new(InstantManipulate::new( + 1000, + end, + 1000, + 1000, + "ts".to_string(), + vec![], + Some("v0".to_string()), + input, + )), + }) + } + use crate::optimizer::test_util::{ + MetaRegionEngine, mock_table_provider, mock_table_provider_with_tsid, + }; + + #[test] + fn single_evaluation_sets_last_row_on_the_rewritten_scan() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let plan = instant_plan(provider.clone(), 1000); + let rewritten = ScanHintRule + .rewrite(plan, &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!( + scan_requests(&rewritten)[0].series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: true }) + ); + + assert_eq!(provider.scan_request().series_row_selector, None); + } + + #[test] + fn single_evaluation_limit_sort_does_not_set_last_row_below_limit() { + let mut selectors = Vec::new(); + for input in [ + // Instant(1000, lookback=1000) -> Limit(0, 1) -> Sort(ts ASC) -> Scan. + LogicalPlanBuilder::from(scan_plan( + Arc::new(mock_table_provider(RegionId::new(1, 1))), + "t", + )) + .sort(vec![col("ts").sort(true, false)]) + .unwrap() + .limit(0, Some(1)) + .unwrap() + .build() + .unwrap(), + // Sort.fetch is a limit embedded in the Sort node and has the same boundary. + LogicalPlanBuilder::from(scan_plan( + Arc::new(mock_table_provider(RegionId::new(1, 1))), + "t", + )) + .sort_with_limit(vec![col("ts").sort(true, false)], Some(1)) + .unwrap() + .build() + .unwrap(), + ] { + let rewritten = ScanHintRule + .rewrite(single_evaluation(input), &OptimizerContext::default()) + .unwrap() + .data; + selectors.push(scan_requests(&rewritten)[0].series_row_selector); + } + + // With samples (ts=100, v=10) and (ts=900, v=1), an unhinted ascending + // sort/limit pipeline returns 10. LastRow at the scan instead leaves only + // (900, 1) before the limit. A LastRow scan hint cannot cross either limit. + assert_eq!(selectors, vec![None, None]); + } + + #[test] + fn single_evaluation_allows_controlled_promql_selector_chain() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let projection = LogicalPlanBuilder::from(scan_plan(provider, "t")) + .project(vec![ + col("k0").alias("k0"), + Expr::Cast(Cast::new( + Box::new(col("ts")), + DataType::Timestamp(ArrowTimeUnit::Millisecond, None), + )) + .alias("ts"), + col("v0"), + ]) + .unwrap() + .sort(vec![col("ts").sort(true, false)]) + .unwrap() + .build() + .unwrap(); + let divide = LogicalPlan::Extension(Extension { + node: Arc::new(SeriesDivide::new( + vec!["k0".to_string()], + "ts".to_string(), + projection, + )), + }); + let normalize = LogicalPlan::Extension(Extension { + node: Arc::new(SeriesNormalize::new( + 42, + "ts", + false, + vec!["k0".to_string()], + divide, + )), + }); + let rewritten = ScanHintRule + .rewrite(single_evaluation(normalize), &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!( + scan_requests(&rewritten)[0].series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: true }) + ); + } + + #[test] + fn single_evaluation_filtering_normalize_does_not_set_last_row() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let normalize = LogicalPlan::Extension(Extension { + node: Arc::new(SeriesNormalize::new( + 42, + "ts", + true, + vec!["k0".to_string()], + scan_plan(provider, "t"), + )), + }); + let rewritten = ScanHintRule + .rewrite(single_evaluation(normalize), &OptimizerContext::default()) + .unwrap() + .data; + + // An older finite sample can precede a newest stale marker. LastRow + // would discard that sample before SeriesNormalize filters the marker. + assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None); + } + + #[test] + fn single_evaluation_rejects_row_changing_nodes() { + let provider = || Arc::new(mock_table_provider(RegionId::new(1, 1))); + let window = LogicalPlanBuilder::from(scan_plan(provider(), "window")) + .window(vec![Expr::WindowFunction(Box::new(WindowFunction { + fun: WindowFunctionDefinition::WindowUDF(Arc::new(RowNumber::new().into())), + params: WindowFunctionParams { + args: vec![], + partition_by: vec![col("k0")], + order_by: vec![col("ts").sort(true, true)], + window_frame: WindowFrame::new(Some(true)), + filter: None, + null_treatment: None, + distinct: false, + }, + }))]) + .unwrap() + .build() + .unwrap(); + let join = LogicalPlanBuilder::from(scan_plan(provider(), "left")) + .join( + scan_plan(provider(), "right"), + JoinType::Inner, + (Vec::::new(), Vec::::new()), + None, + ) + .unwrap() + .build() + .unwrap(); + let nonlast_aggregate = LogicalPlanBuilder::from(scan_plan(provider(), "aggregate")) + .aggregate( + vec![col("k0")], + vec![Expr::AggregateFunction(AggregateFunction { + func: max_udaf(), + params: AggregateFunctionParams { + args: vec![col("v0")], + distinct: false, + filter: None, + order_by: vec![], + null_treatment: None, + }, + })], + ) + .unwrap() + .build() + .unwrap(); + let range = LogicalPlan::Extension(Extension { + node: Arc::new( + RangeManipulate::new( + 1000, + 1000, + 1000, + 1000, + "ts".to_string(), + vec!["v0".to_string()], + scan_plan(provider(), "range"), + ) + .unwrap(), + ), + }); + + for (plan, scan_count) in [(window, 1), (join, 2), (nonlast_aggregate, 1), (range, 1)] { + let rewritten = ScanHintRule + .rewrite(single_evaluation(plan), &OptimizerContext::default()) + .unwrap() + .data; + assert_eq!( + scan_requests(&rewritten) + .into_iter() + .map(|request| request.series_row_selector) + .collect::>(), + vec![None; scan_count] + ); + } + } + + #[test] + fn single_evaluation_last_value_aggregate_keeps_legacy_selector() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let rewritten = ScanHintRule + .rewrite( + single_evaluation(last_value_aggregate(scan_plan(provider, "t"))), + &OptimizerContext::default(), + ) + .unwrap() + .data; + + assert_eq!( + scan_requests(&rewritten)[0].series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: false }) + ); + } + + #[test] + fn single_evaluation_allows_second_to_millisecond_time_index_cast() { + let provider = Arc::new(mock_table_provider_with_timestamp( + RegionId::new(1, 1), + ConcreteDataType::timestamp_second_datatype(), + )); + let input = LogicalPlanBuilder::from(scan_plan(provider, "t")) + .project(vec![ + Expr::Cast(Cast::new( + Box::new(Expr::Column(Column::new(Some("t"), "ts"))), + DataType::Timestamp(ArrowTimeUnit::Millisecond, None), + )) + .alias("ts"), + ]) + .unwrap() + .build() + .unwrap(); + let rewritten = ScanHintRule + .rewrite(single_evaluation(input), &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!( + scan_requests(&rewritten)[0].series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: true }) + ); + } + + #[test] + fn single_evaluation_rejects_microsecond_and_nanosecond_time_index_casts() { + for timestamp_type in [ + ConcreteDataType::timestamp_microsecond_datatype(), + ConcreteDataType::timestamp_nanosecond_datatype(), + ] { + let provider = Arc::new(mock_table_provider_with_timestamp( + RegionId::new(1, 1), + timestamp_type, + )); + let input = LogicalPlanBuilder::from(scan_plan(provider, "t")) + .project(vec![ + Expr::Cast(Cast::new( + Box::new(Expr::Column(Column::new(Some("t"), "ts"))), + DataType::Timestamp(ArrowTimeUnit::Millisecond, None), + )) + .alias("ts"), + ]) + .unwrap() + .build() + .unwrap(); + let rewritten = ScanHintRule + .rewrite(single_evaluation(input), &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None); + } + } + + #[test] + fn single_evaluation_rejects_unresolved_time_index_cast_qualifier() { + let provider = Arc::new(mock_table_provider_with_timestamp( + RegionId::new(1, 1), + ConcreteDataType::timestamp_second_datatype(), + )); + let input = LogicalPlanBuilder::from(scan_plan(provider, "t")) + .project(vec![col("ts")]) + .unwrap() + .build() + .unwrap(); + let LogicalPlan::Projection(projection) = input else { + unreachable!(); + }; + let unresolved_cast = Expr::Cast(Cast::new( + Box::new(Expr::Column(Column::new(Some("missing"), "ts"))), + DataType::Timestamp(ArrowTimeUnit::Millisecond, None), + )) + .alias("ts"); + + assert!(!single_evaluation_projection_expr_allowed( + &unresolved_cast, + &projection + )); + } + + #[test] + fn single_evaluation_rejects_projection_expressions_that_change_rows() { + let invalid_projections = [ + vec![col("ts").alias("renamed")], + vec![ + Expr::BinaryExpr(datafusion_expr::expr::BinaryExpr::new( + Box::new(col("v0")), + datafusion_expr::Operator::Plus, + Box::new(lit(1.0_f64)), + )) + .alias("v0"), + ], + vec![ + Expr::Cast(Cast::new( + Box::new(col("ts")), + DataType::Timestamp(ArrowTimeUnit::Second, None), + )) + .alias("ts"), + ], + vec![Expr::Cast(Cast::new(Box::new(col("v0")), DataType::Int64)).alias("v0")], + vec![ + Expr::Cast(Cast::new( + Box::new(col("ts")), + DataType::Timestamp(ArrowTimeUnit::Microsecond, None), + )) + .alias("ts"), + ], + vec![ + Expr::Cast(Cast::new( + Box::new(col("ts")), + DataType::Timestamp(ArrowTimeUnit::Nanosecond, None), + )) + .alias("ts"), + ], + ]; + + for expressions in invalid_projections { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let input = LogicalPlanBuilder::from(scan_plan(provider, "t")) + .project(expressions) + .unwrap() + .build() + .unwrap(); + let rewritten = ScanHintRule + .rewrite(single_evaluation(input), &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None); + } + } + + #[test] + fn range_evaluation_does_not_set_last_row() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let plan = instant_plan(provider.clone(), 2000); + let rewritten = ScanHintRule + .rewrite(plan, &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None); + + assert_eq!(provider.scan_request().series_row_selector, None); + } + + #[test] + fn residual_field_filter_does_not_set_last_row() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let input = LogicalPlanBuilder::from(scan_plan(provider, "t")) + .filter(col("v0").gt(lit(1.0_f64))) + .unwrap() + .build() + .unwrap(); + let rewritten = ScanHintRule + .rewrite(single_evaluation(input), &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None); + } + + #[test] + fn outer_residual_filter_does_not_block_last_row() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let plan = LogicalPlanBuilder::from(single_evaluation(scan_plan(provider, "t"))) + .filter(col("v0").gt(lit(1.0_f64))) + .unwrap() + .build() + .unwrap(); + let rewritten = ScanHintRule + .rewrite(plan, &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!( + scan_requests(&rewritten)[0].series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: true }) + ); + } + + #[test] + fn branch_local_residual_filter_isolation_in_both_orders() { + for filtered_first in [true, false] { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let filtered = single_evaluation( + LogicalPlanBuilder::from(scan_plan(provider.clone(), "filtered")) + .filter(col("v0").gt(lit(1.0_f64))) + .unwrap() + .build() + .unwrap(), + ); + let plain = single_evaluation(scan_plan(provider.clone(), "plain")); + let union = if filtered_first { + LogicalPlanBuilder::from(filtered) + .union(plain) + .unwrap() + .build() + .unwrap() + } else { + LogicalPlanBuilder::from(plain) + .union(filtered) + .unwrap() + .build() + .unwrap() + }; + let rewritten = ScanHintRule + .rewrite(union, &OptimizerContext::default()) + .unwrap() + .data; + let requests = scan_requests_with_names(&rewritten) + .into_iter() + .collect::>(); + + assert_eq!(requests["filtered"].series_row_selector, None); + assert_eq!( + requests["plain"].series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: true }) + ); + assert_eq!(provider.scan_request().series_row_selector, None); + } + } + + #[test] + fn union_inside_single_evaluation_blocks_both_branches() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let union = LogicalPlanBuilder::from(scan_plan(provider.clone(), "left")) + .union(scan_plan(provider, "right")) + .unwrap() + .build() + .unwrap(); + let rewritten = ScanHintRule + .rewrite(single_evaluation(union), &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!( + scan_requests(&rewritten) + .into_iter() + .map(|request| request.series_row_selector) + .collect::>(), + vec![None, None] + ); + } + + #[test] + fn residual_time_filters_do_not_set_last_row() { + for predicate in [ + col("ts").lt(lit(1_i64)), + Expr::Cast(Cast::new(Box::new(col("ts")), DataType::Int64)).gt(lit(1_i64)), + ] { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let input = LogicalPlanBuilder::from(scan_plan(provider, "t")) + .filter(predicate) + .unwrap() + .build() + .unwrap(); + let rewritten = ScanHintRule + .rewrite(single_evaluation(input), &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None); + } + } + + #[test] + fn inner_single_evaluation_resets_residual_filter() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let filtered = LogicalPlanBuilder::from(single_evaluation(scan_plan(provider, "t"))) + .filter(col("v0").gt(lit(1.0_f64))) + .unwrap() + .build() + .unwrap(); + let rewritten = ScanHintRule + .rewrite(single_evaluation(filtered), &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!( + scan_requests(&rewritten)[0].series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: true }) + ); + } + + #[test] + fn single_evaluation_with_tag_filter_sets_last_row() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let plan = instant_plan_with_filters(provider, vec![col("k0").eq(lit("tag"))]); + let rewritten = ScanHintRule + .rewrite(plan, &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!( + scan_requests(&rewritten)[0].series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: true }) + ); + } + + #[test] + fn single_evaluation_with_timestamp_filter_sets_last_row() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let plan = instant_plan_with_filters(provider, vec![col("ts").gt_eq(lit(1_i64))]); + let rewritten = ScanHintRule + .rewrite(plan, &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!( + scan_requests(&rewritten)[0].series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: true }) + ); + } + + #[test] + fn single_evaluation_with_cast_timestamp_filter_does_not_set_last_row() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let filter = Expr::Cast(Cast::new(Box::new(col("ts")), DataType::Int64)).gt(lit(1_i64)); + let plan = instant_plan_with_filters(provider, vec![filter]); + let rewritten = ScanHintRule + .rewrite(plan, &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None); + } + + #[test] + fn single_evaluation_with_multi_column_timestamp_filter_does_not_set_last_row() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let plan = instant_plan_with_filters(provider, vec![col("ts").gt(col("k0"))]); + let rewritten = ScanHintRule + .rewrite(plan, &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None); + } + + #[test] + fn single_evaluation_with_field_filter_does_not_set_last_row() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let plan = instant_plan_with_filters(provider, vec![col("v0").gt(lit(1.0_f64))]); + let rewritten = ScanHintRule + .rewrite(plan, &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None); + } + + #[test] + fn single_evaluation_with_unknown_filter_does_not_set_last_row() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let plan = instant_plan_with_filters(provider, vec![col("unknown").eq(lit(1_i64))]); + let rewritten = ScanHintRule + .rewrite(plan, &OptimizerContext::default()) + .unwrap() + .data; + + assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None); + } + + #[test] + fn expression_subquery_isolated_from_outer_single_evaluation() { + let outer_provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let inner_provider = Arc::new(mock_table_provider(RegionId::new(2, 1))); + let plan = instant_with_expression_subquery( + outer_provider.clone(), + instant_plan_named(inner_provider, "inner", 2000), + 1000, + ); + let rewritten = ScanHintRule + .rewrite(plan, &OptimizerContext::default()) + .unwrap() + .data; + + let requests = scan_requests_with_names(&rewritten) + .into_iter() + .collect::>(); + assert_eq!(requests["outer"].series_row_selector, None); + assert_eq!(requests["inner"].series_row_selector, None); + } + + #[test] + fn expression_subquery_can_start_its_own_single_evaluation() { + let outer_provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let inner_provider = Arc::new(mock_table_provider(RegionId::new(2, 1))); + let plan = instant_with_expression_subquery( + outer_provider, + instant_plan_named(inner_provider, "inner", 1000), + 1000, + ); + let rewritten = ScanHintRule + .rewrite(plan, &OptimizerContext::default()) + .unwrap() + .data; + + let requests = scan_requests_with_names(&rewritten) + .into_iter() + .collect::>(); + assert_eq!(requests["outer"].series_row_selector, None); + assert_eq!( + requests["inner"].series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: true }) + ); + } + + #[test] + fn nested_single_outer_range_inner_does_not_set_inner_last_row() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let plan = instant_plan_named(provider.clone(), "nested", 2000); + let plan = LogicalPlan::Extension(Extension { + node: Arc::new(InstantManipulate::new( + 1000, + 1000, + 1000, + 1000, + "ts".to_string(), + vec![], + Some("v0".to_string()), + plan, + )), + }); + let rewritten = ScanHintRule + .rewrite(plan, &OptimizerContext::default()) + .unwrap() + .data; + assert_eq!(scan_requests(&rewritten)[0].series_row_selector, None); + assert_eq!(provider.scan_request().series_row_selector, None); + } + + #[test] + fn nested_range_outer_single_inner_sets_inner_last_row() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let plan = instant_plan_named(provider.clone(), "nested", 1000); + let plan = LogicalPlan::Extension(Extension { + node: Arc::new(InstantManipulate::new( + 1000, + 2000, + 1000, + 1000, + "ts".to_string(), + vec![], + Some("v0".to_string()), + plan, + )), + }); + let rewritten = ScanHintRule + .rewrite(plan, &OptimizerContext::default()) + .unwrap() + .data; + assert_eq!( + scan_requests(&rewritten)[0].series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: true }) + ); + } + + #[test] + fn shared_provider_isolated_between_single_and_range_scans() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let plan = LogicalPlanBuilder::from(instant_plan(provider.clone(), 1000)) + .union(instant_plan(provider.clone(), 2000)) + .unwrap() + .build() + .unwrap(); + let rewritten = ScanHintRule + .rewrite(plan, &OptimizerContext::default()) + .unwrap() + .data; + let requests = scan_requests(&rewritten); + + assert_eq!(requests.len(), 2); + assert_eq!( + requests[0].series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: true }) + ); + assert_eq!(requests[1].series_row_selector, None); + assert_eq!(provider.scan_request().series_row_selector, None); + } + + #[test] + fn set_order_hint() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let table_source = Arc::new(DefaultTableSource::new(provider.clone())); + let plan = LogicalPlanBuilder::scan("t", table_source, None) + .unwrap() + .sort(vec![col("ts").sort(true, false)]) + .unwrap() + .sort(vec![col("ts").sort(false, true)]) + .unwrap() + .build() .unwrap(); let context = OptimizerContext::default(); - ScanHintRule.rewrite(plan, &context).unwrap(); + let rewritten = ScanHintRule.rewrite(plan, &context).unwrap().data; - let scan_req = provider.scan_request(); - let _ = scan_req.series_row_selector.unwrap(); + // should read the first (with `.sort(true, false)`) sort option + let scan_req = scan_requests(&rewritten)[0].clone(); + assert_eq!( + OrderOption { + name: "ts".to_string(), + options: SortOptions { + descending: false, + nulls_first: false + } + }, + scan_req.output_ordering.as_ref().unwrap()[0] + ); + } + + #[test] + fn set_time_series_row_selector_hint() { + let provider = Arc::new(mock_table_provider(RegionId::new(1, 1))); + let plan = last_value_aggregate(scan_plan(provider.clone(), "t")); + + let context = OptimizerContext::default(); + let rewritten = ScanHintRule.rewrite(plan, &context).unwrap().data; + + let scan_req = scan_requests(&rewritten)[0].clone(); + assert_eq!( + scan_req.series_row_selector, + Some(TimeSeriesRowSelector::LastRow { after_merge: false }) + ); } #[test] @@ -534,9 +1540,9 @@ mod test { .unwrap(); let context = OptimizerContext::default(); - ScanHintRule.rewrite(plan, &context).unwrap(); + let rewritten = ScanHintRule.rewrite(plan, &context).unwrap().data; - let scan_req = provider.scan_request(); + let scan_req = scan_requests(&rewritten)[0].clone(); assert_eq!( scan_req.distribution, Some(TimeSeriesDistribution::PerSeries) diff --git a/src/query/src/optimizer/scan_hint/vector_search.rs b/src/query/src/optimizer/scan_hint/vector_search.rs index 7f97ad5ac1..1a4d274cca 100644 --- a/src/query/src/optimizer/scan_hint/vector_search.rs +++ b/src/query/src/optimizer/scan_hint/vector_search.rs @@ -410,12 +410,13 @@ mod tests { use datafusion_expr::expr::ScalarFunction; use datafusion_expr::logical_plan::JoinType; use datafusion_expr::{ - Expr, LogicalPlan, LogicalPlanBuilder, Signature, Subquery, Volatility, col, lit, + Expr, Extension, LogicalPlan, LogicalPlanBuilder, Signature, Subquery, Volatility, col, lit, }; use datafusion_optimizer::{OptimizerContext, OptimizerRule}; use datatypes::schema::ColumnSchema; + use promql::extension_plan::InstantManipulate; use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder}; - use store_api::storage::{ConcreteDataType, VectorDistanceMetric}; + use store_api::storage::{ConcreteDataType, VectorDistanceMetric, VectorSearchRequest}; use super::VectorSearchState; use crate::dummy_catalog::DummyTableProvider; @@ -469,16 +470,42 @@ mod tests { } fn vec_distance_expr(function_name: &'static str) -> Expr { + vec_distance_expr_with_query(function_name, "[1.0, 2.0]") + } + + fn vec_distance_expr_with_query(function_name: &'static str, query_vector: &str) -> Expr { let udf = create_udf(Arc::new(TestVectorFunction::new(function_name))); Expr::ScalarFunction(ScalarFunction::new_udf( Arc::new(udf), vec![ col("v"), - lit(ScalarValue::Utf8(Some("[1.0, 2.0]".to_string()))), + lit(ScalarValue::Utf8(Some(query_vector.to_string()))), ], )) } + fn scan_request_from_plan(plan: &LogicalPlan) -> store_api::storage::ScanRequest { + let mut request = None; + plan.apply_with_subqueries(|node| { + if let LogicalPlan::TableScan(scan) = node + && let Some(source) = scan.source.as_any().downcast_ref::() + && let Some(provider) = source + .table_provider + .as_any() + .downcast_ref::() + { + request = Some(provider.scan_request()); + } + Ok(datafusion_common::tree_node::TreeNodeRecursion::Continue) + }) + .unwrap(); + request.unwrap() + } + + fn vector_hint_from_plan(plan: &LogicalPlan) -> Option { + scan_request_from_plan(plan).vector_search + } + fn build_dummy_provider(column_id: u32) -> Arc { build_dummy_provider_with_nullable(column_id, false) } @@ -565,15 +592,147 @@ mod tests { .unwrap(); let context = OptimizerContext::default(); - let _ = ScanHintRule.rewrite(plan, &context).unwrap(); + let rewritten = ScanHintRule.rewrite(plan, &context).unwrap().data; - let hint = dummy_provider.get_vector_search_hint().unwrap(); + let hint = vector_hint_from_plan(&rewritten).unwrap(); assert_eq!(hint.column_id, 10); assert_eq!(hint.k, 5); assert_eq!(hint.metric, VectorDistanceMetric::L2sq); assert_eq!(hint.query_vector, vec![1.0, 2.0]); } + #[test] + fn test_vector_hint_is_applied_when_single_evaluation_limit_blocks_last_row() { + let provider = build_dummy_provider(10); + let source = Arc::new(DefaultTableSource::new(provider)); + let distance = vec_distance_expr(VEC_L2SQ_DISTANCE); + let scan = LogicalPlanBuilder::scan_with_filters("t", source, None, vec![]) + .unwrap() + .sort(vec![distance.sort(true, false)]) + .unwrap() + .limit(0, Some(5)) + .unwrap() + .build() + .unwrap(); + let plan = LogicalPlan::Extension(Extension { + node: Arc::new(InstantManipulate::new( + 1000, + 1000, + 1000, + 1000, + "ts".to_string(), + vec![], + Some("v".to_string()), + scan, + )), + }); + + let rewritten = ScanHintRule + .rewrite(plan, &OptimizerContext::default()) + .unwrap() + .data; + let request = scan_request_from_plan(&rewritten); + assert_eq!(request.series_row_selector, None); + assert_eq!( + request.vector_search, + Some(VectorSearchRequest { + column_id: 10, + query_vector: vec![1.0, 2.0], + k: 5, + metric: VectorDistanceMetric::L2sq, + }) + ); + } + + #[test] + fn test_single_evaluation_limit_keeps_vector_hints_on_both_branches() { + let provider_a = build_dummy_provider(10); + let provider_b = build_dummy_provider(20); + let distance_a = vec_distance_expr_with_query(VEC_L2SQ_DISTANCE, "[1.0, 2.0]"); + let distance_b = vec_distance_expr_with_query(VEC_L2SQ_DISTANCE, "[3.0, 4.0]"); + + let branch_a = LogicalPlanBuilder::scan_with_filters( + "t", + Arc::new(DefaultTableSource::new(provider_a)), + None, + vec![], + ) + .unwrap() + .sort(vec![distance_a.sort(true, false)]) + .unwrap() + .limit(0, Some(5)) + .unwrap() + .build() + .unwrap(); + let branch_a = LogicalPlan::Extension(Extension { + node: Arc::new(InstantManipulate::new( + 1000, + 1000, + 1000, + 1000, + "ts".to_string(), + vec![], + Some("v".to_string()), + branch_a, + )), + }); + + let branch_b = LogicalPlanBuilder::scan_with_filters( + "t", + Arc::new(DefaultTableSource::new(provider_b)), + None, + vec![], + ) + .unwrap() + .sort(vec![distance_b.sort(true, false)]) + .unwrap() + .limit(0, Some(9)) + .unwrap() + .build() + .unwrap(); + + let plan = LogicalPlanBuilder::from(branch_a) + .union(branch_b) + .unwrap() + .build() + .unwrap(); + let rewritten = ScanHintRule + .rewrite(plan, &OptimizerContext::default()) + .unwrap() + .data; + + let LogicalPlan::Union(union) = rewritten else { + panic!("expected union plan") + }; + let request_a = scan_request_from_plan(&union.inputs[0]); + let request_b = scan_request_from_plan(&union.inputs[1]); + + assert_eq!(request_a.series_row_selector, None); + assert_eq!( + request_a.vector_search, + Some(VectorSearchRequest { + column_id: 10, + query_vector: vec![1.0, 2.0], + k: 5, + metric: VectorDistanceMetric::L2sq, + }) + ); + assert_eq!(request_b.series_row_selector, None); + assert_eq!( + request_b.vector_search, + Some(VectorSearchRequest { + column_id: 20, + query_vector: vec![3.0, 4.0], + k: 9, + metric: VectorDistanceMetric::L2sq, + }) + ); + + // The catalog providers are not mutated; the requests above belong to + // the two rewritten scan use-sites. + assert_ne!(request_a.vector_search, request_b.vector_search); + } + #[test] fn test_limit_offset_for_vector_hint() { let dummy_provider = build_dummy_provider(10); @@ -589,9 +748,9 @@ mod tests { .unwrap(); let context = OptimizerContext::default(); - let _ = ScanHintRule.rewrite(plan, &context).unwrap(); + let rewritten = ScanHintRule.rewrite(plan, &context).unwrap().data; - let hint = dummy_provider.get_vector_search_hint().unwrap(); + let hint = vector_hint_from_plan(&rewritten).unwrap(); assert_eq!(hint.k, 15); } @@ -621,8 +780,8 @@ mod tests { .unwrap() .build() .unwrap(); - let _ = ScanHintRule.rewrite(plan, &context).unwrap(); - let hint = dummy_provider.get_vector_search_hint().unwrap(); + let rewritten = ScanHintRule.rewrite(plan, &context).unwrap().data; + let hint = vector_hint_from_plan(&rewritten).unwrap(); assert_eq!(hint.metric, VectorDistanceMetric::InnerProduct); } @@ -681,9 +840,9 @@ mod tests { .unwrap(); let context = OptimizerContext::default(); - let _ = ScanHintRule.rewrite(plan, &context).unwrap(); + let rewritten = ScanHintRule.rewrite(plan, &context).unwrap().data; - let hint = dummy_provider.get_vector_search_hint().unwrap(); + let hint = vector_hint_from_plan(&rewritten).unwrap(); assert_eq!(hint.column_id, 10); assert_eq!(hint.k, 5); } @@ -701,9 +860,9 @@ mod tests { .unwrap(); let context = OptimizerContext::default(); - let _ = ScanHintRule.rewrite(plan, &context).unwrap(); + let rewritten = ScanHintRule.rewrite(plan, &context).unwrap().data; - let hint = dummy_provider.get_vector_search_hint().unwrap(); + let hint = vector_hint_from_plan(&rewritten).unwrap(); assert_eq!(hint.k, 4); } @@ -828,10 +987,10 @@ mod tests { .unwrap(); let context = OptimizerContext::default(); - let _ = ScanHintRule.rewrite(plan, &context).unwrap(); + let rewritten = ScanHintRule.rewrite(plan, &context).unwrap().data; // Hint propagates through simple subquery - let hint = provider.get_vector_search_hint().unwrap(); + let hint = vector_hint_from_plan(&rewritten).unwrap(); assert_eq!(hint.k, 5); } @@ -901,8 +1060,8 @@ mod tests { let _ = ScanHintRule.rewrite(t1_plan, &context).unwrap(); assert!(t1_provider.get_vector_search_hint().is_none()); - let _ = ScanHintRule.rewrite(t2_plan, &context).unwrap(); - let hint = t2_provider.get_vector_search_hint().unwrap(); + let rewritten = ScanHintRule.rewrite(t2_plan, &context).unwrap().data; + let hint = vector_hint_from_plan(&rewritten).unwrap(); assert_eq!(hint.column_id, 20); assert_eq!(hint.k, 5); } diff --git a/src/store-api/src/storage/requests.rs b/src/store-api/src/storage/requests.rs index cb615ea1df..b35a1907d4 100644 --- a/src/store-api/src/storage/requests.rs +++ b/src/store-api/src/storage/requests.rs @@ -82,7 +82,11 @@ pub trait VectorIndexEngine: Send + Sync { #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Display)] pub enum TimeSeriesRowSelector { /// Only keep the last row of each time-series. - LastRow, + #[strum(to_string = "LastRow {{ after_merge: {after_merge} }}")] + LastRow { + /// Whether selection runs after cross-source merge and deduplication. + after_merge: bool, + }, } /// A hint on how to distribute time-series data on the scan output. @@ -323,13 +327,19 @@ mod tests { ); let request = ScanRequest { + series_row_selector: Some(TimeSeriesRowSelector::LastRow { after_merge: true }), snapshot_on_scan: true, exact_sequence_range: true, ..Default::default() }; assert_eq!( request.to_string(), - "ScanRequest { snapshot_on_scan: true, exact_sequence_range: true }" + "ScanRequest { series_row_selector: LastRow { after_merge: true }, snapshot_on_scan: true, exact_sequence_range: true }" + ); + + assert_eq!( + TimeSeriesRowSelector::LastRow { after_merge: false }.to_string(), + "LastRow { after_merge: false }" ); let request = ScanRequest { diff --git a/tests-integration/src/tests/promql_test.rs b/tests-integration/src/tests/promql_test.rs index ede4663118..c816583f13 100644 --- a/tests-integration/src/tests/promql_test.rs +++ b/tests-integration/src/tests/promql_test.rs @@ -15,9 +15,16 @@ use std::sync::Arc; use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use api::v1::value::ValueData; +use api::v1::{ + ColumnDataType, ColumnSchema, Row, RowInsertRequest, RowInsertRequests, Rows, SemanticType, +}; +use common_query::prometheus::PROMETHEUS_STALE_NAN_BITS; use common_query::{Output, OutputData}; use common_recordbatch::util::collect_batches; -use datatypes::arrow::array::{Float64Array, Int64Array}; +use datatypes::arrow::array::{Float64Array, Int64Array, StringArray, TimestampMillisecondArray}; +use datatypes::arrow::compute::cast; +use datatypes::arrow::datatypes::DataType; use frontend::instance::Instance; use query::parser::{PromQuery, QueryLanguageParser, QueryStatement}; use rstest::rstest; @@ -945,3 +952,198 @@ async fn anon_promql_ratio_repro(instance: Arc) { whole[0].pretty_print() ); } + +#[apply(both_instances_cases)] +async fn promql_stale_marker_excludes_series_across_flushes(instance: Arc) { + let ins = instance.frontend(); + let table = "promql_stale_marker"; + execute_all( + &ins, + &format!( + "CREATE TABLE {table} (\ + series STRING, \ + ts TIMESTAMP TIME INDEX, \ + val DOUBLE, \ + PRIMARY KEY (series)\ + ) WITH (sst_format = 'flat')" + ), + QueryContext::arc(), + ) + .await; + + let rows = |samples: &[(&str, f64, i64)]| Rows { + schema: vec![ + ColumnSchema { + column_name: "series".to_string(), + datatype: ColumnDataType::String as i32, + semantic_type: SemanticType::Tag as i32, + ..Default::default() + }, + ColumnSchema { + column_name: "ts".to_string(), + datatype: ColumnDataType::TimestampMillisecond as i32, + semantic_type: SemanticType::Timestamp as i32, + ..Default::default() + }, + ColumnSchema { + column_name: "val".to_string(), + datatype: ColumnDataType::Float64 as i32, + semantic_type: SemanticType::Field as i32, + ..Default::default() + }, + ], + rows: samples + .iter() + .map(|(series, value, timestamp)| Row { + values: vec![ + ValueData::StringValue((*series).to_string()).into(), + ValueData::TimestampMillisecondValue(*timestamp).into(), + ValueData::F64Value(*value).into(), + ], + }) + .collect(), + }; + let insert = |samples: &[(&str, f64, i64)]| RowInsertRequests { + inserts: vec![RowInsertRequest { + table_name: table.to_string(), + rows: Some(rows(samples)), + }], + }; + + ins.handle_row_inserts( + insert(&[("stale", 10.0, 1_000), ("ordinary", 20.0, 1_000)]), + QueryContext::arc(), + false, + false, + ) + .await + .unwrap(); + execute_all( + &ins, + &format!("ADMIN FLUSH_TABLE('{table}')"), + QueryContext::arc(), + ) + .await; + + let ordinary_nan = f64::from_bits(0x7ff8_0000_0000_0000); + ins.handle_row_inserts( + insert(&[ + ("stale", f64::from_bits(PROMETHEUS_STALE_NAN_BITS), 2_000), + ("ordinary", ordinary_nan, 2_000), + ]), + QueryContext::arc(), + false, + false, + ) + .await + .unwrap(); + + let extract_samples = |batches: common_recordbatch::RecordBatches| { + let mut samples = batches + .iter() + .flat_map(|batch| { + let series = cast( + batch.column_by_name("series").unwrap().as_ref(), + &DataType::Utf8, + ) + .unwrap(); + let series = series.as_any().downcast_ref::().unwrap(); + let values = batch + .column_by_name("val") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + let timestamps = batch + .column_by_name("ts") + .unwrap() + .as_any() + .downcast_ref::() + .unwrap(); + (0..batch.num_rows()) + .map(|row| { + ( + series.value(row).to_string(), + values.value(row), + timestamps.value(row), + ) + }) + .collect::>() + }) + .collect::>(); + samples.sort_by(|left, right| left.0.cmp(&right.0)); + samples + }; + + let raw_output = ins + .do_query( + &format!("SELECT series, val, ts FROM {table} WHERE ts = 2000"), + QueryContext::arc(), + ) + .await + .remove(0) + .unwrap(); + let raw_batches = match raw_output.data { + OutputData::Stream(stream) => collect_batches(stream).await.unwrap(), + OutputData::RecordBatches(recordbatches) => recordbatches, + _ => unreachable!(), + }; + let raw_samples = extract_samples(raw_batches); + assert_eq!(raw_samples.len(), 2, "{raw_samples:?}"); + let stale_marker = raw_samples + .iter() + .find(|(series, _, _)| series == "stale") + .unwrap(); + assert_eq!(stale_marker.1.to_bits(), PROMETHEUS_STALE_NAN_BITS); + let ordinary_marker = raw_samples + .iter() + .find(|(series, _, _)| series == "ordinary") + .unwrap(); + assert_eq!(ordinary_marker.1.to_bits(), ordinary_nan.to_bits()); + + // Keep the finite 1000ms sample eligible so a stale marker cannot fall back to it. + let query_at = |at| { + let time = UNIX_EPOCH.checked_add(Duration::from_millis(at)).unwrap(); + promql_query_as_batches( + ins.clone(), + table, + None, + QueryContext::arc(), + time, + time, + Duration::from_secs(1), + Duration::from_secs(3), + ) + }; + let assert_nan_series = |samples: Vec<(String, f64, i64)>, at| { + assert_eq!(samples.len(), 1, "{samples:?}"); + assert_eq!(samples[0].0, "ordinary", "{samples:?}"); + assert_eq!(samples[0].2, at, "{samples:?}"); + assert_eq!( + samples[0].1.to_bits(), + ordinary_nan.to_bits(), + "{samples:?}" + ); + }; + + for flush_markers in [false, true] { + if flush_markers { + execute_all( + &ins, + &format!("ADMIN FLUSH_TABLE('{table}')"), + QueryContext::arc(), + ) + .await; + } + + assert_eq!( + extract_samples(query_at(1_500).await), + vec![ + ("ordinary".to_string(), 20.0, 1_500), + ("stale".to_string(), 10.0, 1_500) + ] + ); + assert_nan_series(extract_samples(query_at(2_000).await), 2_000); + assert_nan_series(extract_samples(query_at(2_500).await), 2_500); + } +} diff --git a/tests/cases/distributed/explain/step_aggr_advance.result b/tests/cases/distributed/explain/step_aggr_advance.result index 59f2946e8b..d3aaea79bc 100644 --- a/tests/cases/distributed/explain/step_aggr_advance.result +++ b/tests/cases/distributed/explain/step_aggr_advance.result @@ -429,7 +429,7 @@ tql analyze sum(aggr_optimize_not); |_|_|_ProjectionExec: expr=[greptime_timestamp@4 as greptime_timestamp, greptime_value@5 as greptime_value] REDACTED |_|_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "mode":"legacy" REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[greptime_timestamp@0 as greptime_timestamp], aggr=[__sum_state(aggr_optimize_not.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED @@ -437,7 +437,7 @@ tql analyze sum(aggr_optimize_not); |_|_|_ProjectionExec: expr=[greptime_timestamp@4 as greptime_timestamp, greptime_value@5 as greptime_value] REDACTED |_|_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "mode":"legacy" REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ diff --git a/tests/cases/distributed/optimizer/last_value_advance.result b/tests/cases/distributed/optimizer/last_value_advance.result index 6269c5d397..0972741b17 100644 --- a/tests/cases/distributed/optimizer/last_value_advance.result +++ b/tests/cases/distributed/optimizer/last_value_advance.result @@ -113,7 +113,7 @@ explain analyze |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -165,7 +165,7 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -277,7 +277,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| | 1_| 1_|_ProjectionExec: expr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@0 as ordered_host, last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST]@1 as last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@2 as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_SortPreservingMergeExec: [last_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@0 ASC NULLS LAST] REDACTED @@ -286,7 +286,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| | 1_| 2_|_ProjectionExec: expr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@0 as ordered_host, last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST]@1 as last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]@2 as last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_SortPreservingMergeExec: [last_value(t.host) ORDER BY [t.ts ASC NULLS LAST]@0 ASC NULLS LAST] REDACTED @@ -295,7 +295,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -355,17 +355,17 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -559,17 +559,17 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -648,7 +648,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| | 1_| 1_|_ProjectionExec: expr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@0 as ordered_host, last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]@1 as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_SortPreservingMergeExec: [last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@0 ASC NULLS LAST] REDACTED @@ -657,7 +657,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| | 1_| 2_|_ProjectionExec: expr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@0 as ordered_host, last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]@1 as last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_SortPreservingMergeExec: [last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST]@0 ASC NULLS LAST] REDACTED @@ -666,7 +666,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ diff --git a/tests/cases/standalone/common/promql/instant_last_row.result b/tests/cases/standalone/common/promql/instant_last_row.result new file mode 100644 index 0000000000..5093a357ea --- /dev/null +++ b/tests/cases/standalone/common/promql/instant_last_row.result @@ -0,0 +1,548 @@ +-- Correctness coverage for the instant-selector last-row optimization. +-- Each TQL EVAL below has distinct values so its chosen physical sample is visible. +-- The default instant lookback is (T - 300s, T]: exclude its lower bound, +-- include a sample just inside it and at T, and exclude a future sample. +CREATE TABLE instant_last_lookback ( + ts TIMESTAMP TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +Affected Rows: 0 + +INSERT INTO instant_last_lookback VALUES + (700000, 10, 'lower_excluded'), + (700001, 11, 'just_inside'), + (1000000, 12, 'upper_included'), + (1000000, 14, 'future_has_prior'), + (1000001, 13, 'future_has_prior'); + +Affected Rows: 5 + +ADMIN FLUSH_TABLE('instant_last_lookback'); + ++--------------------------------------------+ +| ADMIN FLUSH_TABLE('instant_last_lookback') | ++--------------------------------------------+ +| 0 | ++--------------------------------------------+ + +-- Expected pairs: just_inside=11, upper_included=12, future_has_prior=14. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_lookback; + ++---------------------+------+------------------+ +| ts | val | series | ++---------------------+------+------------------+ +| 1970-01-01T00:16:40 | 11.0 | just_inside | +| 1970-01-01T00:16:40 | 12.0 | upper_included | +| 1970-01-01T00:16:40 | 14.0 | future_has_prior | ++---------------------+------+------------------+ + +-- An empty table must remain an empty instant vector. +CREATE TABLE instant_last_empty ( + ts TIMESTAMP TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +Affected Rows: 0 + +-- Expected: no rows. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_empty; + +++ +++ + +DROP TABLE instant_last_empty; + +Affected Rows: 0 + +DROP TABLE instant_last_lookback; + +Affected Rows: 0 + +-- Positive offset reads earlier data; negative offset reads later data. Repeat +-- the earlier evaluation after a later one to catch a cached instant window. +CREATE TABLE instant_last_offset ( + ts TIMESTAMP TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +Affected Rows: 0 + +INSERT INTO instant_last_offset VALUES + (940000, 21, 'offset'), + (1000000, 22, 'offset'), + (1060000, 23, 'offset'); + +Affected Rows: 3 + +ADMIN FLUSH_TABLE('instant_last_offset'); + ++------------------------------------------+ +| ADMIN FLUSH_TABLE('instant_last_offset') | ++------------------------------------------+ +| 0 | ++------------------------------------------+ + +-- Expected value: 22. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_offset; + ++---------------------+------+--------+ +| ts | val | series | ++---------------------+------+--------+ +| 1970-01-01T00:16:40 | 22.0 | offset | ++---------------------+------+--------+ + +-- Expected value: 21 (evaluation time shifted back 60s). +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_offset offset 60s; + ++---------------------+------+--------+ +| ts | val | series | ++---------------------+------+--------+ +| 1970-01-01T00:16:40 | 21.0 | offset | ++---------------------+------+--------+ + +-- Expected value: 23 (evaluation time shifted forward 60s). +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_offset offset -60s; + ++---------------------+------+--------+ +| ts | val | series | ++---------------------+------+--------+ +| 1970-01-01T00:16:40 | 23.0 | offset | ++---------------------+------+--------+ + +-- Expected value: 23. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1060, 1060, '1s') instant_last_offset; + ++---------------------+------+--------+ +| ts | val | series | ++---------------------+------+--------+ +| 1970-01-01T00:17:40 | 23.0 | offset | ++---------------------+------+--------+ + +-- Expected value: 22 again, not the later evaluation's value. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_offset; + ++---------------------+------+--------+ +| ts | val | series | ++---------------------+------+--------+ +| 1970-01-01T00:16:40 | 22.0 | offset | ++---------------------+------+--------+ + +DROP TABLE instant_last_offset; + +Affected Rows: 0 + +-- Put t1 and t2 in one SST. Deleting newest t2 must reveal t1 whether the +-- Delete is still in the memtable or has been flushed to a newer SST. +CREATE TABLE instant_last_delete ( + ts TIMESTAMP TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +Affected Rows: 0 + +INSERT INTO instant_last_delete VALUES + (900000, 31, 'delete'), + (1000000, 32, 'delete'); + +Affected Rows: 2 + +ADMIN FLUSH_TABLE('instant_last_delete'); + ++------------------------------------------+ +| ADMIN FLUSH_TABLE('instant_last_delete') | ++------------------------------------------+ +| 0 | ++------------------------------------------+ + +DELETE FROM instant_last_delete WHERE series = 'delete' AND ts = 1000000; + +Affected Rows: 1 + +-- Expected value: 31; the newest point is a memtable Delete. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_delete; + ++---------------------+------+--------+ +| ts | val | series | ++---------------------+------+--------+ +| 1970-01-01T00:16:40 | 31.0 | delete | ++---------------------+------+--------+ + +ADMIN FLUSH_TABLE('instant_last_delete'); + ++------------------------------------------+ +| ADMIN FLUSH_TABLE('instant_last_delete') | ++------------------------------------------+ +| 0 | ++------------------------------------------+ + +-- Expected value: 31; the Delete is now in an SST. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_delete; + ++---------------------+------+--------+ +| ts | val | series | ++---------------------+------+--------+ +| 1970-01-01T00:16:40 | 31.0 | delete | ++---------------------+------+--------+ + +-- A same-timestamp memtable overwrite of t1 must win over the old SST value. +INSERT INTO instant_last_delete VALUES (900000, 33, 'delete'); + +Affected Rows: 1 + +-- Expected value: 33. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_delete; + ++---------------------+------+--------+ +| ts | val | series | ++---------------------+------+--------+ +| 1970-01-01T00:16:40 | 33.0 | delete | ++---------------------+------+--------+ + +-- A newer memtable point must win, and retain that identity after its flush. +INSERT INTO instant_last_delete VALUES (1010000, 34, 'delete'); + +Affected Rows: 1 + +-- Expected value: 34. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1010, 1010, '1s') instant_last_delete; + ++---------------------+------+--------+ +| ts | val | series | ++---------------------+------+--------+ +| 1970-01-01T00:16:50 | 34.0 | delete | ++---------------------+------+--------+ + +ADMIN FLUSH_TABLE('instant_last_delete'); + ++------------------------------------------+ +| ADMIN FLUSH_TABLE('instant_last_delete') | ++------------------------------------------+ +| 0 | ++------------------------------------------+ + +-- Expected value: 33 after flush; it overwrites t1 in an older SST. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_delete; + ++---------------------+------+--------+ +| ts | val | series | ++---------------------+------+--------+ +| 1970-01-01T00:16:40 | 33.0 | delete | ++---------------------+------+--------+ + +-- Expected value: 34 after flush. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1010, 1010, '1s') instant_last_delete; + ++---------------------+------+--------+ +| ts | val | series | ++---------------------+------+--------+ +| 1970-01-01T00:16:50 | 34.0 | delete | ++---------------------+------+--------+ + +DROP TABLE instant_last_delete; + +Affected Rows: 0 + +-- Ordinary IEEE NaN is a valid PromQL sample and must not be suppressed. +CREATE TABLE instant_last_nan ( + ts TIMESTAMP TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +Affected Rows: 0 + +INSERT INTO instant_last_nan VALUES (1000000, 'NaN'::DOUBLE, 'ordinary_nan'); + +Affected Rows: 1 + +ADMIN FLUSH_TABLE('instant_last_nan'); + ++---------------------------------------+ +| ADMIN FLUSH_TABLE('instant_last_nan') | ++---------------------------------------+ +| 0 | ++---------------------------------------+ + +-- Expected value: NaN, with the ordinary_nan series present. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_nan; + ++---------------------+-----+--------------+ +| ts | val | series | ++---------------------+-----+--------------+ +| 1970-01-01T00:16:40 | NaN | ordinary_nan | ++---------------------+-----+--------------+ + +DROP TABLE instant_last_nan; + +Affected Rows: 0 + +-- PromQL normalizes to milliseconds; these distinct raw timestamps collide at +-- 1s. LastRow must preserve the unhinted path's selected samples: 42 and 52. +CREATE TABLE instant_last_sec ( + ts TIMESTAMP(0) TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +Affected Rows: 0 + +INSERT INTO instant_last_sec VALUES + (0, 61, 'sec'), + (1, 62, 'sec'), + (2, 63, 'sec'); + +Affected Rows: 3 + +ADMIN FLUSH_TABLE('instant_last_sec'); + ++---------------------------------------+ +| ADMIN FLUSH_TABLE('instant_last_sec') | ++---------------------------------------+ +| 0 | ++---------------------------------------+ + +-- Expected value: 62 at the native seconds boundary. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1, 1, '1s') instant_last_sec; + ++------+--------+---------------------+ +| val | series | ts | ++------+--------+---------------------+ +| 62.0 | sec | 1970-01-01T00:00:01 | ++------+--------+---------------------+ + +CREATE TABLE instant_last_micro ( + ts TIMESTAMP(6) TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +Affected Rows: 0 + +INSERT INTO instant_last_micro VALUES + (999999, 41, 'micro'), + (1000000, 42, 'micro'), + (1000001, 43, 'micro'); + +Affected Rows: 3 + +ADMIN FLUSH_TABLE('instant_last_micro'); + ++-----------------------------------------+ +| ADMIN FLUSH_TABLE('instant_last_micro') | ++-----------------------------------------+ +| 0 | ++-----------------------------------------+ + +-- Expected value: 42, matching the unhinted millisecond-normalized path. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1, 1, '1s') instant_last_micro; + ++------+--------+---------------------+ +| val | series | ts | ++------+--------+---------------------+ +| 42.0 | micro | 1970-01-01T00:00:01 | ++------+--------+---------------------+ + +CREATE TABLE instant_last_nano ( + ts TIMESTAMP(9) TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +Affected Rows: 0 + +INSERT INTO instant_last_nano VALUES + (999999999, 51, 'nano'), + (1000000000, 52, 'nano'), + (1000000001, 53, 'nano'); + +Affected Rows: 3 + +ADMIN FLUSH_TABLE('instant_last_nano'); + ++----------------------------------------+ +| ADMIN FLUSH_TABLE('instant_last_nano') | ++----------------------------------------+ +| 0 | ++----------------------------------------+ + +-- Expected value: 52, matching the unhinted millisecond-normalized path. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1, 1, '1s') instant_last_nano; + ++------+--------+---------------------+ +| val | series | ts | ++------+--------+---------------------+ +| 52.0 | nano | 1970-01-01T00:00:01 | ++------+--------+---------------------+ + +DROP TABLE instant_last_sec; + +Affected Rows: 0 + +DROP TABLE instant_last_micro; + +Affected Rows: 0 + +DROP TABLE instant_last_nano; + +Affected Rows: 0 + +-- A field selector chooses the val column; it does not filter val. PromQL must +-- apply the comparison after selecting the latest eligible sample, rather than +-- falling back to an older sample that satisfies the comparison. +CREATE TABLE instant_last_field_filter ( + ts TIMESTAMP TIME INDEX, + val DOUBLE, + host STRING, + instance STRING, + PRIMARY KEY (host, instance) +) ENGINE=mito; + +Affected Rows: 0 + +-- Keep the older matching sample in an SST and the latest non-matching sample +-- in the memtable. Both timestamps are eligible at 1s. +INSERT INTO instant_last_field_filter VALUES (900, 10, 'host-a', 'instance-a'); + +Affected Rows: 1 + +ADMIN FLUSH_TABLE('instant_last_field_filter'); + ++------------------------------------------------+ +| ADMIN FLUSH_TABLE('instant_last_field_filter') | ++------------------------------------------------+ +| 0 | ++------------------------------------------------+ + +INSERT INTO instant_last_field_filter VALUES (1000, 1, 'host-a', 'instance-a'); + +Affected Rows: 1 + +-- Expected: no rows. The selected latest value is 1, so it must not fall back +-- to the older value 10 merely because that value is greater than 5. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1, 1, '1s') instant_last_field_filter{__field__="val"} > 5; + +++ +++ + +-- A value matcher filters the scan before selecting a sample: the older 10 +-- remains eligible, unlike the post-selection comparison above. +TQL EVAL (1, 1, '1s') instant_last_field_filter{val="10.0"}; + ++---------------------+------+--------+------------+ +| ts | val | host | instance | ++---------------------+------+--------+------------+ +| 1970-01-01T00:00:01 | 10.0 | host-a | instance-a | ++---------------------+------+--------+------------+ + +-- SQL filters rows before aggregation, so the older matching value remains. +-- Expected value: 10. +SELECT last_value(val ORDER BY ts) FROM instant_last_field_filter WHERE val > 5; + ++--------------------------------------------------------------------------------------------------+ +| last_value(instant_last_field_filter.val) ORDER BY [instant_last_field_filter.ts ASC NULLS LAST] | ++--------------------------------------------------------------------------------------------------+ +| 10.0 | ++--------------------------------------------------------------------------------------------------+ + +DROP TABLE instant_last_field_filter; + +Affected Rows: 0 + +-- Range evaluation is a control: range functions need their complete windows, +-- not a single last row. At 10s and 20s, [11s] contains two samples. +CREATE TABLE instant_last_range_control ( + ts TIMESTAMP TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +Affected Rows: 0 + +INSERT INTO instant_last_range_control VALUES + (0, 0, 'range'), + (10000, 10, 'range'), + (20000, 20, 'range'); + +Affected Rows: 3 + +ADMIN FLUSH_TABLE('instant_last_range_control'); + ++-------------------------------------------------+ +| ADMIN FLUSH_TABLE('instant_last_range_control') | ++-------------------------------------------------+ +| 0 | ++-------------------------------------------------+ + +-- Expected instant values at 0s, 10s, and 20s: 0, 10, and 20. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 20, '10s') instant_last_range_control; + ++---------------------+------+--------+ +| ts | val | series | ++---------------------+------+--------+ +| 1970-01-01T00:00:00 | 0.0 | range | +| 1970-01-01T00:00:10 | 10.0 | range | +| 1970-01-01T00:00:20 | 20.0 | range | ++---------------------+------+--------+ + +-- Expected last values at 0s, 10s, and 20s: 0, 10, and 20. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 20, '10s') last_over_time(instant_last_range_control[11s]); + ++---------------------+-----------------------------------+--------+ +| ts | prom_last_over_time(ts_range,val) | series | ++---------------------+-----------------------------------+--------+ +| 1970-01-01T00:00:00 | 0.0 | range | +| 1970-01-01T00:00:10 | 10.0 | range | +| 1970-01-01T00:00:20 | 20.0 | range | ++---------------------+-----------------------------------+--------+ + +-- At 10s, rate is 10/11 because counter-zero extrapolation stops at 0s; +-- at 20s it is 1. The 0s window has only one sample. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 20, '10s') rate(instant_last_range_control[11s]); + ++---------------------+-----------------------------------------+--------+ +| ts | prom_rate(ts_range,val,ts,Int64(11000)) | series | ++---------------------+-----------------------------------------+--------+ +| 1970-01-01T00:00:10 | 0.9090909090909092 | range | +| 1970-01-01T00:00:20 | 1.0 | range | ++---------------------+-----------------------------------------+--------+ + +-- Expected rate at 20s: 1 from both points in the [11s] window. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (20, 20, '1s') rate(instant_last_range_control[11s]); + ++---------------------+-----------------------------------------+--------+ +| ts | prom_rate(ts_range,val,ts,Int64(11000)) | series | ++---------------------+-----------------------------------------+--------+ +| 1970-01-01T00:00:20 | 1.0 | range | ++---------------------+-----------------------------------------+--------+ + +DROP TABLE instant_last_range_control; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/promql/instant_last_row.sql b/tests/cases/standalone/common/promql/instant_last_row.sql new file mode 100644 index 0000000000..9df036f2ee --- /dev/null +++ b/tests/cases/standalone/common/promql/instant_last_row.sql @@ -0,0 +1,258 @@ +-- Correctness coverage for the instant-selector last-row optimization. +-- Each TQL EVAL below has distinct values so its chosen physical sample is visible. + +-- The default instant lookback is (T - 300s, T]: exclude its lower bound, +-- include a sample just inside it and at T, and exclude a future sample. +CREATE TABLE instant_last_lookback ( + ts TIMESTAMP TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +INSERT INTO instant_last_lookback VALUES + (700000, 10, 'lower_excluded'), + (700001, 11, 'just_inside'), + (1000000, 12, 'upper_included'), + (1000000, 14, 'future_has_prior'), + (1000001, 13, 'future_has_prior'); +ADMIN FLUSH_TABLE('instant_last_lookback'); + +-- Expected pairs: just_inside=11, upper_included=12, future_has_prior=14. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_lookback; + +-- An empty table must remain an empty instant vector. +CREATE TABLE instant_last_empty ( + ts TIMESTAMP TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +-- Expected: no rows. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_empty; + +DROP TABLE instant_last_empty; +DROP TABLE instant_last_lookback; + +-- Positive offset reads earlier data; negative offset reads later data. Repeat +-- the earlier evaluation after a later one to catch a cached instant window. +CREATE TABLE instant_last_offset ( + ts TIMESTAMP TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +INSERT INTO instant_last_offset VALUES + (940000, 21, 'offset'), + (1000000, 22, 'offset'), + (1060000, 23, 'offset'); +ADMIN FLUSH_TABLE('instant_last_offset'); + +-- Expected value: 22. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_offset; + +-- Expected value: 21 (evaluation time shifted back 60s). +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_offset offset 60s; + +-- Expected value: 23 (evaluation time shifted forward 60s). +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_offset offset -60s; + +-- Expected value: 23. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1060, 1060, '1s') instant_last_offset; + +-- Expected value: 22 again, not the later evaluation's value. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_offset; + +DROP TABLE instant_last_offset; + +-- Put t1 and t2 in one SST. Deleting newest t2 must reveal t1 whether the +-- Delete is still in the memtable or has been flushed to a newer SST. +CREATE TABLE instant_last_delete ( + ts TIMESTAMP TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +INSERT INTO instant_last_delete VALUES + (900000, 31, 'delete'), + (1000000, 32, 'delete'); +ADMIN FLUSH_TABLE('instant_last_delete'); + +DELETE FROM instant_last_delete WHERE series = 'delete' AND ts = 1000000; + +-- Expected value: 31; the newest point is a memtable Delete. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_delete; + +ADMIN FLUSH_TABLE('instant_last_delete'); + +-- Expected value: 31; the Delete is now in an SST. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_delete; + +-- A same-timestamp memtable overwrite of t1 must win over the old SST value. +INSERT INTO instant_last_delete VALUES (900000, 33, 'delete'); + +-- Expected value: 33. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_delete; + +-- A newer memtable point must win, and retain that identity after its flush. +INSERT INTO instant_last_delete VALUES (1010000, 34, 'delete'); + +-- Expected value: 34. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1010, 1010, '1s') instant_last_delete; + +ADMIN FLUSH_TABLE('instant_last_delete'); + +-- Expected value: 33 after flush; it overwrites t1 in an older SST. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_delete; + +-- Expected value: 34 after flush. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1010, 1010, '1s') instant_last_delete; + +DROP TABLE instant_last_delete; + +-- Ordinary IEEE NaN is a valid PromQL sample and must not be suppressed. +CREATE TABLE instant_last_nan ( + ts TIMESTAMP TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +INSERT INTO instant_last_nan VALUES (1000000, 'NaN'::DOUBLE, 'ordinary_nan'); +ADMIN FLUSH_TABLE('instant_last_nan'); + +-- Expected value: NaN, with the ordinary_nan series present. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1000, 1000, '1s') instant_last_nan; + +DROP TABLE instant_last_nan; + +-- PromQL normalizes to milliseconds; these distinct raw timestamps collide at +-- 1s. LastRow must preserve the unhinted path's selected samples: 42 and 52. +CREATE TABLE instant_last_sec ( + ts TIMESTAMP(0) TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +INSERT INTO instant_last_sec VALUES + (0, 61, 'sec'), + (1, 62, 'sec'), + (2, 63, 'sec'); +ADMIN FLUSH_TABLE('instant_last_sec'); + +-- Expected value: 62 at the native seconds boundary. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1, 1, '1s') instant_last_sec; + +CREATE TABLE instant_last_micro ( + ts TIMESTAMP(6) TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +INSERT INTO instant_last_micro VALUES + (999999, 41, 'micro'), + (1000000, 42, 'micro'), + (1000001, 43, 'micro'); +ADMIN FLUSH_TABLE('instant_last_micro'); + +-- Expected value: 42, matching the unhinted millisecond-normalized path. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1, 1, '1s') instant_last_micro; + +CREATE TABLE instant_last_nano ( + ts TIMESTAMP(9) TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +INSERT INTO instant_last_nano VALUES + (999999999, 51, 'nano'), + (1000000000, 52, 'nano'), + (1000000001, 53, 'nano'); +ADMIN FLUSH_TABLE('instant_last_nano'); + +-- Expected value: 52, matching the unhinted millisecond-normalized path. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1, 1, '1s') instant_last_nano; + +DROP TABLE instant_last_sec; +DROP TABLE instant_last_micro; +DROP TABLE instant_last_nano; + +-- A field selector chooses the val column; it does not filter val. PromQL must +-- apply the comparison after selecting the latest eligible sample, rather than +-- falling back to an older sample that satisfies the comparison. +CREATE TABLE instant_last_field_filter ( + ts TIMESTAMP TIME INDEX, + val DOUBLE, + host STRING, + instance STRING, + PRIMARY KEY (host, instance) +) ENGINE=mito; + +-- Keep the older matching sample in an SST and the latest non-matching sample +-- in the memtable. Both timestamps are eligible at 1s. +INSERT INTO instant_last_field_filter VALUES (900, 10, 'host-a', 'instance-a'); +ADMIN FLUSH_TABLE('instant_last_field_filter'); +INSERT INTO instant_last_field_filter VALUES (1000, 1, 'host-a', 'instance-a'); + +-- Expected: no rows. The selected latest value is 1, so it must not fall back +-- to the older value 10 merely because that value is greater than 5. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (1, 1, '1s') instant_last_field_filter{__field__="val"} > 5; + +-- A value matcher filters the scan before selecting a sample: the older 10 +-- remains eligible, unlike the post-selection comparison above. +TQL EVAL (1, 1, '1s') instant_last_field_filter{val="10.0"}; + +-- SQL filters rows before aggregation, so the older matching value remains. +-- Expected value: 10. +SELECT last_value(val ORDER BY ts) FROM instant_last_field_filter WHERE val > 5; + +DROP TABLE instant_last_field_filter; + +-- Range evaluation is a control: range functions need their complete windows, +-- not a single last row. At 10s and 20s, [11s] contains two samples. +CREATE TABLE instant_last_range_control ( + ts TIMESTAMP TIME INDEX, + val DOUBLE, + series STRING PRIMARY KEY +) ENGINE=mito; + +INSERT INTO instant_last_range_control VALUES + (0, 0, 'range'), + (10000, 10, 'range'), + (20000, 20, 'range'); +ADMIN FLUSH_TABLE('instant_last_range_control'); + +-- Expected instant values at 0s, 10s, and 20s: 0, 10, and 20. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 20, '10s') instant_last_range_control; + +-- Expected last values at 0s, 10s, and 20s: 0, 10, and 20. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 20, '10s') last_over_time(instant_last_range_control[11s]); + +-- At 10s, rate is 10/11 because counter-zero extrapolation stops at 0s; +-- at 20s it is 1. The 0s window has only one sample. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (0, 20, '10s') rate(instant_last_range_control[11s]); + +-- Expected rate at 20s: 1 from both points in the [11s] window. +-- SQLNESS SORT_RESULT 3 1 +TQL EVAL (20, 20, '1s') rate(instant_last_range_control[11s]); + +DROP TABLE instant_last_range_control; diff --git a/tests/cases/standalone/common/promql/regex.result b/tests/cases/standalone/common/promql/regex.result index 1c2dd7b651..bccbd0f340 100644 --- a/tests/cases/standalone/common/promql/regex.result +++ b/tests/cases/standalone/common/promql/regex.result @@ -75,7 +75,7 @@ TQL ANALYZE VERBOSE (0, 0, '1s') test{host=~".*"}; | 1_| 0_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[1000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["host"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["ts", "host", "val"], "filters": ["ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(0, None)"], "REDACTED +|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "host", "val"], "filters": ["ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(0, None)"], "REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -99,7 +99,7 @@ TQL ANALYZE VERBOSE (0, 0, '1s') test{host=~".+"}; | 1_| 0_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[1000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["host"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["ts", "host", "val"], "filters": ["host != Dictionary(UInt32, Utf8(\"\"))", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(0, None)"], "REDACTED +|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "host", "val"], "filters": ["host != Dictionary(UInt32, Utf8(\"\"))", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(0, None)"], "REDACTED |_|_|_| |_|_| Total rows: 2_| +-+-+-+ @@ -145,7 +145,7 @@ TQL ANALYZE VERBOSE (0, 0, '1s') test{host!~".+"}; | 1_| 0_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[1000], time index=[ts] REDACTED |_|_|_PromSeriesDivideExec: tags=["host"] REDACTED |_|_|_CooperativeExec REDACTED -|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "distribution":"PerSeries", "projection": ["ts", "host", "val"], "filters": ["host = Dictionary(UInt32, Utf8(\"\"))", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(0, None)"], "REDACTED +|_|_|_SeriesScan: region=REDACTED, {"partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "projection": ["ts", "host", "val"], "filters": ["host = Dictionary(UInt32, Utf8(\"\"))", "ts >= TimestampMillisecond(-299999, None)", "ts <= TimestampMillisecond(0, None)"], "REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ diff --git a/tests/cases/standalone/optimizer/last_value.result b/tests/cases/standalone/optimizer/last_value.result index 42d08f5b12..55ecfc8bb5 100644 --- a/tests/cases/standalone/optimizer/last_value.result +++ b/tests/cases/standalone/optimizer/last_value.result @@ -87,7 +87,7 @@ explain analyze |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -139,7 +139,7 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":1, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ diff --git a/tests/cases/standalone/optimizer/last_value_advance.result b/tests/cases/standalone/optimizer/last_value_advance.result index 3199692a73..15e5ca80cb 100644 --- a/tests/cases/standalone/optimizer/last_value_advance.result +++ b/tests/cases/standalone/optimizer/last_value_advance.result @@ -113,7 +113,7 @@ explain analyze |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -165,7 +165,7 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[last_value(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":{"count":1, "mem_ranges":0, "files":1, "file_ranges":1}, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -276,7 +276,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| | 1_| 1_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ordered_host@0 ASC NULLS LAST], preserve_REDACTED @@ -284,7 +284,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| | 1_| 2_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ordered_host@0 ASC NULLS LAST], preserve_REDACTED @@ -292,7 +292,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@1 as host], aggr=[last_value(t.host) ORDER BY [t.ts ASC NULLS LAST], last_value(t.not_pk) ORDER BY [t.ts ASC NULLS LAST], last_value(t.val) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ @@ -352,17 +352,17 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t.ts) ORDER BY [t.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -556,17 +556,17 @@ explain analyze | 1_| 0_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| | 1_| 2_|_AggregateExec: mode=Final, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_CoalescePartitionsExec REDACTED |_|_|_AggregateExec: mode=Partial, gby=[], aggr=[__last_value_state(t1.ts) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| |_|_| Total rows: 1_| +-+-+-+ @@ -644,7 +644,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| | 1_| 1_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ordered_host@0 ASC NULLS LAST], preserve_REDACTED @@ -652,7 +652,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| | 1_| 2_|_SortPreservingMergeExec: [ordered_host@0 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[ordered_host@0 ASC NULLS LAST], preserve_REDACTED @@ -660,7 +660,7 @@ order by ordered_host; |_|_|_AggregateExec: mode=FinalPartitioned, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED |_|_|_RepartitionExec: REDACTED |_|_|_AggregateExec: mode=Partial, gby=[host@0 as host], aggr=[last_value(t1.host) ORDER BY [t1.ts ASC NULLS LAST], last_value(t1.val) ORDER BY [t1.ts ASC NULLS LAST]] REDACTED -|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow" REDACTED +|_|_|_SeqScan: region=REDACTED, "partition_count":REDACTED, "selector":"LastRow { after_merge: false }" REDACTED |_|_|_| |_|_| Total rows: 4_| +-+-+-+ diff --git a/tests/cases/standalone/tql-explain-analyze/analyze.result b/tests/cases/standalone/tql-explain-analyze/analyze.result index 954020ea4c..cbbce8eaf1 100644 --- a/tests/cases/standalone/tql-explain-analyze/analyze.result +++ b/tests/cases/standalone/tql-explain-analyze/analyze.result @@ -297,7 +297,7 @@ TQL ANALYZE sum(test2); |_|_|_ProjectionExec: expr=[greptime_timestamp@0 as greptime_timestamp, greptime_value@1 as greptime_value] REDACTED |_|_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesDivideExec: tags=["shard"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "mode":"legacy" REDACTED |_|_|_| | 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[greptime_timestamp@0 as greptime_timestamp], aggr=[__sum_state(test2.greptime_value)] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED @@ -305,7 +305,7 @@ TQL ANALYZE sum(test2); |_|_|_ProjectionExec: expr=[greptime_timestamp@0 as greptime_timestamp, greptime_value@1 as greptime_value] REDACTED |_|_|_PromInstantManipulateExec: range=[0..0], lookback=[300000], interval=[300000], time index=[greptime_timestamp] REDACTED |_|_|_PromSeriesDivideExec: tags=["shard"] REDACTED -|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED +|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "selector":"LastRow { after_merge: true }", "distribution":"PerSeries", "mode":"legacy" REDACTED |_|_|_| |_|_| Total rows: 0_| +-+-+-+ diff --git a/tests/perf/README.md b/tests/perf/README.md index 42fd8c286e..d518fec996 100644 --- a/tests/perf/README.md +++ b/tests/perf/README.md @@ -204,7 +204,7 @@ case for issue #7913. It writes 8192 series × 20160 samples through remote-writ in 1440-sample daily time chunks, flushing after each chunk before running 1d/7d/14d TQL selectors. It is not included in the default `all` case set because ingestion cost dominates routine CI validation. Commenting `/query-regression heavy` runs -only this case; `/query-regression` runs the six routine default cases. Manual +only this case; `/query-regression` runs the seven routine default cases. Manual workflow dispatch accepts the `heavy` token to select this case. ## OTLP trace load scenario @@ -393,12 +393,12 @@ parquetbench/scanbench` as the read-bench tool against each target's data direct The workflow runs when an allowlisted repository admin comments `/query-regression` on a non-draft PR. It does not rerun on pushes, -ready-for-review, or reopen events. `/query-regression` runs the six routine -default cases; `/query-regression heavy` runs only the high-cardinality -remote-write #7913 case. PR runs build base/candidate once and -use `--allow-large-fixture`. Manual `workflow_dispatch` runs can pass `all`, -`heavy`, one case path, or a comma/whitespace-separated list of case paths, and -can override refs. +ready-for-review, or reopen events. `/query-regression` runs the seven routine +default cases, including `promql_instant_last_row_9034`; +`/query-regression heavy` runs only the high-cardinality remote-write #7913 case. +PR runs build base/candidate once and use `--allow-large-fixture`. Manual +`workflow_dispatch` runs can pass `all`, `heavy`, one case path, or a +comma/whitespace-separated list of case paths, and can override refs. Comment admission is two workflows. `slash-command-dispatch.yml` uses [peter-evans/slash-command-dispatch](https://github.com/peter-evans/slash-command-dispatch) diff --git a/tests/perf/fixture-format.md b/tests/perf/fixture-format.md index 361f0ec79f..c7d306d852 100644 --- a/tests/perf/fixture-format.md +++ b/tests/perf/fixture-format.md @@ -77,7 +77,12 @@ cycles series labels across rows. `series_layout = "timestamp_major"` writes all series for one timestamp before advancing to the next timestamp; use it for Prometheus-like high-cardinality scrape fixtures where short query windows should still contain many raw samples. `timestamp_major` requires `rows_per_sst` to be -divisible by `series_count`. +divisible by `series_count`. `per_sst` uses one series selected from the SST +index. These describe logical generation order. Before either flat or primary-key +SST format is written, each completed generated batch is physically sorted by +encoded primary key ascending, timestamp ascending, and sequence descending, as +required by the Mito Parquet writer; sorting preserves every generated column and +row, including deterministic-wave values. `[scenario]` is required. Other scenario variants are intentionally unsupported for now, but `scenario.kind` leaves room for future `write_then_query` and diff --git a/tests/perf/query_cases/promql_instant_last_row_9034/case.toml b/tests/perf/query_cases/promql_instant_last_row_9034/case.toml new file mode 100644 index 0000000000..6e61a094b3 --- /dev/null +++ b/tests/perf/query_cases/promql_instant_last_row_9034/case.toml @@ -0,0 +1,92 @@ +# PromQL instant-selector latency regression coverage for #9034. +# +# Instant, range, and aggregate controls cover LastRow latency behavior. The +# candidate instant query must exercise the existing LastRow selector; the range +# query must not. This is latency regression coverage only: optimizer and +# sqlness tests own plan and correctness coverage. Evaluation is 0.7s after the +# final generated sample (last timestamp = start + 1023 * 0.1s = +# 1704067302.3) and remains inside the default lookback window. + +[case] +name = "promql_instant_last_row_9034" +description = "PromQL instant selector LastRow latency regression for #9034" + +[scenario] +kind = "direct_readable_sst" +seed = 9034 + +[[scenario.tables]] +database = "public" +name = "promql_instant_last_row_9034" +engine = "mito" +append_mode = true +sst_format = "flat" +primary_key = ["host", "instance"] +time_index = "ts" + +[[scenario.tables.columns]] +name = "host" +type = "STRING" +semantic = "tag" +distribution = { kind = "cardinality", values = 16, prefix = "host" } + +[[scenario.tables.columns]] +name = "instance" +type = "STRING" +semantic = "tag" +distribution = { kind = "cardinality", values = 256, prefix = "instance" } + +[[scenario.tables.columns]] +name = "value" +type = "DOUBLE" +semantic = "field" +distribution = { kind = "deterministic_wave", min = 0.0, max = 1000.0 } + +[[scenario.tables.columns]] +name = "ts" +type = "TIMESTAMP(3)" +semantic = "timestamp" + +[scenario.layout] +regions = 1 +sst_count = 64 +rows_per_sst = 4096 +row_group_size = 1024 +series_count = 256 +start_unix_nanos = 1704067200000000000 +step_nanos = 100000000 +time_range_layout = "non_overlapping_per_sst" +series_layout = "timestamp_major" + +# Candidate instant query must exercise the existing LastRow selector. +[[scenario.queries]] +name = "instant_selector_last_row_candidate" +kind = "tql" +query = "TQL ANALYZE VERBOSE (1704067303, 1704067303, '1s') promql_instant_last_row_9034{host=~'host.*'}" +warmup = 3 +iterations = 9 + +[scenario.queries.thresholds] +max_candidate_latency_regression_pct = 25 + +# True range control: this must not enable the LastRow selector. +[[scenario.queries]] +name = "range_selector_last_row_control" +kind = "tql" +query = "TQL ANALYZE VERBOSE (1704067293, 1704067303, '1s') promql_instant_last_row_9034{host=~'host.*'}" +warmup = 3 +iterations = 9 + +[scenario.queries.thresholds] +max_candidate_latency_regression_pct = 25 + +# This pre-existing aggregate-hint control should return one row per 256 series; it is latency regression coverage, not a result oracle. +[[scenario.queries]] +name = "sql_aggregate_last_row_control" +kind = "sql" +query = "SELECT host, instance, last_value(value ORDER BY ts ASC) FROM promql_instant_last_row_9034 GROUP BY host, instance" +warmup = 3 +iterations = 9 + +[scenario.queries.thresholds] +max_candidate_latency_regression_pct = 25 diff --git a/tests/perf/test_query_regression_case_selection.py b/tests/perf/test_query_regression_case_selection.py index 642b6ee1f2..84b3cd44fa 100644 --- a/tests/perf/test_query_regression_case_selection.py +++ b/tests/perf/test_query_regression_case_selection.py @@ -40,6 +40,7 @@ class QueryRegressionCaseSelectionTest(unittest.TestCase): "tests/perf/query_cases/prom_remote_write_mixed_every/case.toml", "tests/perf/query_cases/prom_remote_write_integer_counter/case.toml", "tests/perf/query_cases/promql_range_boundary/case.toml", + "tests/perf/query_cases/promql_instant_last_row_9034/case.toml", ], )