From 02283b6ba0493206b1723d749ab3820ec1f19f0b Mon Sep 17 00:00:00 2001 From: discord9 Date: Fri, 10 Jul 2026 20:27:27 +0800 Subject: [PATCH 1/8] test(perf): add remote write storage inspection (#8444) * feat: add remote write value distributions Signed-off-by: discord9 * test: extend remote write value distributions Signed-off-by: discord9 * test: inspect remote write parquet storage Signed-off-by: discord9 * test: normalize remote write perf fixtures Signed-off-by: discord9 * test: add heavy remote write perf case Signed-off-by: discord9 * test: use head greptime for read bench Signed-off-by: discord9 * test: keep heavy remote write case local Signed-off-by: discord9 * test: tune remote write perf smoke case Signed-off-by: discord9 * test: fix query fixture import style Signed-off-by: discord9 * test: cover remote write value distributions Signed-off-by: discord9 * test: add integer counter perf case Signed-off-by: discord9 --------- Signed-off-by: discord9 --- .github/scripts/query-regression-run.py | 14 +- .github/workflows/query-regression.yml | 16 +- src/cmd/Cargo.toml | 7 +- src/cmd/src/bin/greptime.rs | 1 - src/cmd/src/bin/prom_remote_write_fixture.rs | 144 ---- src/cmd/src/bin/query_perf_fixture.rs | 652 +----------------- src/cmd/src/bin/query_perf_fixture/case.rs | 470 +++++++++++++ .../src/bin/query_perf_fixture/direct_sst.rs | 529 ++++++++++++++ .../bin/query_perf_fixture/inspect_footer.rs | 181 +++++ src/cmd/src/bin/query_perf_fixture/mod.rs | 133 ++++ .../query_perf_fixture/prom_remote_write.rs | 333 +++++++++ src/cmd/src/bin/query_perf_fixture/util.rs | 23 + src/cmd/src/datanode.rs | 9 +- tests/perf/AGENTS.md | 8 +- tests/perf/README.md | 166 ++++- .../case.toml | 48 ++ .../prom_remote_write_mixed_every/case.toml | 49 ++ .../prom_remote_write_run_heavy/case.toml | 50 ++ .../prom_remote_write_seeded_random/case.toml | 48 ++ .../prom_remote_write_smoke/case.toml | 31 - tests/perf/query_regression_runner.py | 377 ++++++++-- ...ery_regression_runner_compaction_toctou.py | 158 +++++ 22 files changed, 2520 insertions(+), 927 deletions(-) delete mode 100644 src/cmd/src/bin/prom_remote_write_fixture.rs create mode 100644 src/cmd/src/bin/query_perf_fixture/case.rs create mode 100644 src/cmd/src/bin/query_perf_fixture/direct_sst.rs create mode 100644 src/cmd/src/bin/query_perf_fixture/inspect_footer.rs create mode 100644 src/cmd/src/bin/query_perf_fixture/mod.rs create mode 100644 src/cmd/src/bin/query_perf_fixture/prom_remote_write.rs create mode 100644 src/cmd/src/bin/query_perf_fixture/util.rs create mode 100644 tests/perf/query_cases/prom_remote_write_integer_counter/case.toml create mode 100644 tests/perf/query_cases/prom_remote_write_mixed_every/case.toml create mode 100644 tests/perf/query_cases/prom_remote_write_run_heavy/case.toml create mode 100644 tests/perf/query_cases/prom_remote_write_seeded_random/case.toml delete mode 100644 tests/perf/query_cases/prom_remote_write_smoke/case.toml create mode 100644 tests/perf/test_query_regression_runner_compaction_toctou.py diff --git a/.github/scripts/query-regression-run.py b/.github/scripts/query-regression-run.py index b89857be81f..62d11f44203 100644 --- a/.github/scripts/query-regression-run.py +++ b/.github/scripts/query-regression-run.py @@ -26,6 +26,10 @@ from pathlib import Path DEFAULT_CASES = [ "tests/perf/query_cases/smoke_direct_sst/case.toml", + "tests/perf/query_cases/prom_remote_write_seeded_random/case.toml", + "tests/perf/query_cases/prom_remote_write_run_heavy/case.toml", + "tests/perf/query_cases/prom_remote_write_mixed_every/case.toml", + "tests/perf/query_cases/prom_remote_write_integer_counter/case.toml", "tests/perf/query_cases/promql_pushdown_7913/case.toml", "tests/perf/query_cases/sql_topk_order_by/case.toml", "tests/perf/query_cases/sql_aggregate_order_by/case.toml", @@ -106,8 +110,6 @@ def run_case(args: argparse.Namespace, case_path: Path, work_dir: Path) -> int: str(candidate_bin), "--fixture-generator", str(fixture_generator), - "--remote-write-generator", - str(args.remote_write_generator), "--work-dir", str(work_dir), "--http-timeout", @@ -160,11 +162,6 @@ def main() -> int: parser.add_argument("--work-dir", default=Path("query-regression-work"), type=Path) parser.add_argument("--http-timeout", default=os.environ.get("HTTP_TIMEOUT", "300")) parser.add_argument("--allow-large-fixture", default=os.environ.get("ALLOW_LARGE_FIXTURE", "false")) - parser.add_argument( - "--remote-write-generator", - type=Path, - default=configured_path(os.environ.get("REMOTE_WRITE_GENERATOR")), - ) parser.add_argument( "--summary-script", type=Path, @@ -177,9 +174,6 @@ def main() -> int: parser.add_argument("--candidate-ref", default=os.environ.get("CANDIDATE_REF", "")) parser.add_argument("--github-output", default=os.environ.get("GITHUB_OUTPUT")) args = parser.parse_args() - if args.remote_write_generator is None: - args.remote_write_generator = args.candidate_src / "target" / profile_dir(args.cargo_profile) / "prom_remote_write_fixture" - try: cases = split_cases(args.cases or [os.environ.get("CASE_PATHS", "all")]) except ValueError as err: diff --git a/.github/workflows/query-regression.yml b/.github/workflows/query-regression.yml index a898be6ad29..619442e2f95 100644 --- a/.github/workflows/query-regression.yml +++ b/.github/workflows/query-regression.yml @@ -119,10 +119,15 @@ jobs: git reset --hard FETCH_HEAD git clean -ffdx + - name: Test query regression runner lifecycle + working-directory: src + run: uv run --no-project python tests/perf/test_query_regression_runner_compaction_toctou.py + - name: Build candidate greptime and fixture generators working-directory: src run: | - cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime --bin query_perf_fixture --bin prom_remote_write_fixture + cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime + cargo build --profile "${CARGO_PROFILE}" -p cmd --bin query_perf_fixture --features dev-tools target_dir="${CARGO_PROFILE}" if [[ "${CARGO_PROFILE}" == "dev" ]]; then target_dir="debug" @@ -132,8 +137,6 @@ jobs: "${GITHUB_WORKSPACE}/query-regression-bins/candidate/greptime" cp "${CARGO_TARGET_DIR}/${target_dir}/query_perf_fixture" \ "${GITHUB_WORKSPACE}/query-regression-bins/candidate/query_perf_fixture" - cp "${CARGO_TARGET_DIR}/${target_dir}/prom_remote_write_fixture" \ - "${GITHUB_WORKSPACE}/query-regression-bins/candidate/prom_remote_write_fixture" - name: Run query regression id: run @@ -146,7 +149,6 @@ jobs: BASE_BIN: ${{ github.workspace }}/query-regression-bins/base/greptime CANDIDATE_BIN: ${{ github.workspace }}/query-regression-bins/candidate/greptime FIXTURE_GENERATOR: ${{ github.workspace }}/query-regression-bins/candidate/query_perf_fixture - REMOTE_WRITE_GENERATOR: ${{ github.workspace }}/query-regression-bins/candidate/prom_remote_write_fixture run: >- uv run --no-project python src/.github/scripts/query-regression-run.py --base-src src @@ -171,9 +173,13 @@ jobs: with: name: query-regression-report path: | - query-regression-work/** + query-regression-work/**/query-regression-report.json + query-regression-work/**/base/report.json + query-regression-work/**/candidate/report.json + query-regression-work/**/logs/** query-regression-summary.md if-no-files-found: warn + retention-days: 7 - name: Upload trusted comment artifact if: ${{ always() && github.event_name == 'pull_request' }} diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml index a8adfa01a48..b06da6e517a 100644 --- a/src/cmd/Cargo.toml +++ b/src/cmd/Cargo.toml @@ -12,10 +12,7 @@ path = "src/bin/greptime.rs" [[bin]] name = "query_perf_fixture" path = "src/bin/query_perf_fixture.rs" - -[[bin]] -name = "prom_remote_write_fixture" -path = "src/bin/prom_remote_write_fixture.rs" +required-features = ["dev-tools"] [features] default = [ @@ -25,7 +22,7 @@ default = [ "meta-srv/mysql_kvbackend", ] enterprise = ["common-meta/enterprise", "frontend/enterprise", "meta-srv/enterprise"] -# Developer-only tools (benchmarks, diagnostics) gating the `datanode parquetbench` subcommand. +# Developer-only helper binary gate for `query_perf_fixture`. # Kept out of `default` so normal/release builds don't compile them. dev-tools = [] tokio-console = ["common-telemetry/tokio-console"] diff --git a/src/cmd/src/bin/greptime.rs b/src/cmd/src/bin/greptime.rs index e0f326908a2..957e227355a 100644 --- a/src/cmd/src/bin/greptime.rs +++ b/src/cmd/src/bin/greptime.rs @@ -117,7 +117,6 @@ async fn start(cli: Command) -> Result<()> { } datanode::SubCommand::Objbench(ref bench) => bench.run().await, datanode::SubCommand::Scanbench(ref bench) => bench.run().await, - #[cfg(feature = "dev-tools")] datanode::SubCommand::Parquetbench(ref bench) => bench.run().await, }, SubCommand::Flownode(cmd) => { diff --git a/src/cmd/src/bin/prom_remote_write_fixture.rs b/src/cmd/src/bin/prom_remote_write_fixture.rs deleted file mode 100644 index 0bfdf954823..00000000000 --- a/src/cmd/src/bin/prom_remote_write_fixture.rs +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#![allow(clippy::print_stdout)] - -use std::time::{Duration, Instant}; - -use api::prom_store::remote::{Label, Sample, TimeSeries, WriteRequest}; -use clap::Parser; -use prost::Message; -use serde_json::json; -use servers::prom_store::snappy_compress; - -#[derive(Debug, Parser)] -#[command(about = "Generate a deterministic Prometheus remote-write fixture")] -struct Args { - #[arg(long, default_value = "http://127.0.0.1:4000/v1/prometheus/write")] - endpoint: String, - #[arg(long, default_value = "public")] - database: String, - #[arg(long)] - metric: String, - #[arg(long, default_value = "greptime_physical_table")] - physical_table: String, - #[arg(long, default_value_t = 8)] - series_count: u64, - #[arg(long, default_value_t = 30)] - samples_per_series: u64, - #[arg(long, default_value_t = 1_704_067_200_000)] - start_unix_millis: i64, - #[arg(long, default_value_t = 15_000)] - step_millis: i64, - #[arg(long, alias = "batch-size", default_value_t = 8)] - chunk_series_count: u64, - #[arg(long, default_value_t = 60)] - timeout_seconds: u64, -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - let args = Args::parse(); - let started = Instant::now(); - let client = reqwest::Client::builder() - .timeout(Duration::from_secs(args.timeout_seconds)) - .build()?; - let mut batches = 0_u64; - let mut rows = 0_u64; - let mut http_statuses = Vec::new(); - - let chunk = args.chunk_series_count.max(1); - for first in (0..args.series_count).step_by(chunk as usize) { - let last = (first + chunk).min(args.series_count); - let timeseries = (first..last) - .map(|series_idx| make_series(&args, series_idx)) - .collect::>(); - rows += timeseries - .iter() - .map(|ts| ts.samples.len() as u64) - .sum::(); - let body = snappy_compress( - &WriteRequest { - timeseries, - ..Default::default() - } - .encode_to_vec(), - )?; - let response = client - .post(&args.endpoint) - .header("Content-Encoding", "snappy") - .body(body) - .send() - .await?; - let status = response.status(); - http_statuses.push(status.as_u16()); - if !status.is_success() { - let body_text = response.text().await.unwrap_or_default(); - return Err(format!("remote-write failed with status {status}: {body_text}").into()); - } - batches += 1; - } - - println!( - "{}", - json!({ - "status": "ok", - "endpoint": args.endpoint, - "database": args.database, - "metric": args.metric, - "physical_table": args.physical_table, - "series_count": args.series_count, - "samples_per_series": args.samples_per_series, - "rows": rows, - "samples_written": rows, - "batches": batches, - "elapsed_seconds": started.elapsed().as_secs_f64(), - "http_statuses": http_statuses, - }) - ); - Ok(()) -} - -fn make_series(args: &Args, series_idx: u64) -> TimeSeries { - let mut labels = vec![ - label("__name__", &args.metric), - label("x_greptime_database", &args.database), - label("x_greptime_physical_table", &args.physical_table), - label("host", &format!("host{:04}", series_idx % 1024)), - label("instance", &format!("instance{:06}", series_idx)), - ]; - labels.sort_unstable_by(|left, right| left.name.cmp(&right.name)); - let samples = (0..args.samples_per_series) - .map(|sample_idx| Sample { - value: deterministic_value(series_idx, sample_idx), - timestamp: args.start_unix_millis + (sample_idx as i64 * args.step_millis), - }) - .collect(); - TimeSeries { - labels, - samples, - ..Default::default() - } -} - -fn label(name: &str, value: &str) -> Label { - Label { - name: name.to_string(), - value: value.to_string(), - } -} - -fn deterministic_value(series_idx: u64, sample_idx: u64) -> f64 { - ((series_idx % 97) as f64) + (sample_idx as f64 * 0.125) -} diff --git a/src/cmd/src/bin/query_perf_fixture.rs b/src/cmd/src/bin/query_perf_fixture.rs index dfc3a664a03..d0e185038fa 100644 --- a/src/cmd/src/bin/query_perf_fixture.rs +++ b/src/cmd/src/bin/query_perf_fixture.rs @@ -14,656 +14,10 @@ #![allow(clippy::print_stderr, clippy::print_stdout)] -//! Generic direct-readable-SST fixture generator for query performance labs. -//! -//! This is a bounded MVP: it reads one table/one region case TOML, builds -//! synthetic region metadata, writes real Mito readable SST parquet files, and -//! emits a manifest checkpoint plus validation metadata. The first case uses -//! `sst_format = "flat"`; primary-key output is also accepted for early lab use -//! because the writer can convert the same flat input batch to primary-key SSTs. -//! -//! TODO(query-perf): add multi-region generation and seed-manifest/catalog DB -//! integration so the generated fixture can be mounted directly by a full -//! GreptimeDB open/query path without manual metadata wiring. - -use std::collections::HashMap; -use std::fs; -use std::io::{BufWriter, Write}; -use std::num::{NonZeroU64, NonZeroUsize}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use api::v1::{OpType, SemanticType}; -use async_trait::async_trait; -use clap::Parser; -use datatypes::arrow::array::{ - ArrayRef, BinaryDictionaryBuilder, Float64Array, RecordBatch, StringDictionaryBuilder, - TimestampMillisecondArray, TimestampNanosecondArray, UInt8Array, UInt64Array, -}; -use datatypes::arrow::datatypes::UInt32Type; -use datatypes::prelude::ConcreteDataType; -use datatypes::schema::{ColumnSchema, SkippingIndexOptions}; -use mito_codec::row_converter::{DensePrimaryKeyCodec, PrimaryKeyCodecExt, SortField}; -use mito2::access_layer::{FilePathProvider, Metrics, WriteType}; -use mito2::config::IndexConfig; -use mito2::manifest::action::{RegionCheckpoint, RegionManifest, RemovedFilesRecord}; -use mito2::read::FlatSource; -use mito2::sst::file::{FileMeta, RegionFileId}; -use mito2::sst::index::{Indexer, IndexerBuilder}; -use mito2::sst::parquet::writer::ParquetWriter; -use mito2::sst::parquet::{SstInfo, WriteOptions}; -use mito2::sst::{ - DEFAULT_WRITE_BUFFER_SIZE, FlatSchemaOptions, FormatType, to_flat_sst_arrow_schema, -}; -use object_store::ObjectStore; -use object_store::services::Fs as FsBuilder; -use serde::Deserialize; -use store_api::codec::PrimaryKeyEncoding; -use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataRef}; -use store_api::path_utils::region_name; -use store_api::region_request::PathType; -use store_api::storage::{FileId, RegionId}; - -#[derive(Parser, Debug)] -#[command(name = "query_perf_fixture")] -#[command(about = "Generate direct-readable SST fixtures for query performance cases")] -struct Args { - /// TOML case file, for example tests/perf/query_cases/.../case.toml. - #[arg(long, value_name = "PATH")] - case: PathBuf, - - /// Output directory (creates object-store/, manifest/, files.jsonl, summary.json). - #[arg(long, value_name = "DIR")] - out_dir: PathBuf, - - /// Region ID to synthesize. Defaults to table id 1024, region 0. - #[arg(long, default_value = "4398046511104")] - region_id: u64, - - /// Table directory relative to object-store root. Defaults to data/{database}/{table}/. - #[arg(long, value_name = "DIR")] - table_dir: Option, - - /// Table name to generate when the case contains multiple [[tables]]. - #[arg(long, value_name = "NAME")] - table: Option, - - /// Manifest/checkpoint version. - #[arg(long, default_value = "1000000")] - checkpoint_version: u64, - - /// Safety flag when sst_count exceeds 1000. - #[arg(long)] - allow_large: bool, - - /// Print plan only. - #[arg(long)] - dry_run: bool, -} - -#[derive(Debug, Deserialize)] -struct CaseFile { - scenario: Scenario, -} - -#[derive(Debug, Deserialize)] -#[serde(tag = "kind")] -enum Scenario { - #[serde(rename = "direct_readable_sst")] - DirectReadableSst(DirectReadableSstScenario), -} - -#[derive(Debug, Deserialize)] -struct DirectReadableSstScenario { - #[serde(default)] - seed: Option, - tables: Vec, - layout: LayoutConfig, -} - -impl Scenario { - fn kind(&self) -> &'static str { - match self { - Scenario::DirectReadableSst(_) => "direct_readable_sst", - } - } - - fn direct_readable_sst(&self) -> &DirectReadableSstScenario { - match self { - Scenario::DirectReadableSst(scenario) => scenario, - } - } -} - -#[derive(Debug, Deserialize)] -struct TableConfig { - database: String, - name: String, - engine: String, - #[serde(default)] - append_mode: Option, - #[serde(default)] - sst_format: Option, - primary_key: Vec, - time_index: String, - columns: Vec, -} - -#[derive(Debug, Deserialize)] -struct ColumnConfig { - name: String, - #[serde(rename = "type")] - ty: String, - semantic: String, - distribution: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(tag = "kind")] -enum Distribution { - #[serde(rename = "cardinality")] - Cardinality { - values: NonZeroUsize, - prefix: String, - }, - #[serde(rename = "deterministic_wave")] - DeterministicWave { min: f64, max: f64 }, -} - -#[derive(Debug, Deserialize)] -struct LayoutConfig { - regions: usize, - sst_count: usize, - rows_per_sst: usize, - row_group_size: usize, - series_count: NonZeroUsize, - start_unix_nanos: i64, - step_nanos: i64, - time_range_layout: String, - series_layout: String, -} - -struct NoopIndexBuilder; - -#[async_trait] -impl IndexerBuilder for NoopIndexBuilder { - async fn build(&self, _file_id: FileId, _index_version: u64) -> Indexer { - Indexer::default() - } -} - -#[derive(Clone)] -struct FixedPathProvider { - table_dir: String, -} - -impl FilePathProvider for FixedPathProvider { - fn build_index_file_path(&self, file_id: RegionFileId) -> String { - mito2::sst::location::index_file_path_legacy(&self.table_dir, file_id, PathType::Bare) - } - fn build_index_file_path_with_version( - &self, - index_id: mito2::sst::file::RegionIndexId, - ) -> String { - mito2::sst::location::index_file_path(&self.table_dir, index_id, PathType::Bare) - } - fn build_sst_file_path(&self, file_id: RegionFileId) -> String { - mito2::sst::location::sst_file_path(&self.table_dir, file_id, PathType::Bare) - } -} - -fn semantic_type(s: &str) -> SemanticType { - match s.to_ascii_lowercase().as_str() { - "tag" => SemanticType::Tag, - "field" => SemanticType::Field, - "timestamp" => SemanticType::Timestamp, - other => panic!("unsupported semantic {other}"), - } -} - -fn concrete_type(ty: &str) -> ConcreteDataType { - match ty.to_ascii_uppercase().as_str() { - "STRING" => ConcreteDataType::string_datatype(), - "DOUBLE" => ConcreteDataType::float64_datatype(), - "UINT64" => ConcreteDataType::uint64_datatype(), - "TIMESTAMP(9)" => ConcreteDataType::timestamp_nanosecond_datatype(), - "TIMESTAMP(3)" => ConcreteDataType::timestamp_millisecond_datatype(), - other => panic!("unsupported column type {other}"), - } -} - -fn build_region_metadata(table: &TableConfig, region_id: RegionId) -> RegionMetadata { - let mut builder = store_api::metadata::RegionMetadataBuilder::new(region_id); - for (idx, col) in table.columns.iter().enumerate() { - let mut schema = ColumnSchema::new( - &col.name, - concrete_type(&col.ty), - col.semantic != "timestamp", - ); - if col.semantic == "tag" { - schema = schema.with_inverted_index(true); - } - if col.semantic == "tag" || col.semantic == "field" { - schema = schema - .with_skipping_options(SkippingIndexOptions { - granularity: 1, - ..Default::default() - }) - .expect("valid skipping index options for query perf fixture"); - } - if col.name == table.time_index { - schema = schema.with_time_index(true); - } - builder.push_column_metadata(ColumnMetadata { - column_schema: schema, - semantic_type: semantic_type(&col.semantic), - column_id: idx as u32, - }); - } - let pk_ids = table - .primary_key - .iter() - .map(|name| { - table - .columns - .iter() - .position(|c| c.name == *name) - .unwrap_or_else(|| panic!("primary key column {name} not found")) as u32 - }) - .collect(); - builder.primary_key(pk_ids); - builder.primary_key_encoding(PrimaryKeyEncoding::Dense); - builder - .build() - .expect("region metadata should be valid for fixture table config") -} - -fn encode_dense_primary_key(table: &TableConfig, values: &HashMap) -> Vec { - let fields = table - .primary_key - .iter() - .enumerate() - .map(|(idx, _)| { - ( - idx as u32, - SortField::new(ConcreteDataType::string_datatype()), - ) - }) - .collect(); - let converter = DensePrimaryKeyCodec::with_fields(fields); - converter - .encode( - table - .primary_key - .iter() - .map(|name| datatypes::value::ValueRef::String(values[name].as_str())), - ) - .expect("dense primary key encoding should match fixture primary key fields") -} - -fn tag_value(col: &ColumnConfig, series: usize) -> String { - match col - .distribution - .as_ref() - .expect("tag distribution is required") - { - Distribution::Cardinality { values, prefix } => { - format!("{}{}", prefix, series % values.get()) - } - _ => panic!("tag column {} requires cardinality distribution", col.name), - } -} - -fn series_for_row(layout: &LayoutConfig, sst_idx: usize, base_row: usize, row: usize) -> usize { - match layout.series_layout.as_str() { - "round_robin" | "timestamp_major" => (base_row + row) % layout.series_count.get(), - "per_sst" => sst_idx % layout.series_count.get(), - other => panic!("unsupported series_layout {other}"), - } -} - -fn timestamp_for_row(layout: &LayoutConfig, base_row: usize, row: usize) -> i64 { - let logical_row = base_row + row; - let timestamp_step = if layout.series_layout == "timestamp_major" { - logical_row / layout.series_count.get() - } else { - logical_row - }; - - layout.start_unix_nanos + (timestamp_step as i64 * layout.step_nanos) -} - -fn wave_value(min: f64, max: f64, row: usize) -> f64 { - let span = max - min; - let phase = (row % 1024) as f64 / 1023.0; - min + span * (0.5 - 0.5 * (std::f64::consts::TAU * phase).cos()) -} - -fn generate_record_batch( - table: &TableConfig, - metadata: &RegionMetadataRef, - layout: &LayoutConfig, - sst_idx: usize, - sequence: u64, -) -> RecordBatch { - let flat_schema = to_flat_sst_arrow_schema(metadata, &FlatSchemaOptions::default()); - let rows = layout.rows_per_sst; - let mut columns = Vec::new(); - let base_row = sst_idx * rows; - - for col in &table.columns { - match (col.semantic.as_str(), col.ty.to_ascii_uppercase().as_str()) { - ("tag", "STRING") => { - let mut b = StringDictionaryBuilder::::new(); - for row in 0..rows { - let series = series_for_row(layout, sst_idx, base_row, row); - let value = tag_value(col, series); - b.append_value(value); - } - columns.push(Arc::new(b.finish()) as ArrayRef); - } - ("field", "DOUBLE") => { - let (min, max) = match col.distribution.as_ref() { - Some(Distribution::DeterministicWave { min, max }) => (*min, *max), - _ => (0.0, 1.0), - }; - columns.push(Arc::new(Float64Array::from( - (0..rows) - .map(|r| wave_value(min, max, base_row + r)) - .collect::>(), - )) as ArrayRef); - } - ("field", "UINT64") => columns.push(Arc::new(UInt64Array::from( - (0..rows).map(|r| (base_row + r) as u64).collect::>(), - )) as ArrayRef), - ("timestamp", "TIMESTAMP(9)") => columns.push(Arc::new(TimestampNanosecondArray::from( - (0..rows) - .map(|r| timestamp_for_row(layout, base_row, r)) - .collect::>(), - )) as ArrayRef), - ("timestamp", "TIMESTAMP(3)") => { - columns.push(Arc::new(TimestampMillisecondArray::from( - (0..rows) - .map(|r| timestamp_for_row(layout, base_row, r) / 1_000_000) - .collect::>(), - )) as ArrayRef) - } - other => panic!("unsupported column combination {other:?}"), - } - } - - let mut pk_builder = BinaryDictionaryBuilder::::new(); - for row in 0..rows { - let series = series_for_row(layout, sst_idx, base_row, row); - let values = table - .columns - .iter() - .filter(|c| c.semantic == "tag") - .map(|c| (c.name.clone(), tag_value(c, series))) - .collect(); - pk_builder - .append(encode_dense_primary_key(table, &values)) - .expect("append generated dense primary key to Arrow dictionary builder"); - } - columns.push(Arc::new(pk_builder.finish())); - columns.push(Arc::new(UInt64Array::from_value(sequence, rows))); - columns.push(Arc::new(UInt8Array::from_value(OpType::Put as u8, rows))); - RecordBatch::try_new(flat_schema, columns) - .expect("generated fixture columns should match flat SST Arrow schema") -} - -fn file_meta_from_sst_info( - info: &SstInfo, - region_id: RegionId, - file_id: FileId, - sequence: u64, -) -> FileMeta { - FileMeta { - region_id, - file_id, - time_range: info.time_range, - level: 0, - file_size: info.file_size, - max_row_group_uncompressed_size: info.max_row_group_uncompressed_size, - available_indexes: Default::default(), - indexes: Default::default(), - index_file_size: 0, - index_version: 0, - num_rows: info.num_rows as u64, - num_row_groups: info.num_row_groups, - sequence: NonZeroU64::new(sequence), - partition_expr: None, - num_series: info.num_series, - primary_key_min: None, - primary_key_max: None, - } -} - -fn deterministic_file_id(seed: u64, index: usize) -> FileId { - let seed_hi = ((seed >> 16) & 0xffff) as u16; - let seed_lo = (seed & 0xffff) as u16; - FileId::parse_str(&format!( - "00000000-0000-{seed_hi:04x}-{seed_lo:04x}-{index:012x}" - )) - .expect("deterministic file id must be a valid UUID") -} - -fn remap_sst_file( - object_store_dir: &Path, - table_dir: &str, - region_id: RegionId, - old_file_id: FileId, - new_file_id: FileId, -) { - if old_file_id == new_file_id { - return; - } - let old_path = object_store_dir.join(mito2::sst::location::sst_file_path( - table_dir, - RegionFileId::new(region_id, old_file_id), - PathType::Bare, - )); - let new_path = object_store_dir.join(mito2::sst::location::sst_file_path( - table_dir, - RegionFileId::new(region_id, new_file_id), - PathType::Bare, - )); - if let Some(parent) = new_path.parent() { - fs::create_dir_all(parent).expect("failed to create remapped SST parent directory"); - } - fs::rename(&old_path, &new_path).unwrap_or_else(|err| { - panic!( - "failed to remap SST file {} -> {}: {err}", - old_path.display(), - new_path.display() - ) - }); -} - -fn case_name_from_path(path: &Path) -> String { - path.parent() - .and_then(Path::file_name) - .and_then(|name| name.to_str()) - .unwrap_or("query_perf_case") - .to_string() -} +#[path = "query_perf_fixture/mod.rs"] +mod query_perf_fixture; #[tokio::main] async fn main() { - let args = Args::parse(); - let case_text = fs::read_to_string(&args.case).expect("failed to read query perf case TOML"); - let case: CaseFile = toml::from_str(&case_text).expect("failed to parse query perf case TOML"); - let case_name = case_name_from_path(&args.case); - let scenario = case.scenario.direct_readable_sst(); - if scenario.tables.is_empty() || scenario.layout.regions != 1 { - panic!("fixture generator supports one or more tables and exactly one region per table"); - } - if scenario.layout.time_range_layout != "non_overlapping_per_sst" { - panic!("MVP supports time_range_layout=non_overlapping_per_sst"); - } - if scenario.layout.series_layout == "timestamp_major" - && !scenario - .layout - .rows_per_sst - .is_multiple_of(scenario.layout.series_count.get()) - { - panic!( - "series_layout=timestamp_major requires rows_per_sst to be divisible by series_count" - ); - } - if scenario.layout.sst_count > 1000 && !args.allow_large { - panic!("sst_count exceeds 1000; pass --allow-large"); - } - let seed = scenario.seed.unwrap_or(0); - let table_index = match args.table.as_deref() { - Some(name) => scenario - .tables - .iter() - .position(|table| table.name == name) - .unwrap_or_else(|| panic!("--table {name} was not found in case {case_name}")), - None if scenario.tables.len() == 1 => 0, - None => panic!( - "case {} contains {} tables; pass --table to choose one", - case_name, - scenario.tables.len() - ), - }; - let table = &scenario.tables[table_index]; - if table.engine != "mito" { - panic!("MVP supports engine=mito"); - } - let region_id = RegionId::from(args.region_id); - let table_dir = args - .table_dir - .unwrap_or_else(|| format!("data/{}/{}/", table.database, table.name)); - if Path::new(&table_dir).is_absolute() { - panic!("--table-dir must be relative"); - } - if table_dir.trim_end_matches('/').rsplit('/').next() - == Some(region_name(region_id.table_id(), region_id.region_sequence()).as_str()) - { - panic!("--table-dir must be table-level directory"); - } - let region_dir = - mito2::sst::location::region_dir_from_table_dir(&table_dir, region_id, PathType::Bare); - let format = match table.sst_format.as_deref().unwrap_or("flat") { - "flat" => FormatType::Flat, - "primary_key" | "primary-key" => FormatType::PrimaryKey, - other => panic!("unsupported sst_format {other}"), - }; - println!( - "query_perf_fixture case={} scenario={} table={}.{} ssts={} rows_per_sst={} format={format:?}", - case_name, - case.scenario.kind(), - table.database, - table.name, - scenario.layout.sst_count, - scenario.layout.rows_per_sst - ); - println!( - "out_dir={} table_dir={} region_dir={}", - args.out_dir.display(), - table_dir, - region_dir - ); - if args.dry_run { - return; - } - - let obj_store_dir = args.out_dir.join("object-store"); - let manifest_dir = args.out_dir.join("manifest"); - fs::create_dir_all(&obj_store_dir).expect("failed to create fixture object-store directory"); - fs::create_dir_all(&manifest_dir).expect("failed to create fixture manifest directory"); - let ostorage = ObjectStore::new(FsBuilder::default().root(&obj_store_dir.to_string_lossy())) - .expect("failed to create filesystem object store for fixture output") - .finish(); - let metadata: RegionMetadataRef = Arc::new(build_region_metadata(table, region_id)); - let mut files = HashMap::with_capacity(scenario.layout.sst_count); - let mut next_file_index = 1; - for i in 0..scenario.layout.sst_count { - let sequence = 1000 + i as u64; - let batch = generate_record_batch(table, &metadata, &scenario.layout, i, sequence); - let source = - FlatSource::new_iter(batch.schema(), Box::new(vec![batch].into_iter().map(Ok))); - let mut metrics = Metrics::new(WriteType::Flush); - let mut writer = ParquetWriter::new_with_object_store( - ostorage.clone(), - metadata.clone(), - IndexConfig::default(), - NoopIndexBuilder, - FixedPathProvider { - table_dir: table_dir.clone(), - }, - &mut metrics, - ) - .await; - let opts = WriteOptions { - write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE, - row_group_size: scenario.layout.row_group_size, - max_file_size: None, - }; - let infos = match format { - FormatType::Flat => writer.write_all_flat(source, Some(sequence), &opts).await, - FormatType::PrimaryKey => { - writer - .write_all_flat_as_primary_key(source, Some(sequence), &opts) - .await - } - } - .expect("failed to write fixture SST through Mito parquet writer"); - for info in infos { - let file_id = deterministic_file_id( - seed.wrapping_add((table_index as u64) << 16), - next_file_index, - ); - next_file_index += 1; - remap_sst_file(&obj_store_dir, &table_dir, region_id, info.file_id, file_id); - files.insert( - file_id, - file_meta_from_sst_info(&info, region_id, file_id, sequence), - ); - } - } - let overall_seq = files - .values() - .filter_map(|m| m.sequence) - .map(|s| s.get()) - .max() - .unwrap_or(0); - let manifest = RegionManifest { - metadata, - files, - removed_files: RemovedFilesRecord::default(), - flushed_entry_id: overall_seq, - flushed_sequence: overall_seq, - committed_sequence: Some(overall_seq), - manifest_version: args.checkpoint_version, - truncated_entry_id: None, - compaction_time_window: None, - sst_format: format, - append_mode: table.append_mode, - }; - let checkpoint = RegionCheckpoint { - last_version: args.checkpoint_version, - compacted_actions: manifest.files.len(), - checkpoint: Some(manifest.clone()), - }; - let checkpoint_bytes = checkpoint - .encode() - .expect("failed to encode fixture region checkpoint"); - let checkpoint_path = manifest_dir.join(format!("{:020}.checkpoint", args.checkpoint_version)); - fs::write(&checkpoint_path, &checkpoint_bytes).expect("failed to write checkpoint file"); - fs::write(manifest_dir.join("_last_checkpoint"), serde_json::to_vec_pretty(&serde_json::json!({ "size": checkpoint_bytes.len(), "version": args.checkpoint_version, "checksum": null, "extend_metadata": {} })).expect("failed to serialize _last_checkpoint metadata")).expect("failed to write _last_checkpoint file"); - let files_jsonl_path = args.out_dir.join("files.jsonl"); - let mut jsonl = BufWriter::new( - fs::File::create(&files_jsonl_path).expect("failed to create fixture files.jsonl"), - ); - let mut entries: Vec<_> = manifest.files.iter().collect(); - entries.sort_by_key(|(id, _)| id.to_string()); - for (file_id, meta) in entries { - writeln!(jsonl, "{}", serde_json::to_string(&serde_json::json!({ "file_id": file_id.to_string(), "region_id": meta.region_id.as_u64(), "object_path": mito2::sst::location::sst_file_path(&table_dir, RegionFileId::new(meta.region_id, *file_id), PathType::Bare), "time_range_start": meta.time_range.0.value(), "time_range_end": meta.time_range.1.value(), "num_rows": meta.num_rows, "num_row_groups": meta.num_row_groups, "file_size": meta.file_size, "num_series": meta.num_series, "sequence": meta.sequence.map(|s| s.get()) })).expect("failed to serialize fixture SST metadata JSONL entry")).expect("failed to write fixture SST metadata JSONL entry"); - } - jsonl.flush().expect("failed to flush fixture files.jsonl"); - fs::write(args.out_dir.join("summary.json"), serde_json::to_vec_pretty(&serde_json::json!({ "case": case_name, "seed": seed, "table_index": table_index, "table": table.name, "database": table.database, "region_id": region_id.as_u64(), "table_dir": table_dir, "region_dir": region_dir, "sst_format": format!("{format:?}"), "sst_count": scenario.layout.sst_count, "rows_per_sst": scenario.layout.rows_per_sst, "row_group_size": scenario.layout.row_group_size, "total_rows": scenario.layout.sst_count * scenario.layout.rows_per_sst, "checkpoint_path": checkpoint_path, "files_jsonl_path": files_jsonl_path, "readback_validated": false, "metadata_source": "synthetic" })).expect("failed to serialize fixture summary")).expect("failed to write fixture summary.json"); - println!("Done. wrote {} SST file entries", manifest.files.len()); + query_perf_fixture::run().await; } diff --git a/src/cmd/src/bin/query_perf_fixture/case.rs b/src/cmd/src/bin/query_perf_fixture/case.rs new file mode 100644 index 00000000000..7dce735d7b9 --- /dev/null +++ b/src/cmd/src/bin/query_perf_fixture/case.rs @@ -0,0 +1,470 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::num::NonZeroUsize; + +use clap::ValueEnum; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct CaseFile { + pub(super) scenario: Scenario, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(tag = "kind")] +pub(super) enum Scenario { + #[serde(rename = "direct_readable_sst")] + DirectReadableSst(DirectReadableSstScenario), + #[serde(rename = "prom_remote_write_then_query")] + PromRemoteWriteThenQuery(PromRemoteWriteThenQueryScenario), +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct DirectReadableSstScenario { + #[serde(default)] + pub(super) seed: Option, + pub(super) tables: Vec, + pub(super) layout: LayoutConfig, +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct PromRemoteWriteThenQueryScenario { + #[serde(default)] + pub(super) queries: Vec, + pub(super) remote_write: PromRemoteWritePlan, +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct PromRemoteWritePlan { + #[serde(default = "default_database")] + pub(super) database: String, + #[serde(alias = "metric_name")] + pub(super) metric: String, + #[serde(default = "default_physical_table")] + pub(super) physical_table: String, + #[serde(default = "default_series_count")] + pub(super) series_count: u64, + #[serde(default = "default_samples_per_series")] + pub(super) samples_per_series: u64, + #[serde(default = "default_start_unix_millis")] + pub(super) start_unix_millis: i64, + #[serde(default = "default_step_millis")] + pub(super) step_millis: i64, + #[serde(default = "default_chunk_series_count", alias = "batch_size")] + pub(super) chunk_series_count: u64, + #[serde(default = "default_timeout_seconds")] + pub(super) timeout_seconds: u64, + #[serde(default)] + pub(super) sample_chunk_size: Option, + #[serde(default = "default_flush_every_sample_chunks")] + pub(super) flush_every_sample_chunks: u64, + #[serde(default = "default_visibility_timeout_seconds")] + pub(super) visibility_timeout_seconds: u64, + #[serde(default)] + pub(super) prom_store: PromStoreConfig, + #[serde(default)] + pub(super) value: ValueConfig, + #[serde(default)] + pub(super) storage: Option, + #[serde(default)] + pub(super) read_bench: Option, +} + +pub(super) fn default_database() -> String { + "public".to_string() +} +pub(super) fn default_physical_table() -> String { + "greptime_physical_table".to_string() +} +pub(super) fn default_series_count() -> u64 { + 8 +} +pub(super) fn default_samples_per_series() -> u64 { + 30 +} +pub(super) fn default_start_unix_millis() -> i64 { + 1_704_067_200_000 +} +pub(super) fn default_step_millis() -> i64 { + 15_000 +} +pub(super) fn default_chunk_series_count() -> u64 { + 8 +} +pub(super) fn default_timeout_seconds() -> u64 { + 60 +} +pub(super) fn default_flush_every_sample_chunks() -> u64 { + 1 +} +pub(super) fn default_visibility_timeout_seconds() -> u64 { + 30 +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct PromStoreConfig { + #[serde(default = "default_pending_rows_flush_interval")] + pub(super) pending_rows_flush_interval: String, + #[serde(default = "default_max_batch_rows")] + pub(super) max_batch_rows: u64, + #[serde(default = "default_max_concurrent_flushes")] + pub(super) max_concurrent_flushes: u64, + #[serde(default = "default_worker_channel_capacity")] + pub(super) worker_channel_capacity: u64, + #[serde(default = "default_max_inflight_requests")] + pub(super) max_inflight_requests: u64, +} + +impl Default for PromStoreConfig { + fn default() -> Self { + Self { + pending_rows_flush_interval: default_pending_rows_flush_interval(), + max_batch_rows: default_max_batch_rows(), + max_concurrent_flushes: default_max_concurrent_flushes(), + worker_channel_capacity: default_worker_channel_capacity(), + max_inflight_requests: default_max_inflight_requests(), + } + } +} + +pub(super) fn default_pending_rows_flush_interval() -> String { + "1s".to_string() +} +pub(super) fn default_max_batch_rows() -> u64 { + 100000 +} +pub(super) fn default_max_concurrent_flushes() -> u64 { + 256 +} +pub(super) fn default_worker_channel_capacity() -> u64 { + 65526 +} +pub(super) fn default_max_inflight_requests() -> u64 { + 3000 +} + +#[derive(Debug, Clone, Copy, Deserialize, Serialize, ValueEnum, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +#[value(rename_all = "snake_case")] +pub(super) enum ValuePattern { + Linear, + Constant, + Modulo, + Unique, + SeededRandom, + RunLength, + QuantizedSignal, + SignalWithSporadicStalls, + MixedSignalRepeated, +} + +impl std::fmt::Display for ValuePattern { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + serde_json::to_value(self).unwrap().as_str().unwrap() + ) + } +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct ValueConfig { + #[serde(default = "default_value_pattern")] + pub(super) pattern: ValuePattern, + #[serde(default)] + pub(super) base: f64, + #[serde(default = "default_value_step")] + pub(super) step: f64, + #[serde(default = "default_value_cardinality")] + pub(super) cardinality: u64, + #[serde(default)] + pub(super) seed: u64, + #[serde(default = "default_value_run_length")] + pub(super) run_length: u64, + #[serde(default = "default_value_stall_every")] + pub(super) stall_every: u64, + #[serde(default = "default_value_stall_length")] + pub(super) stall_length: u64, + #[serde(default = "default_value_mixed_every")] + pub(super) mixed_every: u64, +} +impl Default for ValueConfig { + fn default() -> Self { + Self { + pattern: default_value_pattern(), + base: 0.0, + step: default_value_step(), + cardinality: default_value_cardinality(), + seed: 0, + run_length: default_value_run_length(), + stall_every: default_value_stall_every(), + stall_length: default_value_stall_length(), + mixed_every: default_value_mixed_every(), + } + } +} +pub(super) fn default_value_pattern() -> ValuePattern { + ValuePattern::Linear +} +pub(super) fn default_value_step() -> f64 { + 0.125 +} +pub(super) fn default_value_cardinality() -> u64 { + 97 +} +pub(super) fn default_value_run_length() -> u64 { + 8 +} +pub(super) fn default_value_stall_every() -> u64 { + 100 +} +pub(super) fn default_value_stall_length() -> u64 { + 16 +} +pub(super) fn default_value_mixed_every() -> u64 { + 5 +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct StorageConfig { + #[serde(default = "default_true")] + pub(super) inspect: bool, + #[serde(default = "default_storage_column")] + pub(super) column: String, + #[serde(default)] + pub(super) root_suffix: Option, + #[serde(default)] + pub(super) include_metadata_files: bool, + #[serde(default = "default_min_files")] + pub(super) min_files: u64, + #[serde(default = "default_min_files")] + pub(super) min_files_with_column: u64, + #[serde(default)] + pub(super) require_encodings: Vec, + #[serde(default)] + pub(super) forbid_encodings: Vec, + #[serde(default)] + pub(super) max_total_file_size_bytes: Option, + #[serde(default)] + pub(super) max_column_compressed_size_bytes: Option, + #[serde(default)] + pub(super) max_column_uncompressed_size_bytes: Option, + #[serde(default)] + pub(super) max_candidate_total_file_size_regression_pct: Option, + #[serde(default)] + pub(super) max_candidate_column_compressed_size_regression_pct: Option, + #[serde(default)] + pub(super) max_candidate_column_uncompressed_size_regression_pct: Option, + #[serde(skip_deserializing, default)] + pub(super) planned_thresholds: Vec, +} + +#[derive(Debug, Serialize)] +pub(super) struct StorageThresholdPlan { + pub(super) threshold: String, + pub(super) status: String, + pub(super) value: serde_json::Value, +} + +impl StorageConfig { + pub(super) fn populate_planned_thresholds(&mut self) { + let mut planned = Vec::new(); + planned.push(StorageThresholdPlan::new("min_files", self.min_files)); + planned.push(StorageThresholdPlan::new( + "min_files_with_column", + self.min_files_with_column, + )); + if !self.require_encodings.is_empty() { + planned.push(StorageThresholdPlan::new( + "require_encodings", + &self.require_encodings, + )); + } + if !self.forbid_encodings.is_empty() { + planned.push(StorageThresholdPlan::new( + "forbid_encodings", + &self.forbid_encodings, + )); + } + macro_rules! push_optional { + ($name:literal, $value:expr) => { + if let Some(value) = $value { + planned.push(StorageThresholdPlan::new($name, value)); + } + }; + } + push_optional!("max_total_file_size_bytes", self.max_total_file_size_bytes); + push_optional!( + "max_column_compressed_size_bytes", + self.max_column_compressed_size_bytes + ); + push_optional!( + "max_column_uncompressed_size_bytes", + self.max_column_uncompressed_size_bytes + ); + push_optional!( + "max_candidate_total_file_size_regression_pct", + self.max_candidate_total_file_size_regression_pct + ); + push_optional!( + "max_candidate_column_compressed_size_regression_pct", + self.max_candidate_column_compressed_size_regression_pct + ); + push_optional!( + "max_candidate_column_uncompressed_size_regression_pct", + self.max_candidate_column_uncompressed_size_regression_pct + ); + self.planned_thresholds = planned; + } +} + +impl StorageThresholdPlan { + fn new(threshold: &str, value: T) -> Self { + Self { + threshold: threshold.to_string(), + status: "planned".to_string(), + value: serde_json::to_value(value).expect("storage threshold value must serialize"), + } + } +} +pub(super) fn default_true() -> bool { + true +} +pub(super) fn default_storage_column() -> String { + "greptime_value".to_string() +} +pub(super) fn default_min_files() -> u64 { + 1 +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub(super) struct ReadBenchConfig { + #[serde(default = "default_true")] + pub(super) enabled: bool, + #[serde(default = "default_true")] + pub(super) parquetbench: bool, + #[serde(default = "default_true")] + pub(super) scanbench: bool, + #[serde(default = "default_iterations")] + pub(super) iterations: u64, + #[serde(default)] + pub(super) projection: Vec, + #[serde(default = "default_parquet_reader")] + pub(super) parquet_reader: String, + #[serde(default = "default_scan_scanner")] + pub(super) scan_scanner: String, + #[serde(default = "default_parallelism")] + pub(super) parallelism: u64, + #[serde(default)] + pub(super) max_files: Option, + #[serde(flatten)] + pub(super) thresholds: HashMap, +} + +impl Default for ReadBenchConfig { + fn default() -> Self { + Self { + enabled: true, + parquetbench: true, + scanbench: true, + iterations: default_iterations(), + projection: vec![], + parquet_reader: default_parquet_reader(), + scan_scanner: default_scan_scanner(), + parallelism: default_parallelism(), + max_files: None, + thresholds: HashMap::new(), + } + } +} + +pub(super) fn default_iterations() -> u64 { + 7 +} +pub(super) fn default_parquet_reader() -> String { + "direct".to_string() +} +pub(super) fn default_scan_scanner() -> String { + "seq".to_string() +} +pub(super) fn default_parallelism() -> u64 { + 1 +} + +impl Scenario { + pub(super) fn kind(&self) -> &'static str { + match self { + Scenario::DirectReadableSst(_) => "direct_readable_sst", + Scenario::PromRemoteWriteThenQuery(_) => "prom_remote_write_then_query", + } + } + + pub(super) fn direct_readable_sst(&self) -> &DirectReadableSstScenario { + match self { + Scenario::DirectReadableSst(scenario) => scenario, + _ => panic!("scenario is not direct_readable_sst"), + } + } +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct TableConfig { + pub(super) database: String, + pub(super) name: String, + pub(super) engine: String, + #[serde(default)] + pub(super) append_mode: Option, + #[serde(default)] + pub(super) sst_format: Option, + pub(super) primary_key: Vec, + pub(super) time_index: String, + pub(super) columns: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct ColumnConfig { + pub(super) name: String, + #[serde(rename = "type")] + pub(super) ty: String, + pub(super) semantic: String, + pub(super) distribution: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(tag = "kind")] +pub(super) enum Distribution { + #[serde(rename = "cardinality")] + Cardinality { + values: NonZeroUsize, + prefix: String, + }, + #[serde(rename = "deterministic_wave")] + DeterministicWave { min: f64, max: f64 }, +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct LayoutConfig { + pub(super) regions: usize, + pub(super) sst_count: usize, + pub(super) rows_per_sst: usize, + pub(super) row_group_size: usize, + pub(super) series_count: NonZeroUsize, + pub(super) start_unix_nanos: i64, + pub(super) step_nanos: i64, + pub(super) time_range_layout: String, + pub(super) series_layout: String, +} diff --git a/src/cmd/src/bin/query_perf_fixture/direct_sst.rs b/src/cmd/src/bin/query_perf_fixture/direct_sst.rs new file mode 100644 index 00000000000..0f2e3b34876 --- /dev/null +++ b/src/cmd/src/bin/query_perf_fixture/direct_sst.rs @@ -0,0 +1,529 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::fs::{self}; +use std::io::{BufWriter, Write}; +use std::num::NonZeroU64; +use std::path::Path; +use std::sync::Arc; + +use api::v1::{OpType, SemanticType}; +use async_trait::async_trait; +use datatypes::arrow::array::{ + ArrayRef, BinaryDictionaryBuilder, Float64Array, RecordBatch, StringDictionaryBuilder, + TimestampMillisecondArray, TimestampNanosecondArray, UInt8Array, UInt64Array, +}; +use datatypes::arrow::datatypes::UInt32Type; +use datatypes::prelude::ConcreteDataType; +use datatypes::schema::{ColumnSchema, SkippingIndexOptions}; +use mito_codec::row_converter::{DensePrimaryKeyCodec, PrimaryKeyCodecExt, SortField}; +use mito2::access_layer::{FilePathProvider, Metrics, WriteType}; +use mito2::config::IndexConfig; +use mito2::manifest::action::{RegionCheckpoint, RegionManifest, RemovedFilesRecord}; +use mito2::read::FlatSource; +use mito2::sst::file::{FileMeta, RegionFileId}; +use mito2::sst::index::{Indexer, IndexerBuilder}; +use mito2::sst::parquet::writer::ParquetWriter; +use mito2::sst::parquet::{SstInfo, WriteOptions}; +use mito2::sst::{ + DEFAULT_WRITE_BUFFER_SIZE, FlatSchemaOptions, FormatType, to_flat_sst_arrow_schema, +}; +use object_store::ObjectStore; +use object_store::services::Fs as FsBuilder; +use store_api::codec::PrimaryKeyEncoding; +use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataRef}; +use store_api::path_utils::region_name; +use store_api::region_request::PathType; +use store_api::storage::{FileId, RegionId}; + +use crate::query_perf_fixture::DirectArgs; +use crate::query_perf_fixture::case::*; +use crate::query_perf_fixture::util::case_name_from_path; + +struct NoopIndexBuilder; + +#[async_trait] +impl IndexerBuilder for NoopIndexBuilder { + async fn build(&self, _file_id: FileId, _index_version: u64) -> Indexer { + Indexer::default() + } +} + +#[derive(Clone)] +struct FixedPathProvider { + table_dir: String, +} + +impl FilePathProvider for FixedPathProvider { + fn build_index_file_path(&self, file_id: RegionFileId) -> String { + mito2::sst::location::index_file_path_legacy(&self.table_dir, file_id, PathType::Bare) + } + fn build_index_file_path_with_version( + &self, + index_id: mito2::sst::file::RegionIndexId, + ) -> String { + mito2::sst::location::index_file_path(&self.table_dir, index_id, PathType::Bare) + } + fn build_sst_file_path(&self, file_id: RegionFileId) -> String { + mito2::sst::location::sst_file_path(&self.table_dir, file_id, PathType::Bare) + } +} + +fn semantic_type(s: &str) -> SemanticType { + match s.to_ascii_lowercase().as_str() { + "tag" => SemanticType::Tag, + "field" => SemanticType::Field, + "timestamp" => SemanticType::Timestamp, + other => panic!("unsupported semantic {other}"), + } +} + +fn concrete_type(ty: &str) -> ConcreteDataType { + match ty.to_ascii_uppercase().as_str() { + "STRING" => ConcreteDataType::string_datatype(), + "DOUBLE" => ConcreteDataType::float64_datatype(), + "UINT64" => ConcreteDataType::uint64_datatype(), + "TIMESTAMP(9)" => ConcreteDataType::timestamp_nanosecond_datatype(), + "TIMESTAMP(3)" => ConcreteDataType::timestamp_millisecond_datatype(), + other => panic!("unsupported column type {other}"), + } +} + +fn build_region_metadata(table: &TableConfig, region_id: RegionId) -> RegionMetadata { + let mut builder = store_api::metadata::RegionMetadataBuilder::new(region_id); + for (idx, col) in table.columns.iter().enumerate() { + let mut schema = ColumnSchema::new( + &col.name, + concrete_type(&col.ty), + col.semantic != "timestamp", + ); + if col.semantic == "tag" { + schema = schema.with_inverted_index(true); + } + if col.semantic == "tag" || col.semantic == "field" { + schema = schema + .with_skipping_options(SkippingIndexOptions { + granularity: 1, + ..Default::default() + }) + .expect("valid skipping index options for query perf fixture"); + } + if col.name == table.time_index { + schema = schema.with_time_index(true); + } + builder.push_column_metadata(ColumnMetadata { + column_schema: schema, + semantic_type: semantic_type(&col.semantic), + column_id: idx as u32, + }); + } + let pk_ids = table + .primary_key + .iter() + .map(|name| { + table + .columns + .iter() + .position(|c| c.name == *name) + .unwrap_or_else(|| panic!("primary key column {name} not found")) as u32 + }) + .collect(); + builder.primary_key(pk_ids); + builder.primary_key_encoding(PrimaryKeyEncoding::Dense); + builder + .build() + .expect("region metadata should be valid for fixture table config") +} + +fn encode_dense_primary_key(table: &TableConfig, values: &HashMap) -> Vec { + let fields = table + .primary_key + .iter() + .enumerate() + .map(|(idx, _)| { + ( + idx as u32, + SortField::new(ConcreteDataType::string_datatype()), + ) + }) + .collect(); + let converter = DensePrimaryKeyCodec::with_fields(fields); + converter + .encode( + table + .primary_key + .iter() + .map(|name| datatypes::value::ValueRef::String(values[name].as_str())), + ) + .expect("dense primary key encoding should match fixture primary key fields") +} + +fn tag_value(col: &ColumnConfig, series: usize) -> String { + match col + .distribution + .as_ref() + .expect("tag distribution is required") + { + Distribution::Cardinality { values, prefix } => { + format!("{}{}", prefix, series % values.get()) + } + _ => panic!("tag column {} requires cardinality distribution", col.name), + } +} + +fn series_for_row(layout: &LayoutConfig, sst_idx: usize, base_row: usize, row: usize) -> usize { + match layout.series_layout.as_str() { + "round_robin" | "timestamp_major" => (base_row + row) % layout.series_count.get(), + "per_sst" => sst_idx % layout.series_count.get(), + other => panic!("unsupported series_layout {other}"), + } +} + +fn timestamp_for_row(layout: &LayoutConfig, base_row: usize, row: usize) -> i64 { + let logical_row = base_row + row; + let timestamp_step = if layout.series_layout == "timestamp_major" { + logical_row / layout.series_count.get() + } else { + logical_row + }; + + layout.start_unix_nanos + (timestamp_step as i64 * layout.step_nanos) +} + +fn wave_value(min: f64, max: f64, row: usize) -> f64 { + let span = max - min; + let phase = (row % 1024) as f64 / 1023.0; + min + span * (0.5 - 0.5 * (std::f64::consts::TAU * phase).cos()) +} + +fn generate_record_batch( + table: &TableConfig, + metadata: &RegionMetadataRef, + layout: &LayoutConfig, + sst_idx: usize, + sequence: u64, +) -> RecordBatch { + let flat_schema = to_flat_sst_arrow_schema(metadata, &FlatSchemaOptions::default()); + let rows = layout.rows_per_sst; + let mut columns = Vec::new(); + let base_row = sst_idx * rows; + + for col in &table.columns { + match (col.semantic.as_str(), col.ty.to_ascii_uppercase().as_str()) { + ("tag", "STRING") => { + let mut b = StringDictionaryBuilder::::new(); + for row in 0..rows { + let series = series_for_row(layout, sst_idx, base_row, row); + let value = tag_value(col, series); + b.append_value(value); + } + columns.push(Arc::new(b.finish()) as ArrayRef); + } + ("field", "DOUBLE") => { + let (min, max) = match col.distribution.as_ref() { + Some(Distribution::DeterministicWave { min, max }) => (*min, *max), + _ => (0.0, 1.0), + }; + columns.push(Arc::new(Float64Array::from( + (0..rows) + .map(|r| wave_value(min, max, base_row + r)) + .collect::>(), + )) as ArrayRef); + } + ("field", "UINT64") => columns.push(Arc::new(UInt64Array::from( + (0..rows).map(|r| (base_row + r) as u64).collect::>(), + )) as ArrayRef), + ("timestamp", "TIMESTAMP(9)") => columns.push(Arc::new(TimestampNanosecondArray::from( + (0..rows) + .map(|r| timestamp_for_row(layout, base_row, r)) + .collect::>(), + )) as ArrayRef), + ("timestamp", "TIMESTAMP(3)") => { + columns.push(Arc::new(TimestampMillisecondArray::from( + (0..rows) + .map(|r| timestamp_for_row(layout, base_row, r) / 1_000_000) + .collect::>(), + )) as ArrayRef) + } + other => panic!("unsupported column combination {other:?}"), + } + } + + let mut pk_builder = BinaryDictionaryBuilder::::new(); + for row in 0..rows { + let series = series_for_row(layout, sst_idx, base_row, row); + let values = table + .columns + .iter() + .filter(|c| c.semantic == "tag") + .map(|c| (c.name.clone(), tag_value(c, series))) + .collect(); + pk_builder + .append(encode_dense_primary_key(table, &values)) + .expect("append generated dense primary key to Arrow dictionary builder"); + } + columns.push(Arc::new(pk_builder.finish())); + columns.push(Arc::new(UInt64Array::from_value(sequence, rows))); + columns.push(Arc::new(UInt8Array::from_value(OpType::Put as u8, rows))); + RecordBatch::try_new(flat_schema, columns) + .expect("generated fixture columns should match flat SST Arrow schema") +} + +fn file_meta_from_sst_info( + info: &SstInfo, + region_id: RegionId, + file_id: FileId, + sequence: u64, +) -> FileMeta { + FileMeta { + region_id, + file_id, + time_range: info.time_range, + level: 0, + file_size: info.file_size, + max_row_group_uncompressed_size: info.max_row_group_uncompressed_size, + available_indexes: Default::default(), + indexes: Default::default(), + index_file_size: 0, + index_version: 0, + num_rows: info.num_rows as u64, + num_row_groups: info.num_row_groups, + sequence: NonZeroU64::new(sequence), + partition_expr: None, + num_series: info.num_series, + primary_key_min: None, + primary_key_max: None, + } +} + +fn deterministic_file_id(seed: u64, index: usize) -> FileId { + let seed_hi = ((seed >> 16) & 0xffff) as u16; + let seed_lo = (seed & 0xffff) as u16; + FileId::parse_str(&format!( + "00000000-0000-{seed_hi:04x}-{seed_lo:04x}-{index:012x}" + )) + .expect("deterministic file id must be a valid UUID") +} + +fn remap_sst_file( + object_store_dir: &Path, + table_dir: &str, + region_id: RegionId, + old_file_id: FileId, + new_file_id: FileId, +) { + if old_file_id == new_file_id { + return; + } + let old_path = object_store_dir.join(mito2::sst::location::sst_file_path( + table_dir, + RegionFileId::new(region_id, old_file_id), + PathType::Bare, + )); + let new_path = object_store_dir.join(mito2::sst::location::sst_file_path( + table_dir, + RegionFileId::new(region_id, new_file_id), + PathType::Bare, + )); + if let Some(parent) = new_path.parent() { + fs::create_dir_all(parent).expect("failed to create remapped SST parent directory"); + } + fs::rename(&old_path, &new_path).unwrap_or_else(|err| { + panic!( + "failed to remap SST file {} -> {}: {err}", + old_path.display(), + new_path.display() + ) + }); +} +pub(super) async fn run_direct_sst(args: DirectArgs) { + let case_path = args.case.as_ref().expect("--case is required"); + let out_dir = args.out_dir.as_ref().expect("--out-dir is required"); + let case_text = fs::read_to_string(case_path).expect("failed to read query perf case TOML"); + let case: CaseFile = toml::from_str(&case_text).expect("failed to parse query perf case TOML"); + let case_name = case_name_from_path(case_path); + let scenario = case.scenario.direct_readable_sst(); + if scenario.tables.is_empty() || scenario.layout.regions != 1 { + panic!("fixture generator supports one or more tables and exactly one region per table"); + } + if scenario.layout.time_range_layout != "non_overlapping_per_sst" { + panic!("MVP supports time_range_layout=non_overlapping_per_sst"); + } + if scenario.layout.series_layout == "timestamp_major" + && !scenario + .layout + .rows_per_sst + .is_multiple_of(scenario.layout.series_count.get()) + { + panic!( + "series_layout=timestamp_major requires rows_per_sst to be divisible by series_count" + ); + } + if scenario.layout.sst_count > 1000 && !args.allow_large { + panic!("sst_count exceeds 1000; pass --allow-large"); + } + let seed = scenario.seed.unwrap_or(0); + let table_index = match args.table.as_deref() { + Some(name) => scenario + .tables + .iter() + .position(|table| table.name == name) + .unwrap_or_else(|| panic!("--table {name} was not found in case {case_name}")), + None if scenario.tables.len() == 1 => 0, + None => panic!( + "case {} contains {} tables; pass --table to choose one", + case_name, + scenario.tables.len() + ), + }; + let table = &scenario.tables[table_index]; + if table.engine != "mito" { + panic!("MVP supports engine=mito"); + } + let region_id = RegionId::from(args.region_id); + let table_dir = args + .table_dir + .unwrap_or_else(|| format!("data/{}/{}/", table.database, table.name)); + if Path::new(&table_dir).is_absolute() { + panic!("--table-dir must be relative"); + } + if table_dir.trim_end_matches('/').rsplit('/').next() + == Some(region_name(region_id.table_id(), region_id.region_sequence()).as_str()) + { + panic!("--table-dir must be table-level directory"); + } + let region_dir = + mito2::sst::location::region_dir_from_table_dir(&table_dir, region_id, PathType::Bare); + let format = match table.sst_format.as_deref().unwrap_or("flat") { + "flat" => FormatType::Flat, + "primary_key" | "primary-key" => FormatType::PrimaryKey, + other => panic!("unsupported sst_format {other}"), + }; + println!( + "query_perf_fixture case={} scenario={} table={}.{} ssts={} rows_per_sst={} format={format:?}", + case_name, + case.scenario.kind(), + table.database, + table.name, + scenario.layout.sst_count, + scenario.layout.rows_per_sst + ); + println!( + "out_dir={} table_dir={} region_dir={}", + out_dir.display(), + table_dir, + region_dir + ); + if args.dry_run { + return; + } + + let obj_store_dir = out_dir.join("object-store"); + let manifest_dir = out_dir.join("manifest"); + fs::create_dir_all(&obj_store_dir).expect("failed to create fixture object-store directory"); + fs::create_dir_all(&manifest_dir).expect("failed to create fixture manifest directory"); + let ostorage = ObjectStore::new(FsBuilder::default().root(&obj_store_dir.to_string_lossy())) + .expect("failed to create filesystem object store for fixture output") + .finish(); + let metadata: RegionMetadataRef = Arc::new(build_region_metadata(table, region_id)); + let mut files = HashMap::with_capacity(scenario.layout.sst_count); + let mut next_file_index = 1; + for i in 0..scenario.layout.sst_count { + let sequence = 1000 + i as u64; + let batch = generate_record_batch(table, &metadata, &scenario.layout, i, sequence); + let source = + FlatSource::new_iter(batch.schema(), Box::new(vec![batch].into_iter().map(Ok))); + let mut metrics = Metrics::new(WriteType::Flush); + let mut writer = ParquetWriter::new_with_object_store( + ostorage.clone(), + metadata.clone(), + IndexConfig::default(), + NoopIndexBuilder, + FixedPathProvider { + table_dir: table_dir.clone(), + }, + &mut metrics, + ) + .await; + let opts = WriteOptions { + write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE, + row_group_size: scenario.layout.row_group_size, + max_file_size: None, + }; + let infos = match format { + FormatType::Flat => writer.write_all_flat(source, Some(sequence), &opts).await, + FormatType::PrimaryKey => { + writer + .write_all_flat_as_primary_key(source, Some(sequence), &opts) + .await + } + } + .expect("failed to write fixture SST through Mito parquet writer"); + for info in infos { + let file_id = deterministic_file_id( + seed.wrapping_add((table_index as u64) << 16), + next_file_index, + ); + next_file_index += 1; + remap_sst_file(&obj_store_dir, &table_dir, region_id, info.file_id, file_id); + files.insert( + file_id, + file_meta_from_sst_info(&info, region_id, file_id, sequence), + ); + } + } + let overall_seq = files + .values() + .filter_map(|m| m.sequence) + .map(|s| s.get()) + .max() + .unwrap_or(0); + let manifest = RegionManifest { + metadata, + files, + removed_files: RemovedFilesRecord::default(), + flushed_entry_id: overall_seq, + flushed_sequence: overall_seq, + committed_sequence: Some(overall_seq), + manifest_version: args.checkpoint_version, + truncated_entry_id: None, + compaction_time_window: None, + sst_format: format, + append_mode: table.append_mode, + }; + let checkpoint = RegionCheckpoint { + last_version: args.checkpoint_version, + compacted_actions: manifest.files.len(), + checkpoint: Some(manifest.clone()), + }; + let checkpoint_bytes = checkpoint + .encode() + .expect("failed to encode fixture region checkpoint"); + let checkpoint_path = manifest_dir.join(format!("{:020}.checkpoint", args.checkpoint_version)); + fs::write(&checkpoint_path, &checkpoint_bytes).expect("failed to write checkpoint file"); + fs::write(manifest_dir.join("_last_checkpoint"), serde_json::to_vec_pretty(&serde_json::json!({ "size": checkpoint_bytes.len(), "version": args.checkpoint_version, "checksum": null, "extend_metadata": {} })).expect("failed to serialize _last_checkpoint metadata")).expect("failed to write _last_checkpoint file"); + let files_jsonl_path = out_dir.join("files.jsonl"); + let mut jsonl = BufWriter::new( + fs::File::create(&files_jsonl_path).expect("failed to create fixture files.jsonl"), + ); + let mut entries: Vec<_> = manifest.files.iter().collect(); + entries.sort_by_key(|(id, _)| id.to_string()); + for (file_id, meta) in entries { + writeln!(jsonl, "{}", serde_json::to_string(&serde_json::json!({ "file_id": file_id.to_string(), "region_id": meta.region_id.as_u64(), "object_path": mito2::sst::location::sst_file_path(&table_dir, RegionFileId::new(meta.region_id, *file_id), PathType::Bare), "time_range_start": meta.time_range.0.value(), "time_range_end": meta.time_range.1.value(), "num_rows": meta.num_rows, "num_row_groups": meta.num_row_groups, "file_size": meta.file_size, "num_series": meta.num_series, "sequence": meta.sequence.map(|s| s.get()) })).expect("failed to serialize fixture SST metadata JSONL entry")).expect("failed to write fixture SST metadata JSONL entry"); + } + jsonl.flush().expect("failed to flush fixture files.jsonl"); + fs::write(out_dir.join("summary.json"), serde_json::to_vec_pretty(&serde_json::json!({ "case": case_name, "seed": seed, "table_index": table_index, "table": table.name, "database": table.database, "region_id": region_id.as_u64(), "table_dir": table_dir, "region_dir": region_dir, "sst_format": format!("{format:?}"), "sst_count": scenario.layout.sst_count, "rows_per_sst": scenario.layout.rows_per_sst, "row_group_size": scenario.layout.row_group_size, "total_rows": scenario.layout.sst_count * scenario.layout.rows_per_sst, "checkpoint_path": checkpoint_path, "files_jsonl_path": files_jsonl_path, "readback_validated": false, "metadata_source": "synthetic" })).expect("failed to serialize fixture summary")).expect("failed to write fixture summary.json"); + println!("Done. wrote {} SST file entries", manifest.files.len()); +} diff --git a/src/cmd/src/bin/query_perf_fixture/inspect_footer.rs b/src/cmd/src/bin/query_perf_fixture/inspect_footer.rs new file mode 100644 index 00000000000..a52005c4e30 --- /dev/null +++ b/src/cmd/src/bin/query_perf_fixture/inspect_footer.rs @@ -0,0 +1,181 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::BTreeSet; +use std::fs::{self, File}; +use std::path::{Path, PathBuf}; + +use clap::Args as ClapArgs; +use parquet::file::reader::{FileReader, SerializedFileReader}; +use serde::Serialize; + +#[derive(Debug, ClapArgs)] +pub(super) struct InspectFooterArgs { + #[arg(long)] + root: PathBuf, + #[arg(long, default_value = "greptime_value")] + column: String, + #[arg(long)] + include_metadata_files: bool, +} + +#[derive(Debug, Serialize)] +struct FooterReport { + root: String, + summary: FooterSummary, + files: Vec, +} +#[derive(Debug, Default, Serialize)] +struct FooterSummary { + column: String, + file_count: usize, + files_with_column: usize, + total_file_size: u64, + total_rows: i64, + column_compressed_size: i64, + column_uncompressed_size: i64, + unique_encodings: Vec, +} +#[derive(Debug, Serialize)] +struct FooterFileReport { + path: String, + relative_path: String, + file_size: u64, + num_rows: i64, + num_row_groups: usize, + columns: Vec, +} +#[derive(Debug, Serialize)] +struct FooterColumnChunkReport { + row_group_index: usize, + column_index: usize, + column_path: String, + encodings: Vec, + compression: String, + compressed_size: i64, + uncompressed_size: i64, + num_values: i64, +} + +pub(super) fn run_inspect_footer( + args: InspectFooterArgs, +) -> Result<(), Box> { + let files = collect_parquet_files(&args.root, args.include_metadata_files)?; + let reports = files + .iter() + .map(|p| inspect_file(&args.root, p, &args.column)) + .collect::, _>>()?; + println!( + "{}", + serde_json::to_string_pretty(&FooterReport { + root: args.root.display().to_string(), + summary: summarize_footer(&args.column, &reports), + files: reports + })? + ); + Ok(()) +} +fn collect_parquet_files( + root: &Path, + include_metadata_files: bool, +) -> Result, Box> { + let mut files = Vec::new(); + collect_parquet_files_inner(root, root, include_metadata_files, &mut files)?; + files.sort(); + Ok(files) +} +fn collect_parquet_files_inner( + root: &Path, + path: &Path, + include_metadata_files: bool, + files: &mut Vec, +) -> Result<(), Box> { + if path.is_dir() { + for entry in fs::read_dir(path)? { + collect_parquet_files_inner(root, &entry?.path(), include_metadata_files, files)?; + } + } else if path.extension().is_some_and(|ext| ext == "parquet") + && (include_metadata_files + || !path + .strip_prefix(root) + .unwrap_or(path) + .components() + .any(|c| c.as_os_str() == "metadata")) + { + files.push(path.to_path_buf()); + } + Ok(()) +} +fn inspect_file( + root: &Path, + path: &Path, + column: &str, +) -> Result> { + let file_size = fs::metadata(path)?.len(); + let reader = SerializedFileReader::new(File::open(path)?)?; + let metadata = reader.metadata().file_metadata(); + let row_groups = reader.metadata().row_groups(); + let mut columns = Vec::new(); + for (row_group_index, rg) in row_groups.iter().enumerate() { + for (column_index, chunk) in rg.columns().iter().enumerate() { + let column_path = chunk.column_path().string(); + if column_path == column { + columns.push(FooterColumnChunkReport { + row_group_index, + column_index, + column_path, + encodings: chunk.encodings().map(|e| format!("{e:?}")).collect(), + compression: format!("{:?}", chunk.compression()), + compressed_size: chunk.compressed_size(), + uncompressed_size: chunk.uncompressed_size(), + num_values: chunk.num_values(), + }); + } + } + } + Ok(FooterFileReport { + path: path.display().to_string(), + relative_path: path + .strip_prefix(root) + .unwrap_or(path) + .display() + .to_string(), + file_size, + num_rows: metadata.num_rows(), + num_row_groups: row_groups.len(), + columns, + }) +} +fn summarize_footer(column: &str, files: &[FooterFileReport]) -> FooterSummary { + let mut encodings = BTreeSet::new(); + let mut s = FooterSummary { + column: column.to_string(), + file_count: files.len(), + ..Default::default() + }; + for file in files { + s.total_file_size += file.file_size; + s.total_rows += file.num_rows; + if !file.columns.is_empty() { + s.files_with_column += 1; + } + for c in &file.columns { + s.column_compressed_size += c.compressed_size; + s.column_uncompressed_size += c.uncompressed_size; + encodings.extend(c.encodings.iter().cloned()); + } + } + s.unique_encodings = encodings.into_iter().collect(); + s +} diff --git a/src/cmd/src/bin/query_perf_fixture/mod.rs b/src/cmd/src/bin/query_perf_fixture/mod.rs new file mode 100644 index 00000000000..00cef327c78 --- /dev/null +++ b/src/cmd/src/bin/query_perf_fixture/mod.rs @@ -0,0 +1,133 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod case; +mod direct_sst; +mod inspect_footer; +mod prom_remote_write; +mod util; + +use std::fs; +use std::path::PathBuf; + +use case::*; +use clap::{Args as ClapArgs, Parser, Subcommand}; +use direct_sst::run_direct_sst; +use inspect_footer::{InspectFooterArgs, run_inspect_footer}; +use prom_remote_write::{PromRemoteWriteArgs, run_prom_remote_write}; +use serde_json::json; + +#[derive(Parser, Debug)] +#[command(name = "query_perf_fixture")] +#[command(about = "Query performance fixture helper")] +struct Args { + #[command(subcommand)] + command: Option, + + #[command(flatten)] + legacy: DirectArgs, +} + +#[derive(Subcommand, Debug)] +enum Command { + DirectSst(DirectArgs), + PromRemoteWrite(PromRemoteWriteArgs), + InspectFooter(InspectFooterArgs), + Plan(PlanArgs), +} + +#[derive(ClapArgs, Debug, Clone)] +struct DirectArgs { + /// TOML case file, for example tests/perf/query_cases/.../case.toml. + #[arg(long, value_name = "PATH")] + case: Option, + + /// Output directory (creates object-store/, manifest/, files.jsonl, summary.json). + #[arg(long, value_name = "DIR")] + out_dir: Option, + + /// Region ID to synthesize. Defaults to table id 1024, region 0. + #[arg(long, default_value = "4398046511104")] + region_id: u64, + + /// Table directory relative to object-store root. Defaults to data/{database}/{table}/. + #[arg(long, value_name = "DIR")] + table_dir: Option, + + /// Table name to generate when the case contains multiple [[tables]]. + #[arg(long, value_name = "NAME")] + table: Option, + + /// Manifest/checkpoint version. + #[arg(long, default_value = "1000000")] + checkpoint_version: u64, + + /// Safety flag when sst_count exceeds 1000. + #[arg(long)] + allow_large: bool, + + /// Print plan only. + #[arg(long)] + dry_run: bool, +} + +#[derive(ClapArgs, Debug)] +struct PlanArgs { + #[arg(long, value_name = "PATH")] + case: PathBuf, +} + +pub async fn run() { + let args = Args::parse(); + match args.command { + Some(Command::DirectSst(direct)) => run_direct_sst(direct).await, + Some(Command::PromRemoteWrite(rw)) => run_prom_remote_write(rw) + .await + .expect("prom remote write failed"), + Some(Command::InspectFooter(inspect)) => { + run_inspect_footer(inspect).expect("inspect footer failed") + } + Some(Command::Plan(plan)) => run_plan(plan).expect("plan failed"), + None => run_direct_sst(args.legacy).await, + } +} + +fn run_plan(args: PlanArgs) -> Result<(), Box> { + let case_text = fs::read_to_string(&args.case)?; + let mut case: CaseFile = toml::from_str(&case_text)?; + if let Scenario::PromRemoteWriteThenQuery(s) = &mut case.scenario + && let Some(storage) = &mut s.remote_write.storage + { + storage.populate_planned_thresholds(); + if storage.inspect { + let mut rb = s.remote_write.read_bench.clone().unwrap_or_default(); + if rb.enabled && rb.projection.is_empty() { + rb.projection = vec![storage.column.clone()]; + } + s.remote_write.read_bench = Some(rb); + } else if s + .remote_write + .read_bench + .as_ref() + .is_some_and(|rb| rb.enabled) + { + return Err("scenario.remote_write.read_bench requires scenario.remote_write.storage.inspect = true".into()); + } + } + println!( + "{}", + serde_json::to_string_pretty(&json!({"schema_version": 1, "scenario": case.scenario}))? + ); + Ok(()) +} diff --git a/src/cmd/src/bin/query_perf_fixture/prom_remote_write.rs b/src/cmd/src/bin/query_perf_fixture/prom_remote_write.rs new file mode 100644 index 00000000000..1f36b0c2d23 --- /dev/null +++ b/src/cmd/src/bin/query_perf_fixture/prom_remote_write.rs @@ -0,0 +1,333 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use clap::Args as ClapArgs; +use prost::Message; +use serde_json::json; +use servers::prom_store::snappy_compress; + +use crate::query_perf_fixture::case::ValuePattern; + +#[derive(Debug, ClapArgs)] +pub(super) struct PromRemoteWriteArgs { + #[arg(long, default_value = "http://127.0.0.1:4000/v1/prometheus/write")] + endpoint: String, + #[arg(long, default_value = "public")] + database: String, + #[arg(long)] + metric: String, + #[arg(long, default_value = "greptime_physical_table")] + physical_table: String, + #[arg(long, default_value_t = 8)] + series_count: u64, + #[arg(long, default_value_t = 30)] + samples_per_series: u64, + #[arg(long, default_value_t = 1_704_067_200_000)] + start_unix_millis: i64, + #[arg(long, default_value_t = 15_000)] + step_millis: i64, + #[arg(long, alias = "batch-size", default_value_t = 8)] + chunk_series_count: u64, + #[arg(long, default_value_t = 60)] + timeout_seconds: u64, + #[arg(long, default_value_t = ValuePattern::Linear)] + value_pattern: ValuePattern, + #[arg(long, default_value_t = 0.0)] + value_base: f64, + #[arg(long, default_value_t = 0.125)] + value_step: f64, + #[arg(long, default_value_t = 97)] + value_cardinality: u64, + #[arg(long, default_value_t = 0)] + value_seed: u64, + #[arg(long, default_value_t = 8)] + value_run_length: u64, + #[arg(long, default_value_t = 100)] + value_stall_every: u64, + #[arg(long, default_value_t = 16)] + value_stall_length: u64, + #[arg(long, default_value_t = 5)] + value_mixed_every: u64, + #[arg(long, default_value_t = 0)] + value_sample_offset: u64, + #[arg(long)] + value_total_samples_per_series: Option, +} + +pub(super) async fn run_prom_remote_write( + args: PromRemoteWriteArgs, +) -> Result<(), Box> { + use api::prom_store::remote::WriteRequest; + let started = std::time::Instant::now(); + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(args.timeout_seconds)) + .build()?; + let mut batches = 0_u64; + let mut rows = 0_u64; + let mut http_statuses = Vec::new(); + for first in (0..args.series_count).step_by(args.chunk_series_count.max(1) as usize) { + let last = (first + args.chunk_series_count.max(1)).min(args.series_count); + let timeseries = (first..last) + .map(|series_idx| make_prom_series(&args, series_idx)) + .collect::>(); + rows += timeseries + .iter() + .map(|ts| ts.samples.len() as u64) + .sum::(); + let body = snappy_compress( + &WriteRequest { + timeseries, + ..Default::default() + } + .encode_to_vec(), + )?; + let response = client + .post(&args.endpoint) + .header("Content-Encoding", "snappy") + .body(body) + .send() + .await?; + let status = response.status(); + http_statuses.push(status.as_u16()); + if !status.is_success() { + return Err(format!( + "remote-write failed with status {status}: {}", + response.text().await.unwrap_or_default() + ) + .into()); + } + batches += 1; + } + println!( + "{}", + json!({"status":"ok","endpoint":args.endpoint,"database":args.database,"metric":args.metric,"physical_table":args.physical_table,"series_count":args.series_count,"samples_per_series":args.samples_per_series,"rows":rows,"samples_written":rows,"batches":batches,"elapsed_seconds":started.elapsed().as_secs_f64(),"http_statuses":http_statuses,"value":{"pattern":args.value_pattern.to_string(),"base":args.value_base,"step":args.value_step,"cardinality":args.value_cardinality,"seed":args.value_seed,"run_length":args.value_run_length,"stall_every":args.value_stall_every,"stall_length":args.value_stall_length,"mixed_every":args.value_mixed_every,"sample_offset":args.value_sample_offset,"total_samples_per_series":args.value_total_samples_per_series.unwrap_or(args.samples_per_series)}}) + ); + Ok(()) +} + +fn make_prom_series( + args: &PromRemoteWriteArgs, + series_idx: u64, +) -> api::prom_store::remote::TimeSeries { + let mut labels = vec![ + prom_label("__name__", &args.metric), + prom_label("x_greptime_database", &args.database), + prom_label("x_greptime_physical_table", &args.physical_table), + prom_label("host", &format!("host{:04}", series_idx % 1024)), + prom_label("instance", &format!("instance{:06}", series_idx)), + ]; + labels.sort_unstable_by(|l, r| l.name.cmp(&r.name)); + let samples = (0..args.samples_per_series) + .map(|sample_idx| api::prom_store::remote::Sample { + value: deterministic_prom_value(args, series_idx, sample_idx), + timestamp: args.start_unix_millis + (sample_idx as i64 * args.step_millis), + }) + .collect(); + api::prom_store::remote::TimeSeries { + labels, + samples, + ..Default::default() + } +} +fn prom_label(name: &str, value: &str) -> api::prom_store::remote::Label { + api::prom_store::remote::Label { + name: name.to_string(), + value: value.to_string(), + } +} +fn deterministic_prom_value(args: &PromRemoteWriteArgs, series_idx: u64, sample_idx: u64) -> f64 { + let ordinal = series_idx + * args + .value_total_samples_per_series + .unwrap_or(args.samples_per_series) + + args.value_sample_offset + + sample_idx; + let local = args.value_sample_offset + sample_idx; + match args.value_pattern { + ValuePattern::Linear => { + args.value_base + (series_idx % 97) as f64 + local as f64 * args.value_step + } + ValuePattern::Constant => args.value_base, + ValuePattern::Modulo => { + args.value_base + (ordinal % args.value_cardinality.max(1)) as f64 * args.value_step + } + ValuePattern::Unique => args.value_base + ordinal as f64 * args.value_step, + ValuePattern::SeededRandom => { + args.value_base + + (splitmix64(ordinal ^ args.value_seed) % args.value_cardinality.max(1)) as f64 + * args.value_step + } + ValuePattern::RunLength => { + args.value_base + + ((ordinal / args.value_run_length.max(1)) % args.value_cardinality.max(1)) as f64 + * args.value_step + } + ValuePattern::QuantizedSignal => { + args.value_base + + (((series_idx % args.value_cardinality.max(1)) + + (local / args.value_run_length.max(1))) + % args.value_cardinality.max(1)) as f64 + * args.value_step + } + ValuePattern::SignalWithSporadicStalls => { + let every = args.value_stall_every.max(1); + let phase = local % every; + let effective = if phase < args.value_stall_length.min(every) { + local - phase + } else { + local + }; + args.value_base + (series_idx % 97) as f64 + effective as f64 * args.value_step + } + ValuePattern::MixedSignalRepeated => { + if local.is_multiple_of(args.value_mixed_every.max(1)) { + args.value_base + } else { + args.value_base + (series_idx % 97) as f64 + local as f64 * args.value_step + } + } + } +} +fn splitmix64(mut value: u64) -> u64 { + value = value.wrapping_add(0x9e3779b97f4a7c15); + let mut mixed = value; + mixed = (mixed ^ (mixed >> 30)).wrapping_mul(0xbf58476d1ce4e5b9); + mixed = (mixed ^ (mixed >> 27)).wrapping_mul(0x94d049bb133111eb); + mixed ^ (mixed >> 31) +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use super::*; + + fn args(pattern: ValuePattern) -> PromRemoteWriteArgs { + PromRemoteWriteArgs { + endpoint: "http://127.0.0.1:4000/v1/prometheus/write".to_string(), + database: "public".to_string(), + metric: "test_metric".to_string(), + physical_table: "greptime_physical_table".to_string(), + series_count: 2, + samples_per_series: 64, + start_unix_millis: 1_704_067_200_000, + step_millis: 60_000, + chunk_series_count: 2, + timeout_seconds: 60, + value_pattern: pattern, + value_base: 0.0, + value_step: 1.0, + value_cardinality: 1_024, + value_seed: 8_444, + value_run_length: 16, + value_stall_every: 100, + value_stall_length: 16, + value_mixed_every: 5, + value_sample_offset: 0, + value_total_samples_per_series: None, + } + } + + #[test] + fn seeded_random_is_deterministic_in_range_and_highly_distinct() { + let mut args = args(ValuePattern::SeededRandom); + args.samples_per_series = 4_096; + args.value_cardinality = 29_491_200; + + let values = (0..args.samples_per_series) + .map(|sample_idx| deterministic_prom_value(&args, 0, sample_idx)) + .collect::>(); + let repeated = (0..args.samples_per_series) + .map(|sample_idx| deterministic_prom_value(&args, 0, sample_idx)) + .collect::>(); + + assert_eq!(values, repeated); + assert!(values.iter().all(|value| { + (args.value_base..=args.value_base + (args.value_cardinality - 1) as f64) + .contains(value) + })); + assert!( + values + .iter() + .map(|value| value.to_bits()) + .collect::>() + .len() + > 4_000 + ); + } + + #[test] + fn run_length_values_have_exact_sixteen_sample_runs() { + let args = args(ValuePattern::RunLength); + + for sample_idx in 0..64 { + assert_eq!( + deterministic_prom_value(&args, 0, sample_idx), + (sample_idx / 16) as f64 + ); + } + } + + #[test] + fn mixed_every_five_repeats_only_the_expected_positions() { + let mut args = args(ValuePattern::MixedSignalRepeated); + args.value_step = 0.125; + + for sample_idx in 0..20 { + let expected = if sample_idx % 5 == 0 { + 0.0 + } else { + 1.0 + sample_idx as f64 * 0.125 + }; + assert_eq!(deterministic_prom_value(&args, 1, sample_idx), expected); + } + } + + #[test] + fn chunked_values_match_monolithic_values_through_offsets() { + let mut monolithic = args(ValuePattern::SeededRandom); + monolithic.samples_per_series = 14_400; + monolithic.value_total_samples_per_series = Some(14_400); + + for series_idx in 0..2 { + let expected = (0..monolithic.samples_per_series) + .map(|sample_idx| deterministic_prom_value(&monolithic, series_idx, sample_idx)) + .collect::>(); + for chunk_idx in 0..10 { + let mut chunk = args(ValuePattern::SeededRandom); + chunk.samples_per_series = 1_440; + chunk.value_sample_offset = chunk_idx * 1_440; + chunk.value_total_samples_per_series = Some(14_400); + for sample_idx in 0..chunk.samples_per_series { + assert_eq!( + deterministic_prom_value(&chunk, series_idx, sample_idx), + expected[(chunk.value_sample_offset + sample_idx) as usize] + ); + } + } + } + } + + #[test] + fn values_continue_across_series_boundaries() { + let mut args = args(ValuePattern::Unique); + args.samples_per_series = 16; + args.value_total_samples_per_series = Some(16); + args.value_base = 1.0; + args.value_step = 0.5; + + assert_eq!(deterministic_prom_value(&args, 0, 15), 8.5); + assert_eq!(deterministic_prom_value(&args, 1, 0), 9.0); + } +} diff --git a/src/cmd/src/bin/query_perf_fixture/util.rs b/src/cmd/src/bin/query_perf_fixture/util.rs new file mode 100644 index 00000000000..2c86baa6839 --- /dev/null +++ b/src/cmd/src/bin/query_perf_fixture/util.rs @@ -0,0 +1,23 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::path::Path; + +pub(super) fn case_name_from_path(path: &Path) -> String { + path.parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + .unwrap_or("query_perf_case") + .to_string() +} diff --git a/src/cmd/src/datanode.rs b/src/cmd/src/datanode.rs index 744624a5084..c33be6c6f33 100644 --- a/src/cmd/src/datanode.rs +++ b/src/cmd/src/datanode.rs @@ -15,11 +15,10 @@ pub mod builder; #[allow(clippy::print_stdout)] pub(crate) mod objbench; -#[cfg(feature = "dev-tools")] #[allow(clippy::print_stdout)] -mod parquetbench; +pub mod parquetbench; #[allow(clippy::print_stdout)] -mod scanbench; +pub mod scanbench; use std::path::Path; use std::time::Duration; @@ -40,7 +39,6 @@ use tracing_appender::non_blocking::WorkerGuard; use crate::App; use crate::datanode::builder::InstanceBuilder; use crate::datanode::objbench::ObjbenchCommand; -#[cfg(feature = "dev-tools")] use crate::datanode::parquetbench::ParquetbenchCommand; use crate::datanode::scanbench::ScanbenchCommand; use crate::error::{ @@ -115,7 +113,6 @@ impl Command { SubCommand::Start(cmd) => cmd.load_options(global_options), // Bench commands are standalone utilities and don't need to load DatanodeOptions. SubCommand::Objbench(_) | SubCommand::Scanbench(_) => Self::default_bench_options(), - #[cfg(feature = "dev-tools")] SubCommand::Parquetbench(_) => Self::default_bench_options(), } } @@ -141,7 +138,6 @@ pub enum SubCommand { /// Scan benchmark tool - benchmarks scanning a region directly from storage Scanbench(ScanbenchCommand), /// Benchmark scanning a single parquet SST. - #[cfg(feature = "dev-tools")] Parquetbench(ParquetbenchCommand), } @@ -160,7 +156,6 @@ impl SubCommand { cmd.run().await?; std::process::exit(0); } - #[cfg(feature = "dev-tools")] SubCommand::Parquetbench(cmd) => { cmd.run().await?; std::process::exit(0); diff --git a/tests/perf/AGENTS.md b/tests/perf/AGENTS.md index eb95544d22d..7f64b283bdc 100644 --- a/tests/perf/AGENTS.md +++ b/tests/perf/AGENTS.md @@ -11,14 +11,14 @@ docs together. - `[case]` is report metadata only. `[scenario]` is the executable regression configuration and must include `kind`, data layout, tables, queries, and - thresholds. Keep scenario parsing explicit in both - `tests/perf/query_regression_runner.py` and - `src/cmd/src/bin/query_perf_fixture.rs`. + thresholds. Rust owns case schema, defaults, validation, and normalized plan + output through `query_perf_fixture plan`; `tests/perf/query_regression_runner.py` + should only orchestrate from that normalized JSON. - Keep the direct-SST generator generic. Issue-specific behavior belongs in case files and thresholds, not in Rust generator logic. - Before pushing perf harness changes, run at least: - `uv run --no-project python -m py_compile .github/scripts/query-regression-run.py .github/scripts/query-regression-summary.py .github/scripts/query-regression-pr-metadata.py tests/perf/query_regression_runner.py` - `cargo fmt --all -- --check` - - `cargo build -p cmd --bin query_perf_fixture` + - `cargo build -p cmd --bin query_perf_fixture --features dev-tools` - dry-run the Python runner and Rust fixture generator against all built-in cases when the DSL or workflow case selection changes. diff --git a/tests/perf/README.md b/tests/perf/README.md index a59081b5681..91bc6c1637e 100644 --- a/tests/perf/README.md +++ b/tests/perf/README.md @@ -4,6 +4,11 @@ This directory is for query performance cases that compare a base build with a candidate build. It is not a replacement for sqlness: the goal is to measure the effect of optimizer/query-engine changes on realistic scan work. +The Prometheus remote-write cases are end-to-end regression coverage for the +automatic write, flush, SST, and query path. They do not replace controlled +encoding experiments that explicitly compare `plain`/dictionary, +`no_dictionary`, BYTE_STREAM_SPLIT, or Auto policies. + ## Phase 1: direct readable SST fixtures Phase 1 should generate data by writing readable Mito SST files and matching @@ -40,8 +45,10 @@ The runner also supports `scenario.kind = "prom_remote_write_then_query"` for a bounded write-path smoke/regression flow. This path is explicit and separate from the direct-SST fixture path: it starts the base and candidate distributed clusters, writes deterministic Prometheus remote-write v1 samples through -`/v1/prometheus/write`, flushes the configured physical metric table, then runs -the configured SQL/TQL queries against the logical metric table. +`/v1/prometheus/write`, flushes the configured physical metric table, checks +visibility, runs the configured SQL/TQL queries, stops and awaits the datanode, +then optionally inspects the landed SST/Parquet footer, encoding, and size and +runs the read-bench against the quiescent data directory. Remote-write cases configure one database, one logical metric, and one physical table under `[scenario.remote_write]`: @@ -52,34 +59,121 @@ kind = "prom_remote_write_then_query" [scenario.remote_write] database = "public" -metric = "prom_remote_write_smoke" +metric = "prom_remote_write_seeded_random" physical_table = "greptime_physical_table" -series_count = 8 -samples_per_series = 30 +series_count = 512 +samples_per_series = 1440 +sample_chunk_size = 480 +flush_every_sample_chunks = 1 start_unix_millis = 1_704_067_200_000 -step_millis = 15_000 -chunk_series_count = 4 -visibility_timeout_seconds = 30 +step_millis = 60_000 +chunk_series_count = 128 +timeout_seconds = 180 +visibility_timeout_seconds = 120 # Optional: split by time so each helper invocation writes only this many samples # per series, then periodically flush the physical metric table to produce # multiple time-interval SSTs. -# sample_chunk_size = 300 -# flush_every_sample_chunks = 1 [scenario.remote_write.prom_store] pending_rows_flush_interval = "1s" +max_batch_rows = 100000 ``` +Cases can optionally add `[scenario.remote_write.value]` to control the generated +sample values without changing label cardinality or the runner lifecycle: + +```toml +[scenario.remote_write.value] +pattern = "quantized_signal" # linear, constant, modulo, unique, seeded_random, + # run_length, quantized_signal, + # signal_with_sporadic_stalls, mixed_signal_repeated +base = 0.0 +step = 0.125 +cardinality = 4096 # buckets for modulo/seeded_random/quantized_signal/run_length +seed = 12345 # deterministic seeded_random input +run_length = 8 # adjacent samples per bucket for run_length/quantized_signal +stall_every = 100 # interval for signal_with_sporadic_stalls +stall_length = 16 # held samples inside each stall interval +mixed_every = 5 # every Nth sample becomes the repeated base value +``` + +The default `linear` pattern preserves the helper's historical formula. Use +`constant` or low-cardinality `modulo`/`seeded_random` values for repeated-value +data shapes, `run_length` for run-heavy low-cardinality series, `quantized_signal` +for signal-like values collapsed into a finite bucket set, `signal_with_sporadic_stalls` +for mostly continuous signals with periodic flat spots, and `mixed_signal_repeated` +for signal-plus-periodic-default mixtures. `unique` or high-cardinality buckets +still work for broad sample-value distributions. This is a generic sample-value +control for query/ingestion cases; it does not inspect or assert storage +encoding, Parquet footers, or storage policy choices. For chunked remote-write +ingestion, the runner passes the sample offset and total sample count to the +helper so non-linear value patterns use a stable global/per-series ordinal across +chunks. + +Case schema, value distribution defaults, storage defaults, and read-bench +defaults are owned by Rust. The Python runner calls +`query_perf_fixture plan --case ` and orchestrates the normalized JSON. +The same helper exposes `direct-sst`, `prom-remote-write`, and `inspect-footer` +subcommands; the old direct invocation (`query_perf_fixture --case ... --out-dir +...`) remains compatible. + +Remote-write cases that need to validate storage output can add +`[scenario.remote_write.storage]`. When present, the scenario body becomes: +remote-write → flush → visibility check → query measurement → stop and await +datanode → Parquet footer/size/encoding inspection → optional read-bench. Footer +inspection is handled by `query_perf_fixture inspect-footer`, which reads local +Parquet footers directly; it does not require Python `pyarrow`. + +```toml +[scenario.remote_write.storage] +inspect = true +column = "greptime_value" +include_metadata_files = false +# Optional: inspect below datanode data home instead of the whole fresh data dir. +# root_suffix = "greptime/public/" +min_files = 1 +min_files_with_column = 1 +require_encodings = ["BYTE_STREAM_SPLIT"] +forbid_encodings = ["PLAIN_DICTIONARY"] +max_total_file_size_bytes = 104857600 +max_column_compressed_size_bytes = 52428800 +max_column_uncompressed_size_bytes = 209715200 +max_candidate_total_file_size_regression_pct = 10.0 +max_candidate_column_compressed_size_regression_pct = 10.0 +max_candidate_column_uncompressed_size_regression_pct = 10.0 +``` + +The `*_pct` storage thresholds are percent regression limits comparing candidate +against base; `pct` means percent, not percentile. Per-target storage checks +(`min_files`, `min_files_with_column`, required/forbidden encodings, and absolute +`max_*_bytes`) run for both base and candidate. Comparative +`max_candidate_*_regression_pct` checks remain base-vs-candidate. These checks +are generic footer and byte-size assertions and do not encode product-specific +heuristics. When storage inspection is enabled, `min_files` and +`min_files_with_column` default to `1` even if omitted, so dry-runs show these +planned checks and an empty flush or missing target column fails the scenario. +By default the inspector root is the target datanode data home, which is suitable +for a fresh single-case data directory. For reused or more complex data homes, set +`root_suffix` to a path relative to the datanode data home to narrow inspection. + +When storage inspection is enabled, `[scenario.remote_write.read_bench]` defaults +to enabled with both datanode `parquetbench` and `scanbench`. Disable it with +`enabled = false`. `parquetbench` measures per-SST reader cost, `scanbench` +measures region scan cost, and query measurements still exercise the SQL/TQL +frontend path. Treat all performance conclusions as release-only; debug builds +are suitable only for command wiring and correctness checks. + The runner creates the configured database if needed, writes a per-target frontend config enabling `[prom_store]` with metric engine storage and a non-zero `pending_rows_flush_interval`, and validates that the logical metric table reaches `series_count * samples_per_series` rows before trusting the query measurements. -Use `--remote-write-generator /path/to/prom_remote_write_fixture` to provide the -Rust payload helper. `--fixture-only` is rejected for remote-write cases; use -`--dry-run` for planning. +Use `--fixture-generator /path/to/query_perf_fixture` to provide the Rust helper. +Deprecated `--remote-write-generator` and `--storage-inspector` options are kept +only for CLI compatibility. `--fixture-only` is rejected for remote-write cases; +use `--dry-run` for planning. Large manual remote-write cases can set `sample_chunk_size` to split ingestion by -time. For each chunk, the runner invokes `prom_remote_write_fixture` with the +time. For each chunk, the runner invokes `query_perf_fixture prom-remote-write` with the same series cardinality but a shorter `--samples-per-series` and an advanced `--start-unix-millis`. `flush_every_sample_chunks` controls periodic `ADMIN FLUSH_TABLE('')` calls; with `flush_every_sample_chunks = 1`, @@ -89,6 +183,25 @@ need multi-window file distribution should span multiple compaction windows. If `sample_chunk_size` is omitted, the runner keeps the older single-helper-invocation behavior and flushes once at the end. +The default remote-write coverage set contains four cases with the same 2048 +series × 14,400 samples (29,491,200 rows per target) shape. Each writes ten +one-day chunks, flushes every chunk, requires at least two visible SSTs with the +value column, runs seven read-bench iterations with `parquetbench` capped at +four SSTs while `scanbench` covers the landed region, and measures one ten-day +TQL selector with two warmups and 15 iterations: + +- `prom_remote_write_seeded_random`: high-distinct seeded-random values with + cardinality 29,491,200 and seed 8444. +- `prom_remote_write_run_heavy`: low-cardinality values in exact 16-sample runs. +- `prom_remote_write_mixed_every`: continuous signal values with every fifth + sample repeated at the base value. +- `prom_remote_write_integer_counter`: strictly increasing integer counter + values, with disjoint ranges for each series. + +These cases record normal storage/footer and read-bench results but do not gate +specific value encodings or storage-size outcomes. Use the controlled encoding +experiment matrix for those policy comparisons. + `tests/perf/query_cases/prom_remote_write_7913/case.toml` is a larger manual case for issue #7913. It writes 8192 series × 20160 samples through remote-write in 1440-sample daily time chunks, flushing after each chunk before running 1d/7d/14d @@ -246,7 +359,8 @@ as a fallback. Manifest files are reflinked or copied, not hardlinked. Fixture generator smoke test: ```bash -cargo run -p cmd --bin query_perf_fixture -- \ +cargo run -p cmd --bin query_perf_fixture --features dev-tools -- \ + direct-sst \ --case tests/perf/query_cases/smoke_direct_sst/case.toml \ --out-dir /tmp/query-perf-smoke ``` @@ -271,10 +385,10 @@ Remote-write runner dry-run: ```bash uv run --no-project python tests/perf/query_regression_runner.py \ - --case tests/perf/query_cases/prom_remote_write_smoke/case.toml \ + --case tests/perf/query_cases/prom_remote_write_seeded_random/case.toml \ --base-bin /path/to/base/greptime \ --candidate-bin /path/to/candidate/greptime \ - --remote-write-generator /path/to/prom_remote_write_fixture \ + --fixture-generator /path/to/query_perf_fixture \ --work-dir /tmp/query-perf-remote-write \ --dry-run ``` @@ -289,14 +403,22 @@ query regression runs. It builds its own binaries for now: candidate checkout - runner and summary formatter from the candidate checkout -The workflow runs automatically only for non-draft PRs labeled -`query-regression` (on label/ready-for-review/reopen events, not every push). +The workflow builds base and candidate `greptime` as normal release-equivalent +binaries. Candidate `query_perf_fixture` is the extra head-side helper binary; +the runner uses candidate `greptime datanode parquetbench/scanbench` as the +read-bench tool against each target's data directory. + +The workflow runs automatically only when `query-regression` is added to a +non-draft PR; it does not rerun on pushes, ready-for-review, or reopen events. PR runs build base/candidate once and then run the default case set with `--allow-large-fixture`. Manual `workflow_dispatch` runs can pass `all`, one case path, or a comma/whitespace-separated list of case paths, and can override refs. -It always uploads `query-regression-work/**` and `query-regression-summary.md`, -writes the Markdown summary to the workflow step summary, and updates a sticky PR -comment through the trusted follow-up workflow. +The main report artifact uploads only aggregate/per-target JSON reports, +component logs, and `query-regression-summary.md` with seven-day retention; +fixture data, SSTs, and cluster state are excluded. PR runs also upload a +separate trusted-comment artifact containing PR metadata and aggregate reports. +The workflow writes the Markdown summary to the workflow step summary and +updates a sticky PR comment through the trusted follow-up workflow. ## Built-in cases diff --git a/tests/perf/query_cases/prom_remote_write_integer_counter/case.toml b/tests/perf/query_cases/prom_remote_write_integer_counter/case.toml new file mode 100644 index 00000000000..3ac64a94b9d --- /dev/null +++ b/tests/perf/query_cases/prom_remote_write_integer_counter/case.toml @@ -0,0 +1,48 @@ +[case] +name = "prom_remote_write_integer_counter" +description = "End-to-end Prometheus remote-write regression with strictly increasing integer counter values" + +[scenario] +kind = "prom_remote_write_then_query" + +[scenario.remote_write] +database = "public" +metric = "prom_remote_write_integer_counter" +physical_table = "greptime_physical_table" +series_count = 2048 +samples_per_series = 14400 +sample_chunk_size = 1440 +flush_every_sample_chunks = 1 +start_unix_millis = 1704067200000 +step_millis = 60000 +chunk_series_count = 256 +timeout_seconds = 600 +visibility_timeout_seconds = 300 + +[scenario.remote_write.value] +pattern = "unique" +base = 0 +step = 1 + +[scenario.remote_write.prom_store] +pending_rows_flush_interval = "1s" +max_batch_rows = 100000 + +[scenario.remote_write.storage] +column = "greptime_value" +min_files = 2 +min_files_with_column = 2 + +[scenario.remote_write.read_bench] +iterations = 7 +max_files = 4 + +[[scenario.queries]] +name = "selector_10d_integer_counter" +kind = "tql" +query = "TQL ANALYZE VERBOSE (1704067200, 1704931200, '1h') prom_remote_write_integer_counter{host=~'host.*'}" +warmup = 2 +iterations = 15 + +[scenario.queries.thresholds] +max_candidate_latency_regression_pct = 75 diff --git a/tests/perf/query_cases/prom_remote_write_mixed_every/case.toml b/tests/perf/query_cases/prom_remote_write_mixed_every/case.toml new file mode 100644 index 00000000000..9ba39f2fbed --- /dev/null +++ b/tests/perf/query_cases/prom_remote_write_mixed_every/case.toml @@ -0,0 +1,49 @@ +[case] +name = "prom_remote_write_mixed_every" +description = "End-to-end Prometheus remote-write regression with periodic repeated signal values" + +[scenario] +kind = "prom_remote_write_then_query" + +[scenario.remote_write] +database = "public" +metric = "prom_remote_write_mixed_every" +physical_table = "greptime_physical_table" +series_count = 2048 +samples_per_series = 14400 +sample_chunk_size = 1440 +flush_every_sample_chunks = 1 +start_unix_millis = 1704067200000 +step_millis = 60000 +chunk_series_count = 256 +timeout_seconds = 600 +visibility_timeout_seconds = 300 + +[scenario.remote_write.value] +pattern = "mixed_signal_repeated" +base = 0 +step = 0.125 +mixed_every = 5 + +[scenario.remote_write.prom_store] +pending_rows_flush_interval = "1s" +max_batch_rows = 100000 + +[scenario.remote_write.storage] +column = "greptime_value" +min_files = 2 +min_files_with_column = 2 + +[scenario.remote_write.read_bench] +iterations = 7 +max_files = 4 + +[[scenario.queries]] +name = "selector_10d_mixed_every" +kind = "tql" +query = "TQL ANALYZE VERBOSE (1704067200, 1704931200, '1h') prom_remote_write_mixed_every{host=~'host.*'}" +warmup = 2 +iterations = 15 + +[scenario.queries.thresholds] +max_candidate_latency_regression_pct = 75 diff --git a/tests/perf/query_cases/prom_remote_write_run_heavy/case.toml b/tests/perf/query_cases/prom_remote_write_run_heavy/case.toml new file mode 100644 index 00000000000..243892e42b1 --- /dev/null +++ b/tests/perf/query_cases/prom_remote_write_run_heavy/case.toml @@ -0,0 +1,50 @@ +[case] +name = "prom_remote_write_run_heavy" +description = "End-to-end Prometheus remote-write regression with repeated 16-sample value runs" + +[scenario] +kind = "prom_remote_write_then_query" + +[scenario.remote_write] +database = "public" +metric = "prom_remote_write_run_heavy" +physical_table = "greptime_physical_table" +series_count = 2048 +samples_per_series = 14400 +sample_chunk_size = 1440 +flush_every_sample_chunks = 1 +start_unix_millis = 1704067200000 +step_millis = 60000 +chunk_series_count = 256 +timeout_seconds = 600 +visibility_timeout_seconds = 300 + +[scenario.remote_write.value] +pattern = "run_length" +base = 0 +step = 1 +cardinality = 1024 +run_length = 16 + +[scenario.remote_write.prom_store] +pending_rows_flush_interval = "1s" +max_batch_rows = 100000 + +[scenario.remote_write.storage] +column = "greptime_value" +min_files = 2 +min_files_with_column = 2 + +[scenario.remote_write.read_bench] +iterations = 7 +max_files = 4 + +[[scenario.queries]] +name = "selector_10d_run_heavy" +kind = "tql" +query = "TQL ANALYZE VERBOSE (1704067200, 1704931200, '1h') prom_remote_write_run_heavy{host=~'host.*'}" +warmup = 2 +iterations = 15 + +[scenario.queries.thresholds] +max_candidate_latency_regression_pct = 75 diff --git a/tests/perf/query_cases/prom_remote_write_seeded_random/case.toml b/tests/perf/query_cases/prom_remote_write_seeded_random/case.toml new file mode 100644 index 00000000000..6a63edbcbbf --- /dev/null +++ b/tests/perf/query_cases/prom_remote_write_seeded_random/case.toml @@ -0,0 +1,48 @@ +[case] +name = "prom_remote_write_seeded_random" +description = "End-to-end Prometheus remote-write regression with high-distinct seeded-random values" + +[scenario] +kind = "prom_remote_write_then_query" + +[scenario.remote_write] +database = "public" +metric = "prom_remote_write_seeded_random" +physical_table = "greptime_physical_table" +series_count = 2048 +samples_per_series = 14400 +sample_chunk_size = 1440 +flush_every_sample_chunks = 1 +start_unix_millis = 1704067200000 +step_millis = 60000 +chunk_series_count = 256 +timeout_seconds = 600 +visibility_timeout_seconds = 300 + +[scenario.remote_write.value] +pattern = "seeded_random" +cardinality = 29_491_200 +seed = 8444 + +[scenario.remote_write.prom_store] +pending_rows_flush_interval = "1s" +max_batch_rows = 100000 + +[scenario.remote_write.storage] +column = "greptime_value" +min_files = 2 +min_files_with_column = 2 + +[scenario.remote_write.read_bench] +iterations = 7 +max_files = 4 + +[[scenario.queries]] +name = "selector_10d_seeded_random" +kind = "tql" +query = "TQL ANALYZE VERBOSE (1704067200, 1704931200, '1h') prom_remote_write_seeded_random{host=~'host.*'}" +warmup = 2 +iterations = 15 + +[scenario.queries.thresholds] +max_candidate_latency_regression_pct = 75 diff --git a/tests/perf/query_cases/prom_remote_write_smoke/case.toml b/tests/perf/query_cases/prom_remote_write_smoke/case.toml deleted file mode 100644 index b946fbc8f51..00000000000 --- a/tests/perf/query_cases/prom_remote_write_smoke/case.toml +++ /dev/null @@ -1,31 +0,0 @@ -[case] -name = "prom_remote_write_smoke" -description = "Tiny Prometheus remote-write ingestion then TQL query smoke case" - -[scenario] -kind = "prom_remote_write_then_query" - -[scenario.remote_write] -database = "public" -metric = "prom_remote_write_smoke" -physical_table = "greptime_physical_table" -series_count = 8 -samples_per_series = 30 -start_unix_millis = 1_704_067_200_000 -step_millis = 15_000 -chunk_series_count = 4 -timeout_seconds = 30 - -[scenario.remote_write.prom_store] -pending_rows_flush_interval = "1s" -max_batch_rows = 10000 - -[[scenario.queries]] -name = "selector_5m_smoke" -kind = "tql" -query = "TQL ANALYZE VERBOSE (1704067200, 1704067650, '15s') prom_remote_write_smoke{host=~'host.*'}" -warmup = 0 -iterations = 2 - -[scenario.queries.thresholds] -max_candidate_latency_regression_pct = 100 diff --git a/tests/perf/query_regression_runner.py b/tests/perf/query_regression_runner.py index a7f2646ea8b..f5c9628b83d 100644 --- a/tests/perf/query_regression_runner.py +++ b/tests/perf/query_regression_runner.py @@ -66,7 +66,8 @@ def parse_args() -> argparse.Namespace: p.add_argument("--base-bin", required=True, type=Path) p.add_argument("--candidate-bin", required=True, type=Path) p.add_argument("--fixture-generator", type=Path) - p.add_argument("--remote-write-generator", type=Path, help="prom_remote_write_fixture helper binary") + p.add_argument("--remote-write-generator", type=Path, help="deprecated: use --fixture-generator query_perf_fixture") + p.add_argument("--storage-inspector", type=Path, help="deprecated: use --fixture-generator query_perf_fixture") p.add_argument("--work-dir", required=True, type=Path) p.add_argument("--fixture-cache-dir", type=Path, help="persistent directory for generated fixtures, keyed by case content") p.add_argument("--reuse-fixture", action="store_true") @@ -83,6 +84,16 @@ def load_case(path: Path) -> dict[str, Any]: return tomllib.load(f) +def load_normalized_case(case_path: Path, fixture_generator: Path | None) -> dict[str, Any]: + if fixture_generator is None: + raise ValueError("--fixture-generator query_perf_fixture is required for Rust-owned case planning") + result = run_command([str(fixture_generator), "plan", "--case", str(case_path)]) + if result["returncode"] != 0: + raise RuntimeError(f"query_perf_fixture plan failed: {result['stderr'][:2000]}") + plan = json.loads(result["stdout"]) + return {"case": load_case(case_path).get("case", {}), "scenario": plan["scenario"], "schema_version": plan.get("schema_version")} + + def require_binary(path: Path, name: str, *, dry_run: bool) -> None: if dry_run: return @@ -157,11 +168,9 @@ def scenario(case: dict[str, Any]) -> dict[str, Any]: def case_tables(case: dict[str, Any]) -> list[dict[str, Any]]: value = scenario(case) if value.get("kind") == "prom_remote_write_then_query": - remote = value.get("remote_write") or {} - metric = remote.get("metric") or remote.get("metric_name") - database = remote.get("database", "public") - if not metric: - raise ValueError("remote-write scenario requires scenario.remote_write.metric") + remote = value["remote_write"] + metric = remote["metric"] + database = remote["database"] return [{"database": database, "name": metric, "engine": "metric", "validate_show_create_engine": False}] tables = value.get("tables") or [] if not tables or value.get("layout", {}).get("regions") != 1: @@ -548,7 +557,7 @@ def generate_fixture(generator: Path | None, case_path: Path, fixture_dir: Path, if fixture_dir.exists() and not reuse_fixture: shutil.rmtree(fixture_dir) fixture_dir.mkdir(parents=True, exist_ok=True) - cmd = [str(generator), "--case", str(case_path), "--out-dir", str(fixture_dir)] + cmd = [str(generator), "direct-sst", "--case", str(case_path), "--out-dir", str(fixture_dir)] if table_name is not None: cmd += ["--table", table_name] if region_id is not None: @@ -777,6 +786,243 @@ def enforce_thresholds(case: dict[str, Any], base: dict[str, Any], candidate: di return results +def storage_config(remote: dict[str, Any]) -> dict[str, Any] | None: + value = remote["storage"] + if not isinstance(value, dict): + return None + if value["inspect"] is False: + return None + return value + + +def run_storage_inspection(helper: Path | None, target: RunTarget, storage: dict[str, Any], *, dry_run: bool) -> dict[str, Any]: + if helper is None: + if not dry_run: + raise ValueError("--fixture-generator query_perf_fixture is required when storage inspection is enabled") + helper = Path("query_perf_fixture") + root = target.datanode_data_dir + if storage.get("root_suffix"): + root = root / str(storage["root_suffix"]) + cmd = [ + str(helper), + "inspect-footer", + "--root", str(root), + "--column", str(storage["column"]), + ] + if storage["include_metadata_files"]: + cmd.append("--include-metadata-files") + if dry_run: + return {"status": "dry-run", "cmd": cmd, "root": str(root)} + result = run_command(cmd) + result["status"] = "ok" if result["returncode"] == 0 else "failed" + if result["returncode"] != 0: + raise RuntimeError(f"storage inspector failed for {target.name}: {result['stderr'][:2000]}") + try: + result["summary"] = json.loads(result["stdout"]) + except json.JSONDecodeError: + result["summary_parse_error"] = result["stdout"] + result["status"] = "failed" + return result + + +def storage_summary(inspection: dict[str, Any]) -> dict[str, Any]: + summary = inspection.get("summary") + if isinstance(summary, dict): + nested = summary.get("summary") + if isinstance(nested, dict): + return nested + return {} + + +def enforce_storage_thresholds(storage: dict[str, Any] | None, base: dict[str, Any] | None, candidate: dict[str, Any] | None) -> list[dict[str, Any]]: + if not storage: + return [] + results: list[dict[str, Any]] = [] + target_summaries = [("base", storage_summary(base or {})), ("candidate", storage_summary(candidate or {}))] + cand = target_summaries[1][1] + base_summary = storage_summary(base or {}) + + def add_abs(name: str, field: str) -> None: + limit = storage.get(name) + if limit is None: + return + for target_name, summary in target_summaries: + actual = summary.get(field) + ok = actual is not None and float(actual) <= float(limit) + results.append({"target": target_name, "threshold": name, "status": "passed" if ok else "failed", "actual": actual, "limit": limit}) + + def add_min(name: str, field: str) -> None: + limit = storage[name] + if limit is None: + return + for target_name, summary in target_summaries: + actual = summary.get(field) + ok = actual is not None and int(actual) >= int(limit) + results.append({"target": target_name, "threshold": name, "status": "passed" if ok else "failed", "actual": actual, "limit": limit}) + + def add_cmp(name: str, field: str) -> None: + limit = storage.get(name) + if limit is None: + return + base_value = base_summary.get(field) + candidate_value = cand.get(field) + if base_value in (None, 0) or candidate_value is None: + results.append({"threshold": name, "status": "failed", "reason": "missing or zero base/candidate value", "base": base_value, "candidate": candidate_value}) + return + actual = (float(candidate_value) - float(base_value)) / float(base_value) * 100.0 + results.append({"threshold": name, "status": "passed" if actual <= float(limit) else "failed", "actual_pct": actual, "limit_pct": limit, "base": base_value, "candidate": candidate_value}) + + add_min("min_files", "file_count") + add_min("min_files_with_column", "files_with_column") + add_abs("max_total_file_size_bytes", "total_file_size") + add_abs("max_column_compressed_size_bytes", "column_compressed_size") + add_abs("max_column_uncompressed_size_bytes", "column_uncompressed_size") + for target_name, summary in target_summaries: + encodings = set(summary.get("unique_encodings") or []) + for required in storage["require_encodings"]: + results.append({"target": target_name, "threshold": "require_encodings", "encoding": required, "status": "passed" if required in encodings else "failed"}) + for forbidden in storage["forbid_encodings"]: + results.append({"target": target_name, "threshold": "forbid_encodings", "encoding": forbidden, "status": "passed" if forbidden not in encodings else "failed"}) + add_cmp("max_candidate_total_file_size_regression_pct", "total_file_size") + add_cmp("max_candidate_column_compressed_size_regression_pct", "column_compressed_size") + add_cmp("max_candidate_column_uncompressed_size_regression_pct", "column_uncompressed_size") + return results + + +def planned_storage_thresholds(storage: dict[str, Any] | None) -> list[dict[str, Any]]: + if not storage: + return [] + return list(storage["planned_thresholds"]) + + +REGION_DIR_RE = re.compile(r"^(\d+)_(\d{10})$") + + +def inspection_report(inspection: dict[str, Any] | None) -> dict[str, Any]: + if not inspection: + return {} + report = inspection.get("summary") + return report if isinstance(report, dict) else {} + + +def inspected_relative_path(target: RunTarget, inspection: dict[str, Any], relative_path: str) -> Path: + report = inspection_report(inspection) + root_text = report.get("root") or inspection.get("root") + if not root_text: + return Path(relative_path) + root = Path(root_text).resolve(strict=False) + data_home = target.datanode_data_dir.resolve(strict=False) + try: + root_suffix = root.relative_to(data_home) + except ValueError as e: + raise RuntimeError(f"storage inspection root {root} is not under datanode data home {data_home}") from e + return root_suffix / relative_path + + +def parse_sst_bench_target(target: RunTarget, inspection: dict[str, Any], file: dict[str, Any], *, dry_run: bool) -> dict[str, str] | None: + relative_path = str(file.get("relative_path") or "") + if dry_run and (not relative_path or "<" in relative_path): + return { + "relative_path": relative_path or "/_/data/.parquet", + "table_dir": "/", + "region_id": ":", + "path_type": "data", + "file_id": "", + } + if not relative_path: + return None + full_relative = inspected_relative_path(target, inspection, relative_path) + parts = full_relative.parts + if not parts or not parts[-1].endswith(".parquet"): + return None + region_index = next((idx for idx, part in enumerate(parts) if REGION_DIR_RE.match(part)), None) + if region_index is None or region_index == 0: + if dry_run: + return { + "relative_path": relative_path, + "table_dir": "/", + "region_id": ":", + "path_type": "data", + "file_id": Path(parts[-1]).stem, + } + return None + region_match = REGION_DIR_RE.match(parts[region_index]) + assert region_match is not None + file_index = len(parts) - 1 + path_type = "bare" + if region_index + 1 < file_index and parts[region_index + 1] in ("data", "metadata"): + path_type = parts[region_index + 1] + if path_type == "metadata": + return None + table_dir = "/".join(parts[:region_index]).rstrip("/") + "/" + return { + "relative_path": str(full_relative), + "table_dir": table_dir, + "region_id": f"{int(region_match.group(1))}:{int(region_match.group(2))}", + "path_type": path_type, + "file_id": Path(parts[-1]).stem, + } + + +def run_read_bench(bench_binary: Path, target: RunTarget, read_bench: dict[str, Any] | None, storage_inspection: dict[str, Any] | None, *, dry_run: bool) -> dict[str, Any]: + if not read_bench or read_bench["enabled"] is False: + return {"status": "skipped", "reason": "read_bench disabled"} + run_parquetbench = read_bench["parquetbench"] + run_scanbench = read_bench["scanbench"] + if not run_parquetbench and not run_scanbench: + return {"status": "skipped", "reason": "parquetbench and scanbench disabled"} + if storage_inspection is None: + raise ValueError("read_bench requires storage inspection") + bench_dir = target.work_dir / "read_bench" + config_toml = bench_dir / "bench.toml" + scan_json = bench_dir / "scan.json" + files = inspection_report(storage_inspection).get("files") or [] + ssts = [f for f in files if f.get("relative_path", "").endswith(".parquet") and f.get("columns")] + if dry_run and not ssts: + ssts = [{"relative_path": ".parquet"}] + if read_bench["max_files"] is not None: + ssts = ssts[: int(read_bench["max_files"])] + bench_targets = [parsed for f in ssts if (parsed := parse_sst_bench_target(target, storage_inspection, f, dry_run=dry_run)) is not None] + if not bench_targets: + return {"status": "failed", "reason": "no inspected data SST files available for read_bench"} + config_text = f'[storage]\ndata_home = "{target.datanode_data_dir}"\ntype = "File"\n\n[[region_engine]]\n[region_engine.mito]\n' + commands = [] + parquet_runs = [] + scan_runs = [] + iterations = str(int(read_bench["iterations"])) + if run_parquetbench: + for bench_target in bench_targets: + cmd = [str(bench_binary), "datanode", "parquetbench", "--config", str(config_toml), "--region-id", bench_target["region_id"], "--table-dir", bench_target["table_dir"], "--file-id", bench_target["file_id"], "--scan-config", str(scan_json), "--path-type", bench_target["path_type"], "--iterations", iterations, "--reader", str(read_bench["parquet_reader"])] + commands.append(cmd) + parquet_runs.append({**bench_target, "cmd": cmd, "status": "dry-run" if dry_run else "planned"}) + regions: dict[tuple[str, str, str], list[str]] = {} + if run_scanbench: + for bench_target in bench_targets: + regions.setdefault((bench_target["table_dir"], bench_target["region_id"], bench_target["path_type"]), []).append(bench_target["relative_path"]) + for (table_dir, region_id, path_type), region_files in regions.items(): + cmd = [str(bench_binary), "datanode", "scanbench", "--config", str(config_toml), "--region-id", region_id, "--table-dir", table_dir, "--scan-config", str(scan_json), "--path-type", path_type, "--scanner", str(read_bench["scan_scanner"]), "--parallelism", str(int(read_bench["parallelism"])), "--iterations", iterations] + commands.append(cmd) + scan_runs.append({"table_dir": table_dir, "region_id": region_id, "path_type": path_type, "files": region_files, "cmd": cmd, "status": "dry-run" if dry_run else "planned"}) + if dry_run: + return {"status": "dry-run", "config_path": str(config_toml), "scan_config_path": str(scan_json), "commands": commands, "parquetbench": parquet_runs, "scanbench": scan_runs} + bench_dir.mkdir(parents=True, exist_ok=True) + config_toml.write_text(config_text) + scan_json.write_text(json.dumps({"projection_names": read_bench["projection"]}, indent=2) + "\n") + avg_re = re.compile(r"Average duration[^0-9]*([0-9.]+)\s*ms", re.I) + for runs in (parquet_runs, scan_runs): + for run in runs: + result = run_command(run["cmd"]) + run.update(result) + run["status"] = "ok" if result["returncode"] == 0 else "failed" + m = avg_re.search(result.get("stdout", "")) + if m: + run["average_ms"] = float(m.group(1)) + def median(runs: list[dict[str, Any]]) -> float | None: + vals = [r["average_ms"] for r in runs if "average_ms" in r] + return statistics.median(vals) if vals else None + return {"status": "ok" if all(r.get("status") == "ok" for r in parquet_runs + scan_runs) else "failed", "config_path": str(config_toml), "scan_config_path": str(scan_json), "parquetbench": parquet_runs, "scanbench": scan_runs, "aggregate": {"parquetbench_median_average_ms": median(parquet_runs), "scanbench_median_average_ms": median(scan_runs)}} + + def write_json(path: Path, payload: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") @@ -791,18 +1037,18 @@ def require_fresh_work_dirs(targets: list[RunTarget], *, reuse_work_dir: bool, d def write_frontend_prom_config(target: RunTarget, remote: dict[str, Any]) -> Path: - prom = remote.get("prom_store") or {} + prom = remote["prom_store"] path = target.work_dir / "frontend-prom-store.toml" path.parent.mkdir(parents=True, exist_ok=True) path.write_text( "[prom_store]\n" "enable = true\n" "with_metric_engine = true\n" - f"pending_rows_flush_interval = {json.dumps(str(prom.get('pending_rows_flush_interval', '1s')))}\n" - f"max_batch_rows = {int(prom.get('max_batch_rows', 100000))}\n" - f"max_concurrent_flushes = {int(prom.get('max_concurrent_flushes', 256))}\n" - f"worker_channel_capacity = {int(prom.get('worker_channel_capacity', 65526))}\n" - f"max_inflight_requests = {int(prom.get('max_inflight_requests', 3000))}\n" + f"pending_rows_flush_interval = {json.dumps(str(prom['pending_rows_flush_interval']))}\n" + f"max_batch_rows = {int(prom['max_batch_rows'])}\n" + f"max_concurrent_flushes = {int(prom['max_concurrent_flushes'])}\n" + f"worker_channel_capacity = {int(prom['worker_channel_capacity'])}\n" + f"max_inflight_requests = {int(prom['max_inflight_requests'])}\n" ) return path @@ -810,23 +1056,38 @@ def write_frontend_prom_config(target: RunTarget, remote: dict[str, Any]) -> Pat def run_remote_write(generator: Path | None, target: RunTarget, remote: dict[str, Any], *, dry_run: bool) -> dict[str, Any]: if generator is None: if not dry_run: - raise ValueError("--remote-write-generator is required for prom_remote_write_then_query unless --dry-run is set") - generator = Path("prom_remote_write_fixture") - metric = remote.get("metric") or remote.get("metric_name") - physical_table = remote.get("physical_table", "greptime_physical_table") + raise ValueError("--fixture-generator query_perf_fixture is required for prom_remote_write_then_query") + generator = Path("query_perf_fixture") + metric = remote["metric"] + physical_table = remote["physical_table"] cmd = [ str(generator), + "prom-remote-write", "--endpoint", f"http://127.0.0.1:{target.http_port}/v1/prometheus/write", - "--database", remote.get("database", "public"), + "--database", remote["database"], "--metric", metric, "--physical-table", physical_table, - "--series-count", str(int(remote.get("series_count", 8))), - "--samples-per-series", str(int(remote.get("samples_per_series", 30))), - "--start-unix-millis", str(int(remote.get("start_unix_millis", 1_704_067_200_000))), - "--step-millis", str(int(remote.get("step_millis", 15_000))), - "--chunk-series-count", str(int(remote.get("chunk_series_count", remote.get("batch_size", 8)))), - "--timeout-seconds", str(int(remote.get("timeout_seconds", 60))), + "--series-count", str(int(remote["series_count"])), + "--samples-per-series", str(int(remote["samples_per_series"])), + "--start-unix-millis", str(int(remote["start_unix_millis"])), + "--step-millis", str(int(remote["step_millis"])), + "--chunk-series-count", str(int(remote["chunk_series_count"])), + "--timeout-seconds", str(int(remote["timeout_seconds"])), ] + value = remote["value"] + cmd.extend(["--value-pattern", str(value["pattern"])]) + cmd.extend(["--value-base", str(value["base"])]) + cmd.extend(["--value-step", str(value["step"])]) + cmd.extend(["--value-cardinality", str(int(value["cardinality"]))]) + cmd.extend(["--value-seed", str(int(value["seed"]))]) + cmd.extend(["--value-run-length", str(int(value["run_length"]))]) + cmd.extend(["--value-stall-every", str(int(value["stall_every"]))]) + cmd.extend(["--value-stall-length", str(int(value["stall_length"]))]) + cmd.extend(["--value-mixed-every", str(int(value["mixed_every"]))]) + if "sample_offset" in remote: + cmd.extend(["--value-sample-offset", str(int(remote["sample_offset"]))]) + if "total_samples_per_series" in remote: + cmd.extend(["--value-total-samples-per-series", str(int(remote["total_samples_per_series"]))]) if dry_run: return {"status": "dry-run", "cmd": cmd} result = run_command(cmd) @@ -862,23 +1123,23 @@ def flush_remote_physical_table(target: RunTarget, db: str, physical_table: str, def run_remote_write_ingestion(generator: Path | None, target: RunTarget, remote: dict[str, Any], args: argparse.Namespace, *, dry_run: bool) -> tuple[dict[str, Any], list[dict[str, Any]]]: - sample_chunk_size = remote.get("sample_chunk_size") - db = remote.get("database", "public") - physical_table = remote.get("physical_table", "greptime_physical_table") + sample_chunk_size = remote["sample_chunk_size"] + db = remote["database"] + physical_table = remote["physical_table"] if sample_chunk_size is None: rw = run_remote_write(generator, target, remote, dry_run=dry_run) flush = flush_remote_physical_table(target, db, physical_table, args, dry_run=dry_run, reason="final") return rw, [flush] - total_samples = int(remote.get("samples_per_series", 30)) + total_samples = int(remote["samples_per_series"]) chunk_samples = int(sample_chunk_size) if chunk_samples <= 0: raise ValueError("scenario.remote_write.sample_chunk_size must be positive") - flush_every = int(remote.get("flush_every_sample_chunks", 1)) + flush_every = int(remote["flush_every_sample_chunks"]) if flush_every <= 0: raise ValueError("scenario.remote_write.flush_every_sample_chunks must be positive") - start = int(remote.get("start_unix_millis", 1_704_067_200_000)) - step = int(remote.get("step_millis", 15_000)) + start = int(remote["start_unix_millis"]) + step = int(remote["step_millis"]) chunks: list[dict[str, Any]] = [] flushes: list[dict[str, Any]] = [] chunk_index = 0 @@ -889,6 +1150,8 @@ def run_remote_write_ingestion(generator: Path | None, target: RunTarget, remote chunk_remote = dict(remote) chunk_remote["samples_per_series"] = current chunk_remote["start_unix_millis"] = start + offset * step + chunk_remote["sample_offset"] = offset + chunk_remote["total_samples_per_series"] = total_samples result = run_remote_write(generator, target, chunk_remote, dry_run=dry_run) result["sample_offset"] = offset result["samples_per_series"] = current @@ -901,8 +1164,8 @@ def run_remote_write_ingestion(generator: Path | None, target: RunTarget, remote flushes.append(flush_remote_physical_table(target, db, physical_table, args, dry_run=dry_run, reason="final", chunk_index=chunk_index)) aggregate = summarize_remote_write_chunks(chunks) if dry_run: - series_count = int(remote.get("series_count", 8)) - chunk_series_count = int(remote.get("chunk_series_count", remote.get("batch_size", 8))) + series_count = int(remote["series_count"]) + chunk_series_count = int(remote["chunk_series_count"]) aggregate["rows"] = series_count * total_samples aggregate["samples_written"] = aggregate["rows"] aggregate["batches"] = chunk_index * ((series_count + chunk_series_count - 1) // chunk_series_count) @@ -917,7 +1180,7 @@ def run_remote_write_ingestion(generator: Path | None, target: RunTarget, remote def expected_remote_write_rows(remote: dict[str, Any]) -> int: - return int(remote.get("series_count", 8)) * int(remote.get("samples_per_series", 30)) + return int(remote["series_count"]) * int(remote["samples_per_series"]) def extract_count_value(result: dict[str, Any]) -> int | None: @@ -966,16 +1229,21 @@ def poll_expected_count(target: RunTarget, table_name: str, db: str, expected_ro def run_remote_write_scenario(args: argparse.Namespace, case: dict[str, Any], case_path: Path, targets: list[RunTarget], report: dict[str, Any]) -> None: - remote = scenario(case).get("remote_write") or {} + remote = scenario(case)["remote_write"] tables = case_tables(case) if args.fixture_only: raise ValueError("--fixture-only is not supported for prom_remote_write_then_query; use --dry-run for planning") - if args.remote_write_generator is not None: - require_binary(args.remote_write_generator, "remote-write generator", dry_run=args.dry_run) + helper = args.fixture_generator or args.remote_write_generator + if helper is not None: + require_binary(helper, "query_perf_fixture", dry_run=args.dry_run) elif not args.dry_run: - raise ValueError("--remote-write-generator is required for prom_remote_write_then_query") + raise ValueError("--fixture-generator is required for prom_remote_write_then_query") + storage = storage_config(remote) + if storage and helper is None and not args.dry_run: + raise ValueError("--fixture-generator is required when scenario.remote_write.storage.inspect is enabled") clusters: list[DistributedCluster] = [] target_results = [] + storage_results = [] try: for target in targets: target.work_dir.mkdir(parents=True, exist_ok=True) @@ -984,28 +1252,40 @@ def run_remote_write_scenario(args: argparse.Namespace, case: dict[str, Any], ca clusters.append(cluster) if not args.dry_run: cluster.start_all(config_path) - db = remote.get("database", "public") + db = remote["database"] create_database = {"status": "dry-run", "database": db} if not args.dry_run: create_database = http_post_sql(target.http_port, f"CREATE DATABASE IF NOT EXISTS {sql_ident(db)}", "public", args.http_timeout) - rw, flushes = run_remote_write_ingestion(args.remote_write_generator, target, remote, args, dry_run=args.dry_run) - physical = remote.get("physical_table", "greptime_physical_table") + rw, flushes = run_remote_write_ingestion(helper, target, remote, args, dry_run=args.dry_run) visibility = {"status": "dry-run"} query_result = {"validation": [], "validation_errors": [], "measurements": [], "status": "planned"} if not args.dry_run: - visibility = poll_expected_count(target, tables[0]["name"], db, expected_remote_write_rows(remote), float(remote.get("visibility_timeout_seconds", 30)), args.http_timeout) + visibility = poll_expected_count(target, tables[0]["name"], db, expected_remote_write_rows(remote), float(remote["visibility_timeout_seconds"]), args.http_timeout) query_result = run_queries(target, case, tables, args.http_timeout) + cluster.stop_component("datanode") + storage_inspection = None + if storage: + storage_inspection = run_storage_inspection(helper or args.storage_inspector, target, storage, dry_run=args.dry_run) + storage_results.append(storage_inspection) + read_bench_result = run_read_bench(args.candidate_bin, target, remote["read_bench"], storage_inspection, dry_run=args.dry_run) tr = {"name": target.name, "binary": str(target.binary), "work_dir": str(target.work_dir), "components": cluster.component_report(), "frontend_config": str(config_path), "create_database": create_database, "remote_write": rw, "flushes": flushes, "flush": flushes[-1] if flushes else None, "visibility": visibility, **query_result} + if storage_inspection is not None: + tr["storage_inspection"] = storage_inspection + tr["read_bench"] = read_bench_result flushes_ok = all(flush.get("ok") for flush in flushes) - remote_checks_ok = args.dry_run or (create_database.get("ok") and flushes_ok and visibility.get("ok") and visibility.get("row_count_ok")) + storage_ok = storage_inspection is None or args.dry_run or storage_inspection.get("status") == "ok" + read_bench_ok = args.dry_run or read_bench_result.get("status") in ("ok", "skipped") + remote_checks_ok = args.dry_run or (create_database.get("ok") and flushes_ok and visibility.get("ok") and visibility.get("row_count_ok") and storage_ok and read_bench_ok) if not remote_checks_ok: - tr.setdefault("validation_errors", []).append({"phase": "remote_write_visibility", "create_database_ok": create_database.get("ok"), "flushes_ok": flushes_ok, "visibility_ok": visibility.get("ok"), "row_count_ok": visibility.get("row_count_ok"), "create_database": create_database, "flushes": flushes, "visibility": visibility}) + tr.setdefault("validation_errors", []).append({"phase": "remote_write_visibility_storage", "create_database_ok": create_database.get("ok"), "flushes_ok": flushes_ok, "visibility_ok": visibility.get("ok"), "row_count_ok": visibility.get("row_count_ok"), "storage_ok": storage_ok, "read_bench_ok": read_bench_ok, "create_database": create_database, "flushes": flushes, "visibility": visibility, "read_bench": read_bench_result}) tr["status"] = "planned" if args.dry_run else ("measured" if remote_checks_ok and query_result["status"] == "ok" else "failed") write_json(target.report_path, tr) report["targets"].append(tr) target_results.append(query_result) if not args.dry_run: - report["thresholds"] = enforce_thresholds(case, target_results[0], target_results[1]) + report["thresholds"] = enforce_thresholds(case, target_results[0], target_results[1]) + enforce_storage_thresholds(storage, storage_results[0] if storage_results else None, storage_results[1] if len(storage_results) > 1 else None) + elif storage: + report["thresholds"] = planned_storage_thresholds(storage) report["status"] = "planned" if args.dry_run else ("failed" if any(t["status"] == "failed" for t in report["thresholds"]) or any(t.get("status") == "failed" for t in report["targets"]) else "ok") finally: for cluster in reversed(clusters): @@ -1015,7 +1295,9 @@ def run_remote_write_scenario(args: argparse.Namespace, case: dict[str, Any], ca def main() -> int: args = parse_args() case_path = args.case.resolve() - case = load_case(case_path) + if args.fixture_generator is not None: + require_binary(args.fixture_generator, "fixture generator", dry_run=False) + case = load_normalized_case(case_path, args.fixture_generator) scenario_config = scenario(case) scenario_kind = scenario_config.get("kind") tables = case_tables(case) @@ -1025,9 +1307,6 @@ def main() -> int: require_binary(args.candidate_bin, "candidate", dry_run=args.dry_run or args.fixture_only) if scenario_kind == "direct_readable_sst" and not args.dry_run and args.fixture_generator is None: raise ValueError("--fixture-generator is required unless --dry-run is set") - if args.fixture_generator is not None: - require_binary(args.fixture_generator, "fixture generator", dry_run=args.dry_run) - ports = allocate_ports(16) targets = [make_target("base", args.base_bin.resolve(), work_root, ports[:8]), make_target("candidate", args.candidate_bin.resolve(), work_root, ports[8:])] require_fresh_work_dirs(targets, reuse_work_dir=args.reuse_work_dir, dry_run=args.dry_run, fixture_only=args.fixture_only) diff --git a/tests/perf/test_query_regression_runner_compaction_toctou.py b/tests/perf/test_query_regression_runner_compaction_toctou.py new file mode 100644 index 00000000000..675f451a10e --- /dev/null +++ b/tests/perf/test_query_regression_runner_compaction_toctou.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression coverage for quiescing remote-write storage before read benches.""" + +import argparse +import importlib.util +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + + +RUNNER_PATH = Path(__file__).with_name("query_regression_runner.py") +SPEC = importlib.util.spec_from_file_location("query_regression_runner_under_test", RUNNER_PATH) +assert SPEC is not None and SPEC.loader is not None +runner = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = runner +SPEC.loader.exec_module(runner) + + +class RemoteWriteCompactionToctouTest(unittest.TestCase): + def test_quiesces_each_target_before_storage_inspection_and_read_bench(self) -> None: + lifecycle: dict[str, list[str]] = {"base": [], "candidate": []} + clusters = [] + + class RecordingCluster: + def __init__(self, target): + self.target = target + self.active = set() + self.stop_all_calls = 0 + clusters.append(self) + + def start_all(self, _config_path): + self.active.update(("metasrv", "datanode", "frontend")) + + def stop_component(self, name): + if name in self.active: + lifecycle[self.target.name].append(f'cluster.stop_component("{name}")') + self.active.remove(name) + + def stop_all(self): + self.stop_all_calls += 1 + for name in ("frontend", "datanode", "metasrv"): + self.stop_component(name) + + def component_report(self): + return {} + + def run_queries(target, _case, _tables, _http_timeout): + lifecycle[target.name].append("query measurement completes") + return {"validation": [], "validation_errors": [], "measurements": [], "status": "ok"} + + def inspect_storage(_helper, target, _storage, *, dry_run): + self.assertFalse(dry_run) + lifecycle[target.name].append("storage inspection") + return {"status": "ok", "summary": {"summary": {}}} + + def read_bench(_binary, target, _read_bench, _inspection, *, dry_run): + self.assertFalse(dry_run) + lifecycle[target.name].append("read bench") + return {"status": "ok"} + + remote_write = { + "database": "public", + "metric": "cpu_usage", + "physical_table": "cpu_usage_physical", + "series_count": 1, + "samples_per_series": 1, + "visibility_timeout_seconds": 1, + "storage": { + "inspect": True, + "min_files": None, + "min_files_with_column": None, + "max_total_file_size_bytes": None, + "max_column_compressed_size_bytes": None, + "max_column_uncompressed_size_bytes": None, + "require_encodings": [], + "forbid_encodings": [], + }, + "read_bench": {"enabled": True}, + "prom_store": { + "pending_rows_flush_interval": "1s", + "max_batch_rows": 1, + "max_concurrent_flushes": 1, + "worker_channel_capacity": 1, + "max_inflight_requests": 1, + }, + } + case = { + "scenario": { + "kind": "prom_remote_write_then_query", + "remote_write": remote_write, + "queries": [], + } + } + args = argparse.Namespace( + fixture_only=False, + fixture_generator=Path("/bin/true"), + remote_write_generator=None, + storage_inspector=None, + candidate_bin=Path("/bin/true"), + dry_run=False, + http_timeout=1.0, + ) + + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + targets = [ + runner.make_target("base", Path("/bin/true"), root, list(range(10000, 10008))), + runner.make_target("candidate", Path("/bin/true"), root, list(range(10008, 10016))), + ] + report = {"targets": [], "thresholds": []} + with ( + patch.object(runner, "DistributedCluster", RecordingCluster), + patch.object(runner, "run_remote_write_ingestion", return_value=({"status": "ok"}, [{"ok": True}])), + patch.object(runner, "http_post_sql", return_value={"ok": True}), + patch.object(runner, "poll_expected_count", return_value={"ok": True, "row_count_ok": True}), + patch.object(runner, "run_queries", side_effect=run_queries), + patch.object(runner, "run_storage_inspection", side_effect=inspect_storage), + patch.object(runner, "run_read_bench", side_effect=read_bench), + ): + runner.run_remote_write_scenario(args, case, root / "case.toml", targets, report) + + for cluster in clusters: + self.assertEqual(cluster.stop_all_calls, 1) + self.assertEqual(cluster.active, set()) + cluster.stop_all() + self.assertEqual(cluster.stop_all_calls, 2) + self.assertEqual(cluster.active, set()) + + expected = [ + "query measurement completes", + 'cluster.stop_component("datanode")', + "storage inspection", + "read bench", + ] + relevant = {name: [event for event in events if event in expected] for name, events in lifecycle.items()} + for target_name in ("base", "candidate"): + with self.subTest(target=target_name): + self.assertEqual(relevant[target_name], expected) + + +if __name__ == "__main__": + unittest.main() From c12f40cec232dda23429a0995d70bb4a230a562c Mon Sep 17 00:00:00 2001 From: dennis zhuang Date: Sat, 11 Jul 2026 21:13:36 +0800 Subject: [PATCH 2/8] chore: update star history link (#8483) Signed-off-by: Dennis Zhuang --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3ba52bc5266..99c78814609 100644 --- a/README.md +++ b/README.md @@ -209,7 +209,13 @@ Read the [v1.0 highlights](https://greptime.com/blogs/2025-11-05-greptimedb-v1-h If GreptimeDB is useful to you, please star the repo. -[![Star History Chart](https://api.star-history.com/svg?repos=GreptimeTeam/GreptimeDB&type=Date)](https://www.star-history.com/#GreptimeTeam/GreptimeDB&Date) + + + + + Star History Chart + + Known Users From b411c6265c19debefddac5a38f9365cd59122398 Mon Sep 17 00:00:00 2001 From: discord9 Date: Mon, 13 Jul 2026 10:46:33 +0800 Subject: [PATCH 3/8] test: add json2 variant payload compat case (#8459) * test: add json2 variant payload compat case Signed-off-by: discord9 * test: refine json2 variant compat case Signed-off-by: discord9 --------- Signed-off-by: discord9 --- .../legacy_json2_variant_payload/case.toml | 8 +++ .../legacy_json2_variant_payload/setup.sql | 29 +++++++++ .../verify.result | 60 +++++++++++++++++++ .../legacy_json2_variant_payload/verify.sql | 16 +++++ 4 files changed, 113 insertions(+) create mode 100644 tests/compatibility/cases/legacy_json2_variant_payload/case.toml create mode 100644 tests/compatibility/cases/legacy_json2_variant_payload/setup.sql create mode 100644 tests/compatibility/cases/legacy_json2_variant_payload/verify.result create mode 100644 tests/compatibility/cases/legacy_json2_variant_payload/verify.sql diff --git a/tests/compatibility/cases/legacy_json2_variant_payload/case.toml b/tests/compatibility/cases/legacy_json2_variant_payload/case.toml new file mode 100644 index 00000000000..dd46e28ba4d --- /dev/null +++ b/tests/compatibility/cases/legacy_json2_variant_payload/case.toml @@ -0,0 +1,8 @@ +name = "legacy_json2_variant_payload" +reason = "Verify current binaries can read JSON2 variant payloads written by old binaries after the JSON2 encoding change." +introduced_by = "PR #8435" +topologies = ["distributed"] +from_range = ["=v1.1.0", "=v1.1.1", "=v1.1.2"] +to_range = [">v1.1.1"] +features = ["json", "json2", "table", "read"] +owner = "query" diff --git a/tests/compatibility/cases/legacy_json2_variant_payload/setup.sql b/tests/compatibility/cases/legacy_json2_variant_payload/setup.sql new file mode 100644 index 00000000000..cef3a913c93 --- /dev/null +++ b/tests/compatibility/cases/legacy_json2_variant_payload/setup.sql @@ -0,0 +1,29 @@ +CREATE TABLE t_legacy_json2_variant_payload ( + ts TIMESTAMP TIME INDEX, + j JSON2 +) WITH ( + 'append_mode' = 'true' +); + +INSERT INTO t_legacy_json2_variant_payload (ts, j) VALUES + ('2026-07-08 00:00:00+0000', '{"a": {"b": 1}, "c": "s1", "d": [{"e": {"f": 0.1}}]}'), + ('2026-07-08 00:01:00+0000', '{"a": {"b": -2}, "c": [1], "d": [{"e": {"f": 0.2}}]}'), + ('2026-07-08 00:02:00+0000', '{"a": {"b": "s3"}, "c": "s3", "d": [{"e": {"g": -0.3}}]}'), + ('2026-07-08 00:03:00+0000', '{"a": {"b": null}, "c": null}'), + ('2026-07-08 00:04:00+0000', '{"c": "s5", "d": {"e": true}}'), + ('2026-07-08 00:05:00+0000', '{"a": {"b": 6}}'); + +ADMIN FLUSH_TABLE('t_legacy_json2_variant_payload'); + +CREATE TABLE t_legacy_json2_variant_payload_object_d ( + ts TIMESTAMP TIME INDEX, + j JSON2 +) WITH ( + 'append_mode' = 'true' +); + +INSERT INTO t_legacy_json2_variant_payload_object_d (ts, j) VALUES + ('2026-07-08 01:00:00+0000', '{"d": {"e": true}}'), + ('2026-07-08 01:01:00+0000', '{"d": {"e": false}}'); + +ADMIN FLUSH_TABLE('t_legacy_json2_variant_payload_object_d'); diff --git a/tests/compatibility/cases/legacy_json2_variant_payload/verify.result b/tests/compatibility/cases/legacy_json2_variant_payload/verify.result new file mode 100644 index 00000000000..a474cc84345 --- /dev/null +++ b/tests/compatibility/cases/legacy_json2_variant_payload/verify.result @@ -0,0 +1,60 @@ +SELECT ts, j.a.b AS a_b, j.c AS c +FROM t_legacy_json2_variant_payload +ORDER BY ts; + ++---------------------+-----+-----+ +| ts | a_b | c | ++---------------------+-----+-----+ +| 2026-07-08T00:00:00 | 1 | s1 | +| 2026-07-08T00:01:00 | -2 | [1] | +| 2026-07-08T00:02:00 | s3 | s3 | +| 2026-07-08T00:03:00 | | | +| 2026-07-08T00:04:00 | | s5 | +| 2026-07-08T00:05:00 | 6 | | ++---------------------+-----+-----+ + +SELECT + ts, + j.d.e AS d_e +FROM t_legacy_json2_variant_payload_object_d +ORDER BY ts; + ++---------------------+-------+ +| ts | d_e | ++---------------------+-------+ +| 2026-07-08T01:00:00 | true | +| 2026-07-08T01:01:00 | false | ++---------------------+-------+ + +-- JSON2 projection rendering is a user-visible compatibility contract. +SELECT ts, j.d AS d +FROM t_legacy_json2_variant_payload +ORDER BY ts; + ++---------------------+--------------------+ +| ts | d | ++---------------------+--------------------+ +| 2026-07-08T00:00:00 | [{"e":{"f":0.1}}] | +| 2026-07-08T00:01:00 | [{"e":{"f":0.2}}] | +| 2026-07-08T00:02:00 | [{"e":{"g":-0.3}}] | +| 2026-07-08T00:03:00 | | +| 2026-07-08T00:04:00 | {"e":true} | +| 2026-07-08T00:05:00 | | ++---------------------+--------------------+ + +SHOW CREATE TABLE t_legacy_json2_variant_payload; + ++--------------------------------+---------------------------------------------------------------+ +| Table | Create Table | ++--------------------------------+---------------------------------------------------------------+ +| t_legacy_json2_variant_payload | CREATE TABLE IF NOT EXISTS "t_legacy_json2_variant_payload" ( | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "j" JSON2 NULL, | +| | TIME INDEX ("ts") | +| | ) | +| | | +| | ENGINE=mito | +| | WITH( | +| | append_mode = 'true' | +| | ) | ++--------------------------------+---------------------------------------------------------------+ diff --git a/tests/compatibility/cases/legacy_json2_variant_payload/verify.sql b/tests/compatibility/cases/legacy_json2_variant_payload/verify.sql new file mode 100644 index 00000000000..a3c2cab1b92 --- /dev/null +++ b/tests/compatibility/cases/legacy_json2_variant_payload/verify.sql @@ -0,0 +1,16 @@ +SELECT ts, j.a.b AS a_b, j.c AS c +FROM t_legacy_json2_variant_payload +ORDER BY ts; + +SELECT + ts, + j.d.e AS d_e +FROM t_legacy_json2_variant_payload_object_d +ORDER BY ts; + +-- JSON2 projection rendering is a user-visible compatibility contract. +SELECT ts, j.d AS d +FROM t_legacy_json2_variant_payload +ORDER BY ts; + +SHOW CREATE TABLE t_legacy_json2_variant_payload; From d667728fdeb5a55e63665f070453b4649801af60 Mon Sep 17 00:00:00 2001 From: shuiyisong <113876041+shuiyisong@users.noreply.github.com> Date: Mon, 13 Jul 2026 11:27:06 +0800 Subject: [PATCH 4/8] chore!: update promql-parser to v0.10.0, remove `holt_winters` (#8457) * chore: update promql parser and fix compile Signed-off-by: shuiyisong * fix: sqlness Signed-off-by: shuiyisong * fix: cr issue Signed-off-by: shuiyisong * fix: cr issue Signed-off-by: shuiyisong --------- Signed-off-by: shuiyisong --- Cargo.lock | 8 +-- Cargo.toml | 2 +- src/query/src/parser.rs | 49 ++++++++++++------- src/query/src/promql/planner.rs | 41 ++++++++++++++++ .../standalone/common/promql/functions.result | 10 ---- .../standalone/common/promql/functions.sql | 4 -- 6 files changed, 76 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e00e30e28a6..75ba5ce410b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11038,9 +11038,9 @@ dependencies = [ [[package]] name = "promql-parser" -version = "0.7.3" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f514ab521bb2b4ce26aca29bf133498b2a2651b785d03a262bbf5772329c279f" +checksum = "9e8c90c956be4237a93be6f2298302b56974ef862ab11e0f98869d1f27c69f88" dependencies = [ "cfgrammar", "chrono", @@ -11183,7 +11183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -11196,7 +11196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", diff --git a/Cargo.toml b/Cargo.toml index 5ee60c450e8..fd878c7c0a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -198,7 +198,7 @@ paste = "1.0" pin-project = "1.0" pretty_assertions = "1.4.0" prometheus = { version = "0.14", features = ["process"] } -promql-parser = { version = "0.7.3", features = ["ser"] } +promql-parser = { version = "0.10.0", features = ["ser"] } prost = { version = "0.14", features = ["no-recursion-limit"] } prost-types = "0.14" raft-engine = { version = "0.4.1", default-features = false } diff --git a/src/query/src/parser.rs b/src/query/src/parser.rs index 53f24557c73..87b827e0cc1 100644 --- a/src/query/src/parser.rs +++ b/src/query/src/parser.rs @@ -52,45 +52,37 @@ pub enum QueryStatement { } impl QueryStatement { - pub fn post_process(&self, params: HashMap) -> Result { + pub fn post_process(self, params: HashMap) -> Result { match self { QueryStatement::Sql(_) => UnimplementedSnafu { operation: "sql post process", } .fail(), - QueryStatement::Promql(eval_stmt, alias) => { + QueryStatement::Promql(mut eval_stmt, alias) => { let node_name = match params.get("name") { Some(name) => name.as_str(), None => "", }; - let extension_node = Self::create_extension_node(node_name, &eval_stmt.expr); - Ok(QueryStatement::Promql( - EvalStmt { - expr: Extension(extension_node.unwrap()), - start: eval_stmt.start, - end: eval_stmt.end, - interval: eval_stmt.interval, - lookback_delta: eval_stmt.lookback_delta, - }, - alias.clone(), - )) + let extension_node = Self::create_extension_node(node_name, eval_stmt.expr); + eval_stmt.expr = Extension(extension_node.unwrap()); + Ok(QueryStatement::Promql(eval_stmt, alias)) } } } - fn create_extension_node(node_name: &str, expr: &Expr) -> Option { + fn create_extension_node(node_name: &str, expr: Expr) -> Option { match node_name { ANALYZE_NODE_NAME => Some(NodeExtension { - expr: Arc::new(AnalyzeExpr { expr: expr.clone() }), + expr: Arc::new(AnalyzeExpr { expr }), }), ANALYZE_VERBOSE_NODE_NAME => Some(NodeExtension { - expr: Arc::new(AnalyzeVerboseExpr { expr: expr.clone() }), + expr: Arc::new(AnalyzeVerboseExpr { expr }), }), EXPLAIN_NODE_NAME => Some(NodeExtension { - expr: Arc::new(ExplainExpr { expr: expr.clone() }), + expr: Arc::new(ExplainExpr { expr }), }), EXPLAIN_VERBOSE_NODE_NAME => Some(NodeExtension { - expr: Arc::new(ExplainVerboseExpr { expr: expr.clone() }), + expr: Arc::new(ExplainVerboseExpr { expr }), }), _ => None, } @@ -204,9 +196,10 @@ impl QueryLanguageParser { } pub(crate) fn apply_alias_extension(mut eval_stmt: EvalStmt, alias: &str) -> EvalStmt { + let expr = eval_stmt.expr; eval_stmt.expr = Extension(NodeExtension { expr: Arc::new(AliasExpr { - expr: eval_stmt.expr.clone(), + expr, alias: alias.to_string(), }), }); @@ -263,6 +256,14 @@ macro_rules! define_node_ast_extension { fn children(&self) -> &[Expr] { std::slice::from_ref(&self.expr) } + + fn with_new_children(&self, children: Vec) -> Arc { + let mut iter = children.into_iter(); + match (iter.next(), iter.next()) { + (Some(expr), None) => Arc::new($name_expr { expr }), + _ => Arc::new(self.clone()), + } + } } #[allow(rustdoc::broken_intra_doc_links)] @@ -313,6 +314,16 @@ impl ExtensionExpr for AliasExpr { fn children(&self) -> &[Expr] { std::slice::from_ref(&self.expr) } + fn with_new_children(&self, children: Vec) -> Arc { + let mut iter = children.into_iter(); + match (iter.next(), iter.next()) { + (Some(expr), None) => Arc::new(Self { + expr, + alias: self.alias.clone(), + }), + _ => Arc::new(self.clone()), + } + } } #[derive(Debug, Clone)] pub struct Alias { diff --git a/src/query/src/promql/planner.rs b/src/query/src/promql/planner.rs index c2cd6316ada..61913156cf8 100644 --- a/src/query/src/promql/planner.rs +++ b/src/query/src/promql/planner.rs @@ -229,6 +229,8 @@ impl IslandExpr { !modifier.return_bool && modifier.matching.is_none() && matches!(modifier.card, VectorMatchCardinality::OneToOne) + && modifier.fill_values.lhs.is_none() + && modifier.fill_values.rhs.is_none() }) => { let lhs = Self::try_new(lhs, env)?; @@ -1093,6 +1095,18 @@ impl PromPlanner { query_engine_state: &QueryEngineState, binary_expr: &PromBinaryExpr, ) -> Result { + // promql-parser accepts fill modifiers, but Greptime does not implement the + // required outer joins and missing-value substitution. Reject them before the + // binary-island fast path so they cannot silently behave like normal inner joins. + if let Some(modifier) = &binary_expr.modifier { + ensure!( + modifier.fill_values.lhs.is_none() && modifier.fill_values.rhs.is_none(), + UnsupportedExprSnafu { + name: "PromQL fill modifiers" + } + ); + } + if let Some(plan) = self.try_plan_binary_island(binary_expr).await? { return Ok(plan); } @@ -5542,6 +5556,33 @@ mod test { ); } + #[tokio::test] + async fn reject_binary_fill_modifiers() { + let state = build_query_engine_state(); + + for query in [ + "some_metric + fill(0) some_alt_metric", + "some_metric + fill_left(0) some_alt_metric", + "some_metric + fill_right(0) some_alt_metric", + "(some_metric + fill(0) some_alt_metric) + some_metric", + ] { + let eval_stmt = build_eval_stmt(query); + let table_provider = build_test_table_provider(&[], 0, 0).await; + let err = PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &state) + .await + .unwrap_err(); + + assert!( + matches!( + &err, + crate::promql::error::Error::UnsupportedExpr { name, .. } + if name == "PromQL fill modifiers" + ), + "{err}" + ); + } + } + #[tokio::test] async fn timestamp_binary_join_falls_back_when_tsid_is_projected_out() { for query in [ diff --git a/tests/cases/standalone/common/promql/functions.result b/tests/cases/standalone/common/promql/functions.result index 02ff599ced9..325fbccd17d 100644 --- a/tests/cases/standalone/common/promql/functions.result +++ b/tests/cases/standalone/common/promql/functions.result @@ -62,16 +62,6 @@ tql eval (10, 10, '1s') double_exponential_smoothing(prom_series[10s], 0.4 + 0.1 | 1970-01-01T00:00:10 | 47.0806953125 | p | +---------------------+------------------------------------------------------------------------------------------+------+ --- holt_winters (backward compatibility) --- SQLNESS SORT_RESULT 3 1 -tql eval (10, 10, '1s') holt_winters(prom_series[10s], 0.4 + 0.1, 0.1); - -+---------------------+------------------------------------------------------------------------------------------+------+ -| ts | prom_double_exponential_smoothing(ts_range,val,Float64(0.4) + Float64(0.1),Float64(0.1)) | host | -+---------------------+------------------------------------------------------------------------------------------+------+ -| 1970-01-01T00:00:10 | 47.0806953125 | p | -+---------------------+------------------------------------------------------------------------------------------+------+ - DROP TABLE prom_series; Affected Rows: 0 diff --git a/tests/cases/standalone/common/promql/functions.sql b/tests/cases/standalone/common/promql/functions.sql index 6657fc49e81..0a257f29da8 100644 --- a/tests/cases/standalone/common/promql/functions.sql +++ b/tests/cases/standalone/common/promql/functions.sql @@ -34,10 +34,6 @@ tql eval (3, 3, '1s') predict_linear(prom_series[3s], 40 + 2); -- SQLNESS SORT_RESULT 3 1 tql eval (10, 10, '1s') double_exponential_smoothing(prom_series[10s], 0.4 + 0.1, 0.1); --- holt_winters (backward compatibility) --- SQLNESS SORT_RESULT 3 1 -tql eval (10, 10, '1s') holt_winters(prom_series[10s], 0.4 + 0.1, 0.1); - DROP TABLE prom_series; CREATE TABLE From 29b345fe2d07a04937bf5e64d37ea6c69ab4dbc0 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Mon, 13 Jul 2026 11:56:34 +0800 Subject: [PATCH 5/8] ci: increase Kafka resource requests (#8482) * ci: increase Kafka resource requests Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus Signed-off-by: WenyXu * ci: enable rskafka debug logs in remote WAL fuzz Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent) Co-authored-by: Sisyphus Signed-off-by: WenyXu --------- Signed-off-by: WenyXu Co-authored-by: Sisyphus --- .../with-remote-wal-logical-prune.yaml | 1 + .github/actions/setup-greptimedb-cluster/with-remote-wal.yaml | 1 + .github/actions/setup-kafka-cluster/action.yml | 4 ++-- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/actions/setup-greptimedb-cluster/with-remote-wal-logical-prune.yaml b/.github/actions/setup-greptimedb-cluster/with-remote-wal-logical-prune.yaml index 60c23def6e2..3b25e80a8f2 100644 --- a/.github/actions/setup-greptimedb-cluster/with-remote-wal-logical-prune.yaml +++ b/.github/actions/setup-greptimedb-cluster/with-remote-wal-logical-prune.yaml @@ -3,6 +3,7 @@ logging: format: "json" filters: - log_store=debug + - rskafka=debug meta: configData: |- [runtime] diff --git a/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml b/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml index ec595b9e9b8..6a15b25b422 100644 --- a/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml +++ b/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml @@ -3,6 +3,7 @@ logging: format: "json" filters: - log_store=debug + - rskafka=debug meta: configData: |- [runtime] diff --git a/.github/actions/setup-kafka-cluster/action.yml b/.github/actions/setup-kafka-cluster/action.yml index fca420da709..3c30ca285e3 100644 --- a/.github/actions/setup-kafka-cluster/action.yml +++ b/.github/actions/setup-kafka-cluster/action.yml @@ -16,8 +16,8 @@ runs: helm upgrade \ --install kafka oci://registry-1.docker.io/bitnamicharts/kafka \ --set controller.replicaCount=${{ inputs.controller-replicas }} \ - --set controller.resources.requests.cpu=50m \ - --set controller.resources.requests.memory=128Mi \ + --set controller.resources.requests.cpu=500m \ + --set controller.resources.requests.memory=512Mi \ --set controller.resources.limits.cpu=2000m \ --set controller.resources.limits.memory=2Gi \ --set listeners.controller.protocol=PLAINTEXT \ From c70cbaa132d45f69f0ad5548e98e5eb208fa5ac0 Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Mon, 13 Jul 2026 13:21:21 +0800 Subject: [PATCH 6/8] feat: more region lifecycle hooks (#8467) * feat: hook extension for region/file close/drop * feat: cover region lifecycle with open/close/drop hooks * docs(mito2): clarify on_region_opened runs in spawned task on open path Address review feedback: the module/trait docs claimed on_region_opened runs inline in the region worker loop. That holds for the create path but not for the open path, where it fires inside the spawned open task (common_runtime::spawn_global) after WAL replay and before registration. Correct both the Notes block and the trait method doc so hook authors don't assume worker-loop-thread affinity or strict ordering on open. Signed-off-by: Ning Sun * docs(mito2): fix region_hook inventory and lifecycle wording Address shuiyisong's review feedback: - Overview no longer hardcodes 'two methods'; the lifecycle bullet list now includes on_region_opened. - on_region_opened is described as firing after open/create succeeds but before registration (it runs before insert_region on both paths). - on_region_closed drops the inaccurate 'follower/catchup regions' exclusion: remove_region fires it for any role, which is consistent with on_region_opened firing for followers too. Signed-off-by: Ning Sun --------- Signed-off-by: Ning Sun --- src/mito2/src/engine/close_test.rs | 43 +++++++++ src/mito2/src/engine/drop_test.rs | 76 ++++++++++++++++ src/mito2/src/engine/flush_test.rs | 59 +++++++++++- src/mito2/src/engine/open_test.rs | 35 +++++++ src/mito2/src/engine/region_hook.rs | 126 +++++++++++++++++++++++++- src/mito2/src/worker/handle_close.rs | 8 ++ src/mito2/src/worker/handle_create.rs | 6 ++ src/mito2/src/worker/handle_drop.rs | 47 ++++++++++ src/mito2/src/worker/handle_open.rs | 7 ++ 9 files changed, 399 insertions(+), 8 deletions(-) diff --git a/src/mito2/src/engine/close_test.rs b/src/mito2/src/engine/close_test.rs index 4c06583b0b8..0092634ec74 100644 --- a/src/mito2/src/engine/close_test.rs +++ b/src/mito2/src/engine/close_test.rs @@ -12,11 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use common_base::Plugins; use store_api::region_engine::RegionEngine; use store_api::region_request::{RegionCloseRequest, RegionRequest}; use store_api::storage::RegionId; use crate::config::MitoConfig; +use crate::engine::flush_test::MockRegionHook; +use crate::engine::region_hook::RegionHookRef; use crate::test_util::{CreateRequestBuilder, TestEnv}; #[tokio::test] @@ -60,3 +66,40 @@ async fn test_engine_close_region_with_format(flat_format: bool) { .await .unwrap(); } + +#[tokio::test] +async fn test_region_hook_on_close() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::new().await; + + let hook = Arc::new(MockRegionHook::new()); + let plugins = Plugins::new(); + plugins.insert(hook.clone() as RegionHookRef); + + let engine = env + .create_engine_with_plugins(MitoConfig::default(), plugins) + .await; + + let region_id = RegionId::new(1, 1); + let request = CreateRequestBuilder::new().build(); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + + // Sanity: no lifecycle events before closing besides the create open. + assert_eq!(hook.opened_count.load(Ordering::Relaxed), 1); + assert_eq!(hook.closed_count.load(Ordering::Relaxed), 0); + assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 0); + + engine + .handle_request(region_id, RegionRequest::Close(RegionCloseRequest {})) + .await + .unwrap(); + + // Closing fires on_region_closed exactly once. + assert_eq!(hook.closed_count.load(Ordering::Relaxed), 1); + // Close must not be confused with drop. + assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 0); + assert_eq!(hook.files_removed_count.load(Ordering::Relaxed), 0); +} diff --git a/src/mito2/src/engine/drop_test.rs b/src/mito2/src/engine/drop_test.rs index a34a5d11727..90afd66ba32 100644 --- a/src/mito2/src/engine/drop_test.rs +++ b/src/mito2/src/engine/drop_test.rs @@ -13,9 +13,11 @@ // limitations under the License. use std::sync::Arc; +use std::sync::atomic::Ordering; use std::time::Duration; use api::v1::Rows; +use common_base::Plugins; use common_meta::key::SchemaMetadataManager; use common_meta::kv_backend::KvBackendRef; use object_store::util::join_path; @@ -25,7 +27,9 @@ use store_api::storage::RegionId; use crate::config::MitoConfig; use crate::engine::MitoEngine; +use crate::engine::flush_test::MockRegionHook; use crate::engine::listener::DropListener; +use crate::engine::region_hook::RegionHookRef; use crate::test_util::{ CreateRequestBuilder, TestEnv, build_rows_for_key, flush_region, put_rows, rows_schema, }; @@ -302,3 +306,75 @@ async fn test_engine_drop_region_for_custom_store_with_format(flat_format: bool) .unwrap() ); } + +#[tokio::test] +async fn test_region_hook_on_drop() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::with_prefix("drop_hook").await; + + let hook = Arc::new(MockRegionHook::new()); + let plugins = Plugins::new(); + plugins.insert(hook.clone() as RegionHookRef); + + let engine = env + .create_engine_with_plugins(MitoConfig::default(), plugins) + .await; + + let region_id = RegionId::new(1, 1); + env.get_schema_metadata_manager() + .register_region_table_info( + region_id.table_id(), + "test_table", + "test_catalog", + "test_schema", + None, + env.get_kv_backend(), + ) + .await; + + let request = CreateRequestBuilder::new().build(); + let column_schemas = rows_schema(&request); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + + // Put some rows so the region has active data (kept in memtable; we + // intentionally do not flush so the drop GC worker finds no parquet files + // and removes the directory immediately, avoiding the multi-minute GC wait). + let rows = Rows { + schema: column_schemas, + rows: build_rows_for_key("a", 0, 2, 0), + }; + put_rows(&engine, region_id, rows).await; + + // Sanity: nothing fired yet besides the open from create. + assert_eq!(hook.opened_count.load(Ordering::Relaxed), 1); + assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 0); + assert_eq!(hook.closed_count.load(Ordering::Relaxed), 0); + + // full drop (partial_drop = false) so the GC worker physically deletes the + // directory and fires on_region_files_removed. + engine + .handle_request( + region_id, + RegionRequest::Drop(RegionDropRequest { + fast_path: false, + force: false, + partial_drop: false, + }), + ) + .await + .unwrap(); + + // on_region_dropped fires inline while handling the drop request. + hook.wait_for_dropped().await; + assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 1); + // Drop must not be confused with close. + assert_eq!(hook.closed_count.load(Ordering::Relaxed), 0); + + // on_region_files_removed fires from the background GC worker once the + // directory is physically deleted. + hook.wait_for_files_removed().await; + assert_eq!(hook.files_removed_count.load(Ordering::Relaxed), 1); +} diff --git a/src/mito2/src/engine/flush_test.rs b/src/mito2/src/engine/flush_test.rs index 4870e8c2bbe..dc0063a7e63 100644 --- a/src/mito2/src/engine/flush_test.rs +++ b/src/mito2/src/engine/flush_test.rs @@ -895,24 +895,46 @@ async fn test_update_topic_latest_entry_id(factory: Option) { } #[derive(Debug)] -struct MockRegionHook { - sst_written_count: AtomicUsize, - manifest_updated_count: AtomicUsize, +pub(super) struct MockRegionHook { + pub(super) sst_written_count: AtomicUsize, + pub(super) manifest_updated_count: AtomicUsize, + pub(super) opened_count: AtomicUsize, + pub(super) closed_count: AtomicUsize, + pub(super) dropped_count: AtomicUsize, + pub(super) files_removed_count: AtomicUsize, notify: Notify, + dropped_notify: Notify, + files_removed_notify: Notify, } impl MockRegionHook { - fn new() -> Self { + pub(super) fn new() -> Self { Self { sst_written_count: AtomicUsize::new(0), manifest_updated_count: AtomicUsize::new(0), + opened_count: AtomicUsize::new(0), + closed_count: AtomicUsize::new(0), + dropped_count: AtomicUsize::new(0), + files_removed_count: AtomicUsize::new(0), notify: Notify::new(), + dropped_notify: Notify::new(), + files_removed_notify: Notify::new(), } } async fn wait_for_manifest_update(&self) { self.notify.notified().await; } + + /// Waits until `on_region_dropped` has fired at least once. + pub(super) async fn wait_for_dropped(&self) { + self.dropped_notify.notified().await; + } + + /// Waits until `on_region_files_removed` has fired at least once. + pub(super) async fn wait_for_files_removed(&self) { + self.files_removed_notify.notified().await; + } } #[async_trait] @@ -966,6 +988,35 @@ impl RegionHook for MockRegionHook { ); self.notify.notify_one(); } + + async fn on_region_opened(&self, region_id: RegionId, _region_metadata: &RegionMetadataRef) { + self.opened_count.fetch_add(1, Ordering::Relaxed); + common_telemetry::info!("MockRegionHook::on_region_opened: region={}", region_id); + } + + async fn on_region_closed(&self, region_id: RegionId, _region_metadata: &RegionMetadataRef) { + self.closed_count.fetch_add(1, Ordering::Relaxed); + common_telemetry::info!("MockRegionHook::on_region_closed: region={}", region_id); + } + + async fn on_region_dropped(&self, region_id: RegionId, _region_metadata: &RegionMetadataRef) { + self.dropped_count.fetch_add(1, Ordering::Relaxed); + common_telemetry::info!("MockRegionHook::on_region_dropped: region={}", region_id); + self.dropped_notify.notify_one(); + } + + async fn on_region_files_removed( + &self, + region_id: RegionId, + _region_metadata: &RegionMetadataRef, + ) { + self.files_removed_count.fetch_add(1, Ordering::Relaxed); + common_telemetry::info!( + "MockRegionHook::on_region_files_removed: region={}", + region_id + ); + self.files_removed_notify.notify_one(); + } } #[tokio::test] diff --git a/src/mito2/src/engine/open_test.rs b/src/mito2/src/engine/open_test.rs index cf4bfb1ff1d..ee0d9560093 100644 --- a/src/mito2/src/engine/open_test.rs +++ b/src/mito2/src/engine/open_test.rs @@ -32,6 +32,8 @@ use tokio::sync::oneshot; use crate::compaction::compactor::{OpenCompactionRegionRequest, open_compaction_region}; use crate::config::MitoConfig; +use crate::engine::flush_test::MockRegionHook; +use crate::engine::region_hook::RegionHookRef; use crate::error; use crate::region::opener::{PartitionExprFetcher, PartitionExprFetcherRef}; use crate::region::options::RegionOptions; @@ -805,3 +807,36 @@ async fn test_open_keeps_none_without_fetcher() { let meta = engine.get_region(region_id).unwrap().metadata(); assert!(meta.partition_expr.is_none()); } + +#[tokio::test] +async fn test_region_hook_on_open() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::with_prefix("open_hook").await; + + let hook = Arc::new(MockRegionHook::new()); + let plugins = Plugins::new(); + plugins.insert(hook.clone() as RegionHookRef); + + let engine = env + .create_engine_with_plugins(MitoConfig::default(), plugins) + .await; + + let region_id = RegionId::new(1, 1); + let request = CreateRequestBuilder::new().build(); + let table_dir = request.table_dir.clone(); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + + // Creating a region fires on_region_opened exactly once. + assert_eq!(hook.opened_count.load(Ordering::Relaxed), 1); + assert_eq!(hook.closed_count.load(Ordering::Relaxed), 0); + + // `reopen_region` closes then opens; the open must fire the hook again. + reopen_region(&engine, region_id, table_dir, false, Default::default()).await; + + assert_eq!(hook.opened_count.load(Ordering::Relaxed), 2); + assert_eq!(hook.closed_count.load(Ordering::Relaxed), 1); + assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 0); +} diff --git a/src/mito2/src/engine/region_hook.rs b/src/mito2/src/engine/region_hook.rs index d1b28f6ebae..c4b683ca831 100644 --- a/src/mito2/src/engine/region_hook.rs +++ b/src/mito2/src/engine/region_hook.rs @@ -16,7 +16,8 @@ //! //! ## Design //! -//! The [`RegionHook`] trait provides two methods with clear separation of concerns: +//! The [`RegionHook`] trait observes region activity through two categories of +//! callbacks — manifest/file observation and region lifecycle: //! //! - [`on_sst_files_written`]: Fires when mito2 physically writes SST **data files**. //! Provides per-file [`SstInfo`] + [`FileMeta`]; metadata richness varies by path @@ -26,6 +27,10 @@ //! Receives the full [`RegionMetaActionList`] so consumers can inspect what changed //! (file additions/removals, schema changes, truncation, partition expression changes, etc.). //! +//! - [`on_region_opened`] / [`on_region_closed`] / [`on_region_dropped`] / [`on_region_files_removed`]: +//! Region **lifecycle** callbacks for open, close, logical drop, and physical file removal. +//! See [Region lifecycle](#region-lifecycle) below. +//! //! Hook implementations are registered via the [`Plugins`](common_base::Plugins) system: //! ```ignore //! plugins.insert(Arc::new(MyHook) as RegionHookRef); @@ -56,7 +61,39 @@ //! //! The following paths do **not** trigger any hook: //! - Follower region sync / catchup (manifest read-only; followers don't author changes) -//! - GC / checkpoint / drop / remap (internal bookkeeping, not logical state changes) +//! - GC / checkpoint / remap (internal bookkeeping, not logical state changes) +//! +//! An explicit region **drop** does fire lifecycle hooks — see +//! [Region lifecycle](#region-lifecycle). +//! +//! ## Region lifecycle +//! +//! Beyond manifest/SST observation, the hook observes the high-level lifecycle of an +//! active region: +//! +//! | Event | Method | When | +//! |-------|--------|------| +//! | Open | [`on_region_opened`] | A create or open request registers the region as active (the counterpart to close/drop). Does not fire for the compactor's transient regions or the catch-up reopen. | +//! | Close | [`on_region_closed`] | A close request (or a close-after-flush) removes the region from the active set. Data files, manifest and WAL state are **preserved**; the region may be reopened. | +//! | Logical drop | [`on_region_dropped`] | A drop request has been handled: the region leaves the active set and its WAL entries are marked obsolete. Data files are **not yet deleted**. | +//! | Physical file removal | [`on_region_files_removed`] | The drop GC worker has deleted the region directory. Terminal file-lifecycle event. | +//! +//! Notes: +//! - `on_region_closed` / `on_region_dropped` run **inline in the region worker loop**, +//! so implementations must be fast (same contract as `on_manifest_updated`). +//! - `on_region_opened` runs inline in the worker loop on the **create** path, but on the +//! **open** path it fires inside the spawned open task (`common_runtime::spawn_global`), +//! i.e. concurrently with the worker loop — after WAL replay, before the region is +//! registered and its open request is acknowledged. Implementations must still be fast +//! and must not assume worker-loop-thread affinity or strict ordering against concurrent +//! requests to other regions. +//! - `on_region_files_removed` runs on the background drop GC task, outside the worker loop. +//! - When global GC is enabled and a normal table region is dropped with `partial_drop`, its +//! directory is left for global reclamation and `on_region_files_removed` is **not** fired +//! by the drop worker (observe it via the global GC path instead). +//! - Logical file removal (compaction, region edit, truncate) is already observable via +//! [`on_manifest_updated`] (`Edit.files_to_remove` / `Truncate` action); only the drop +//! worker's physical directory deletion needs a dedicated file hook. //! //! ## Invocation points //! @@ -77,11 +114,18 @@ //! //! ## Future work //! -//! A future `on_files_removed` hook may be added to observe file lifecycle end -//! (GC, drop, truncate, compaction removal). This is not yet implemented. +//! `on_region_files_removed` currently covers only the **drop** GC worker's physical +//! directory removal. A broader per-file `on_files_removed` hook covering compaction +//! removal, truncate, and the global GC reclamation path is not yet implemented +//! (though logical file removal is already observable via `on_manifest_updated`). +//! Role/leadership transitions (`on_region_role_changed`) are also not hooked. //! //! [`on_sst_files_written`]: RegionHook::on_sst_files_written //! [`on_manifest_updated`]: RegionHook::on_manifest_updated +//! [`on_region_opened`]: RegionHook::on_region_opened +//! [`on_region_closed`]: RegionHook::on_region_closed +//! [`on_region_dropped`]: RegionHook::on_region_dropped +//! [`on_region_files_removed`]: RegionHook::on_region_files_removed //! [`RegionManifestManager::update`]: crate::manifest::manager::RegionManifestManager::update //! [`ManifestContext::update_locked`]: crate::region::ManifestContext::update_locked //! [`ManifestContext::update_manifest`]: crate::region::ManifestContext::update_manifest @@ -228,6 +272,80 @@ pub trait RegionHook: Send + Sync + Debug { ) { let _ = (region_id, action_list, manifest_version); } + + /// Called once a region **open** or **create** succeeds, but **before** the + /// region is registered in the engine's active set (`insert_region` runs + /// immediately afterwards). + /// + /// Fires once when a region becomes active via a create or open request — + /// the natural counterpart to [`on_region_closed`] / [`on_region_dropped`]. + /// It does **not** fire for the compactor's transient compaction regions + /// (`open_compaction_region`), nor for the internal reopen performed during + /// follower catch-up / leadership promotion. + /// + /// On the **create** path it runs inline in the region worker loop; on the + /// **open** path it runs inside the spawned open task + /// (`common_runtime::spawn_global`), concurrently with the worker loop + /// (after WAL replay, before the region is registered/acknowledged). + /// Implementations must be fast and must **not** assume worker-loop-thread + /// affinity or strict ordering against concurrent requests to other regions. + /// + /// [`on_region_closed`]: RegionHook::on_region_closed + /// [`on_region_dropped`]: RegionHook::on_region_dropped + async fn on_region_opened(&self, region_id: RegionId, region_metadata: &RegionMetadataRef) { + let _ = (region_id, region_metadata); + } + + /// Called after a region is **closed** via a close request. + /// + /// The region is removed from the engine's active set, but its data files, + /// manifest, and WAL state are **preserved**; the region may be reopened + /// later. Fires once per successful close, after the region's background + /// tasks (flush/compaction) have been stopped. + /// + /// Fires for a region of **any** role (leader or follower) that is closed. + /// Does **not** fire when a region is dropped (see [`on_region_dropped`]). + /// + /// Runs inline in the region worker loop; implementations should be fast. + /// + /// [`on_region_dropped`]: RegionHook::on_region_dropped + async fn on_region_closed(&self, region_id: RegionId, region_metadata: &RegionMetadataRef) { + let _ = (region_id, region_metadata); + } + + /// Called after a region is **logically dropped** (a drop request has been + /// handled). + /// + /// The region is removed from the active set and its WAL entries are marked + /// obsolete. Its data files are **not yet deleted** — they are scheduled for + /// asynchronous removal by the GC worker. Observe physical deletion via + /// [`on_region_files_removed`]. + /// + /// Runs inline in the region worker loop; implementations should be fast. + /// + /// [`on_region_files_removed`]: RegionHook::on_region_files_removed + async fn on_region_dropped(&self, region_id: RegionId, region_metadata: &RegionMetadataRef) { + let _ = (region_id, region_metadata); + } + + /// Called after a dropped region's data files are **physically removed** by + /// the drop GC worker (the region directory has been deleted). + /// + /// This is the terminal event in a region's file lifecycle; no further + /// callbacks fire for this region id afterwards. Fires only when the drop + /// worker itself deletes the directory. When global GC is enabled and the + /// region is a normal table region dropped with `partial_drop`, the + /// directory is left for global reclamation and this hook is **not** fired + /// by the drop worker. + /// + /// Runs on a background task, outside the region worker loop. + async fn on_region_files_removed( + &self, + region_id: RegionId, + region_metadata: &RegionMetadataRef, + ) { + let _ = (region_id, region_metadata); + } } pub type RegionHookRef = Arc; diff --git a/src/mito2/src/worker/handle_close.rs b/src/mito2/src/worker/handle_close.rs index a616a0e32dc..b582c276927 100644 --- a/src/mito2/src/worker/handle_close.rs +++ b/src/mito2/src/worker/handle_close.rs @@ -89,5 +89,13 @@ impl RegionWorkerLoop { // clean index build status. self.index_build_scheduler.on_region_closed(region_id).await; self.region_count.dec(); + + // Notify the region hook that the region has been closed. The region is + // fully stopped and unregistered, but its files/manifest are preserved. + // Runs inline; the hook contract requires it to be fast. + if let Some(hook) = region.manifest_ctx.hook() { + let metadata = region.metadata(); + hook.on_region_closed(region_id, &metadata).await; + } } } diff --git a/src/mito2/src/worker/handle_create.rs b/src/mito2/src/worker/handle_create.rs index 6e43e74a0e4..bf254dc1e6b 100644 --- a/src/mito2/src/worker/handle_create.rs +++ b/src/mito2/src/worker/handle_create.rs @@ -86,6 +86,12 @@ impl RegionWorkerLoop { self.region_count.inc(); + // Notify the region hook that the region has been opened (created). + // Fires before registration; allocates nothing when no hook is registered. + if let Some(hook) = region.manifest_ctx.hook() { + hook.on_region_opened(region_id, ®ion.metadata()).await; + } + // Insert the MitoRegion into the RegionMap. self.regions.insert_region(region); diff --git a/src/mito2/src/worker/handle_drop.rs b/src/mito2/src/worker/handle_drop.rs index 00a96b0d166..3d111776b82 100644 --- a/src/mito2/src/worker/handle_drop.rs +++ b/src/mito2/src/worker/handle_drop.rs @@ -23,10 +23,12 @@ use object_store::util::join_path; use object_store::{EntryMode, ObjectStore}; use snafu::ResultExt; use store_api::logstore::LogStore; +use store_api::metadata::RegionMetadataRef; use store_api::region_request::{AffectedRows, PathType}; use store_api::storage::RegionId; use tokio::time::sleep; +use crate::engine::region_hook::RegionHookRef; use crate::error::{OpenDalSnafu, Result}; use crate::region::{RegionLeaderState, RegionMapRef}; use crate::worker::{DROPPING_MARKER_FILE, RegionWorkerLoop}; @@ -106,6 +108,22 @@ where self.region_count.dec(); + // Notify registered hooks that the region has been logically dropped, and + // prepare a payload for the background GC task to fire the terminal + // `on_region_files_removed`. When no hook is registered this allocates + // nothing — no metadata snapshot is taken and no payload is built. + let hook_payload = match region.manifest_ctx.hook() { + Some(hook) => { + let region_metadata = region.metadata(); + hook.on_region_dropped(region_id, ®ion_metadata).await; + Some(DropHookPayload { + hook, + metadata: region_metadata, + }) + } + None => None, + }; + let object_store = region.access_layer.object_store().clone(); let dropping_regions = self.dropping_regions.clone(); let listener = self.listener.clone(); @@ -122,6 +140,7 @@ where object_store, dropping_regions, partial_drop, + hook_payload.as_ref(), ) .await } else { @@ -135,6 +154,7 @@ where object_store, dropping_regions, gc_duration, + hook_payload.as_ref(), ) .await }; @@ -156,6 +176,19 @@ where } } +/// Carries the region hook and region metadata into the background GC task so +/// it can fire [`RegionHook::on_region_files_removed`] once the dropped region's +/// directory is physically deleted. +/// +/// Only constructed when a hook is registered; the task receives it as +/// `Option<&DropHookPayload>` so the no-hook path allocates nothing. +/// +/// [`RegionHook::on_region_files_removed`]: crate::engine::region_hook::RegionHook::on_region_files_removed +struct DropHookPayload { + hook: RegionHookRef, + metadata: RegionMetadataRef, +} + /// Background GC task to remove the entire region path once one of the following /// conditions is true: /// - It finds there is no parquet file left. @@ -172,6 +205,7 @@ async fn later_drop_task_without_global_gc( object_store: ObjectStore, dropping_regions: RegionMapRef, gc_duration: Duration, + hook_payload: Option<&DropHookPayload>, ) -> bool { remove_region_with_retry( region_id, @@ -180,6 +214,7 @@ async fn later_drop_task_without_global_gc( dropping_regions, Some(gc_duration), false, + hook_payload, ) .await } @@ -191,6 +226,7 @@ async fn remove_region_with_retry( dropping_regions: std::sync::Arc, gc_duration: Option, mut force: bool, + hook_payload: Option<&DropHookPayload>, ) -> bool { for _ in 0..MAX_RETRY_TIMES { let result = remove_region_dir_once(®ion_path, &object_store, force).await; @@ -204,6 +240,15 @@ async fn remove_region_with_retry( Ok(true) => { dropping_regions.remove_region(region_id); info!("Region {} is dropped, force: {}", region_path, force); + // The region directory has been physically deleted; fire the + // terminal file-lifecycle event. The partial-drop/global-GC path + // never reaches here (it does not delete the directory itself). + if let Some(hook_payload) = hook_payload { + hook_payload + .hook + .on_region_files_removed(region_id, &hook_payload.metadata) + .await; + } return true; } Ok(false) => (), @@ -230,6 +275,7 @@ async fn later_drop_task_with_global_gc( object_store: ObjectStore, dropping_regions: RegionMapRef, partial_drop: bool, + hook_payload: Option<&DropHookPayload>, ) -> bool { // For metadata regions or regions marked for full deletion (such as when dropping a table) // the region directory is forcefully removed immediately. @@ -243,6 +289,7 @@ async fn later_drop_task_with_global_gc( dropping_regions, None, true, + hook_payload, ) .await } else { diff --git a/src/mito2/src/worker/handle_open.rs b/src/mito2/src/worker/handle_open.rs index 3d3520b0cc8..48fef5ddd6f 100644 --- a/src/mito2/src/worker/handle_open.rs +++ b/src/mito2/src/worker/handle_open.rs @@ -149,6 +149,13 @@ impl RegionWorkerLoop { ); region_count.inc(); + // Notify the region hook that the region has been opened. + // Fires before registration; allocates nothing when no hook + // is registered. + if let Some(hook) = region.manifest_ctx.hook() { + hook.on_region_opened(region_id, ®ion.metadata()).await; + } + // Insert the Region into the RegionMap. regions.insert_region(region); From a12ad8df525dd25a7ae2050ad298b9a39e4ee87d Mon Sep 17 00:00:00 2001 From: Ning Sun Date: Mon, 13 Jul 2026 14:35:54 +0800 Subject: [PATCH 7/8] refactor: suppress region hook notifications for staging-only manifest writes (#8471) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only fire `on_manifest_updated` for writes to the live (normal) manifest directory. Writes to the staging directory (enter staging, operations during staging, the intermediate apply-staging `RegionEdit`) are suppressed — their effects accumulate and are delivered in a single notification when `exit_staging_on_success` promotes all staged actions to the live manifest. This simplifies hook implementations that only care about "live" state, since they no longer receive intermediate staging notifications they cannot act on, and no file-list information is lost (the promote step carries all accumulated files from staging operations). Key changes: - `PendingManifestHook` gains an `is_staging` field; `fire()` is a no-op when `is_staging` is `true` - `ManifestContext::update_locked` threads its `is_staging` parameter through - Updated module-level docs and coverage table Signed-off-by: Ning Sun --- src/mito2/src/engine/region_hook.rs | 62 +++++++++++++++++++++++------ src/mito2/src/region.rs | 1 + 2 files changed, 50 insertions(+), 13 deletions(-) diff --git a/src/mito2/src/engine/region_hook.rs b/src/mito2/src/engine/region_hook.rs index c4b683ca831..9c5a7dfc95d 100644 --- a/src/mito2/src/engine/region_hook.rs +++ b/src/mito2/src/engine/region_hook.rs @@ -23,9 +23,13 @@ //! Provides per-file [`SstInfo`] + [`FileMeta`]; metadata richness varies by path //! (see [`SstFileInfo`] and the coverage footnote). //! -//! - [`on_manifest_updated`]: Fires after **any** manifest write is successfully committed. -//! Receives the full [`RegionMetaActionList`] so consumers can inspect what changed -//! (file additions/removals, schema changes, truncation, partition expression changes, etc.). +//! - [`on_manifest_updated`]: Fires after a manifest write is committed to the **live** +//! (normal) manifest directory. Writes to the staging directory (enter staging, +//! operations during staging, the intermediate apply-staging edit) are suppressed — +//! their effects are accumulated and delivered in a single notification when the +//! staged actions are promoted to the live manifest. Receives the full +//! [`RegionMetaActionList`] so consumers can inspect what changed (file additions/ +//! removals, schema changes, truncation, partition expression changes, etc.). //! //! - [`on_region_opened`] / [`on_region_closed`] / [`on_region_dropped`] / [`on_region_files_removed`]: //! Region **lifecycle** callbacks for open, close, logical drop, and physical file removal. @@ -38,6 +42,12 @@ //! //! ## Coverage //! +//! Only manifest writes to the **normal** (live) manifest directory trigger +//! `on_manifest_updated`. Writes to the staging manifest directory (operations +//! that happen while the region is in staging mode) are intentionally suppressed: +//! their effects are accumulated and delivered in a single notification when +//! the staged actions are promoted to the live manifest via `exit_staging_on_success`. +//! //! | Scenario | `on_sst_files_written` | `on_manifest_updated` | //! |------------------------------|:----------------------:|:---------------------:| //! | Flush (memtable → SST) | ✅ Yes | ✅ Yes | @@ -45,19 +55,21 @@ //! | Remote compaction | ✅ (compactor node) ¹ | ✅ (compactor node) ¹ | //! | RegionEdit / bulk ingestion | ❌ (files pre-written) | ✅ Yes | //! | Copy region | ❌ (object-store copy) | ✅ Yes | -//! | Apply staging | ❌ (delegates to edit) | ✅ Yes ² | +//! | Apply staging (promote) | ❌ (delegates to edit) | ✅ Yes ² | //! | Alter (schema change) | ❌ (no SST files) | ✅ Yes | //! | Truncate | ❌ (removes files) | ✅ Yes | -//! | Enter staging | ❌ (no SST files) | ✅ Yes | +//! | Enter staging | ❌ (no SST files) | ❌ (staging dir) | +//! | Operations during staging | N/A | ❌ (staging dir) | //! | Async index build | ❌ (index files only) | ✅ Yes | //! //! ¹ Remote compaction runs on a dedicated compactor node via `open_compaction_region()`; //! pass plugins via `OpenCompactionRegionRequest` to enable hooks there. `sst_infos` is //! `#[serde(skip)]` over the wire, so each [`SstInfo`] is rebuilt from [`FileMeta`] with //! empty footer/index — see [`SstFileInfo`] for field-level detail. -//! ² Apply staging fires `on_manifest_updated` twice: once when the staging SST files are -//! committed via `RegionEdit`, and once when `exit_staging_on_success` merges all staged -//! manifest actions into the live manifest. +//! ² Apply staging fires `on_manifest_updated` once when `exit_staging_on_success` promotes +//! all staged manifest actions (including the SST file additions) into the live manifest. +//! The intermediate staging `RegionEdit` is written to the staging directory and does not +//! fire the hook — its file list is included in the promote notification. //! //! The following paths do **not** trigger any hook: //! - Follower region sync / catchup (manifest read-only; followers don't author changes) @@ -147,6 +159,13 @@ use crate::sst::parquet::SstInfo; /// /// Must be [`fire`](Self::fire)d **after** the manifest write lock is released /// (the hook may read the manifest). `#[must_use]` so a forgotten receipt warns. +/// +/// ## Staging suppression +/// +/// When `is_staging` is `true`, [`fire`](Self::fire) is a no-op — the write went to +/// the staging manifest directory. The hook only observes writes to the live (normal) +/// manifest directory. Staging actions are accumulated and delivered in a single +/// notification when `exit_staging_on_success` promotes them (`is_staging = false`). #[must_use = "the region hook must be fired after releasing the manifest write lock"] pub(crate) struct PendingManifestHook { region_id: RegionId, @@ -154,6 +173,9 @@ pub(crate) struct PendingManifestHook { action_list: Option, version: ManifestVersion, hook: Option, + /// Whether the manifest write went to the staging directory. + /// When `true`, `fire()` is suppressed — the hook only observes live manifest writes. + is_staging: bool, } impl PendingManifestHook { @@ -162,12 +184,14 @@ impl PendingManifestHook { action_list: Option, version: ManifestVersion, hook: Option, + is_staging: bool, ) -> Self { Self { region_id, action_list, version, hook, + is_staging, } } @@ -176,9 +200,13 @@ impl PendingManifestHook { self.version } - /// Fires the hook if one is registered. Safe to call unconditionally: it is - /// a no-op when no hook is registered. + /// Fires the hook if one is registered, **unless** the write went to the staging + /// manifest directory (`is_staging = true`). Safe to call unconditionally: + /// it is a no-op when no hook is registered or when the write is staging-only. pub(crate) async fn fire(self) { + if self.is_staging { + return; + } if let (Some(hook), Some(action_list)) = (self.hook, self.action_list) { hook.on_manifest_updated(self.region_id, &action_list, self.version) .await; @@ -190,6 +218,9 @@ impl PendingManifestHook { /// keeps `self`'s actions followed by `other`'s, and the *later* manifest /// version wins. Used when a sequence of writes (e.g. staging-exit followed /// by metadata backfill) should notify the hook exactly once. + /// + /// `is_staging` is `true` only if **both** sides are staging; if either side + /// is a live write, the merged result is also live. pub(crate) fn merge(self, other: PendingManifestHook) -> PendingManifestHook { debug_assert_eq!( self.region_id, other.region_id, @@ -208,6 +239,7 @@ impl PendingManifestHook { }, version: self.version.max(other.version), hook: self.hook.or(other.hook), + is_staging: self.is_staging && other.is_staging, } } } @@ -255,10 +287,14 @@ pub trait RegionHook: Send + Sync + Debug { let _ = (region_id, region_metadata, files); } - /// Called after the region manifest is successfully committed. + /// Called after the region manifest is successfully committed to the **live** + /// (normal) manifest directory. /// - /// Fires for **all** manifest write paths: flush, compaction, region edit, - /// copy region, alter, truncate, enter staging, index build, etc. + /// Fires for: flush, compaction, region edit, copy region, alter, truncate, + /// async index build, and apply-staging promote. Does **not** fire for writes + /// to the staging manifest directory (enter staging, operations during staging, + /// the intermediate apply-staging edit) — those are suppressed because their + /// effects are accumulated and delivered in a single promote notification. /// /// Does **not** fire for: /// - Manifest reads / follower sync (no write) diff --git a/src/mito2/src/region.rs b/src/mito2/src/region.rs index 1e173ab62d2..c566dd47c04 100644 --- a/src/mito2/src/region.rs +++ b/src/mito2/src/region.rs @@ -1236,6 +1236,7 @@ impl ManifestContext { action_list_for_hook, version, self.hook.clone(), + is_staging, )) } From 758664eefab604ccbf3130ec40e7c38bfbcaede4 Mon Sep 17 00:00:00 2001 From: Yingwen Date: Mon, 13 Jul 2026 18:19:57 +0800 Subject: [PATCH 8/8] fix: Use prepared file locations for CSV strict headers integration test (#8493) fix: normalize Windows COPY test paths Signed-off-by: Codex --- tests-integration/src/tests/instance_test.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests-integration/src/tests/instance_test.rs b/tests-integration/src/tests/instance_test.rs index 628f97de96a..8572e9ce5b2 100644 --- a/tests-integration/src/tests/instance_test.rs +++ b/tests-integration/src/tests/instance_test.rs @@ -1098,6 +1098,10 @@ async fn test_execute_copy_from_csv_strict_headers(instance: Arc