diff --git a/.github/actions/setup-greptimedb-cluster/with-remote-wal-logical-prune.yaml b/.github/actions/setup-greptimedb-cluster/with-remote-wal-logical-prune.yaml index 60c23def6e2..3b25e80a8f2 100644 --- a/.github/actions/setup-greptimedb-cluster/with-remote-wal-logical-prune.yaml +++ b/.github/actions/setup-greptimedb-cluster/with-remote-wal-logical-prune.yaml @@ -3,6 +3,7 @@ logging: format: "json" filters: - log_store=debug + - rskafka=debug meta: configData: |- [runtime] diff --git a/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml b/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml index ec595b9e9b8..6a15b25b422 100644 --- a/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml +++ b/.github/actions/setup-greptimedb-cluster/with-remote-wal.yaml @@ -3,6 +3,7 @@ logging: format: "json" filters: - log_store=debug + - rskafka=debug meta: configData: |- [runtime] diff --git a/.github/actions/setup-kafka-cluster/action.yml b/.github/actions/setup-kafka-cluster/action.yml index fca420da709..3c30ca285e3 100644 --- a/.github/actions/setup-kafka-cluster/action.yml +++ b/.github/actions/setup-kafka-cluster/action.yml @@ -16,8 +16,8 @@ runs: helm upgrade \ --install kafka oci://registry-1.docker.io/bitnamicharts/kafka \ --set controller.replicaCount=${{ inputs.controller-replicas }} \ - --set controller.resources.requests.cpu=50m \ - --set controller.resources.requests.memory=128Mi \ + --set controller.resources.requests.cpu=500m \ + --set controller.resources.requests.memory=512Mi \ --set controller.resources.limits.cpu=2000m \ --set controller.resources.limits.memory=2Gi \ --set listeners.controller.protocol=PLAINTEXT \ diff --git a/.github/scripts/query-regression-run.py b/.github/scripts/query-regression-run.py index b89857be81f..62d11f44203 100644 --- a/.github/scripts/query-regression-run.py +++ b/.github/scripts/query-regression-run.py @@ -26,6 +26,10 @@ from pathlib import Path DEFAULT_CASES = [ "tests/perf/query_cases/smoke_direct_sst/case.toml", + "tests/perf/query_cases/prom_remote_write_seeded_random/case.toml", + "tests/perf/query_cases/prom_remote_write_run_heavy/case.toml", + "tests/perf/query_cases/prom_remote_write_mixed_every/case.toml", + "tests/perf/query_cases/prom_remote_write_integer_counter/case.toml", "tests/perf/query_cases/promql_pushdown_7913/case.toml", "tests/perf/query_cases/sql_topk_order_by/case.toml", "tests/perf/query_cases/sql_aggregate_order_by/case.toml", @@ -106,8 +110,6 @@ def run_case(args: argparse.Namespace, case_path: Path, work_dir: Path) -> int: str(candidate_bin), "--fixture-generator", str(fixture_generator), - "--remote-write-generator", - str(args.remote_write_generator), "--work-dir", str(work_dir), "--http-timeout", @@ -160,11 +162,6 @@ def main() -> int: parser.add_argument("--work-dir", default=Path("query-regression-work"), type=Path) parser.add_argument("--http-timeout", default=os.environ.get("HTTP_TIMEOUT", "300")) parser.add_argument("--allow-large-fixture", default=os.environ.get("ALLOW_LARGE_FIXTURE", "false")) - parser.add_argument( - "--remote-write-generator", - type=Path, - default=configured_path(os.environ.get("REMOTE_WRITE_GENERATOR")), - ) parser.add_argument( "--summary-script", type=Path, @@ -177,9 +174,6 @@ def main() -> int: parser.add_argument("--candidate-ref", default=os.environ.get("CANDIDATE_REF", "")) parser.add_argument("--github-output", default=os.environ.get("GITHUB_OUTPUT")) args = parser.parse_args() - if args.remote_write_generator is None: - args.remote_write_generator = args.candidate_src / "target" / profile_dir(args.cargo_profile) / "prom_remote_write_fixture" - try: cases = split_cases(args.cases or [os.environ.get("CASE_PATHS", "all")]) except ValueError as err: diff --git a/.github/workflows/query-regression.yml b/.github/workflows/query-regression.yml index a898be6ad29..619442e2f95 100644 --- a/.github/workflows/query-regression.yml +++ b/.github/workflows/query-regression.yml @@ -119,10 +119,15 @@ jobs: git reset --hard FETCH_HEAD git clean -ffdx + - name: Test query regression runner lifecycle + working-directory: src + run: uv run --no-project python tests/perf/test_query_regression_runner_compaction_toctou.py + - name: Build candidate greptime and fixture generators working-directory: src run: | - cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime --bin query_perf_fixture --bin prom_remote_write_fixture + cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime + cargo build --profile "${CARGO_PROFILE}" -p cmd --bin query_perf_fixture --features dev-tools target_dir="${CARGO_PROFILE}" if [[ "${CARGO_PROFILE}" == "dev" ]]; then target_dir="debug" @@ -132,8 +137,6 @@ jobs: "${GITHUB_WORKSPACE}/query-regression-bins/candidate/greptime" cp "${CARGO_TARGET_DIR}/${target_dir}/query_perf_fixture" \ "${GITHUB_WORKSPACE}/query-regression-bins/candidate/query_perf_fixture" - cp "${CARGO_TARGET_DIR}/${target_dir}/prom_remote_write_fixture" \ - "${GITHUB_WORKSPACE}/query-regression-bins/candidate/prom_remote_write_fixture" - name: Run query regression id: run @@ -146,7 +149,6 @@ jobs: BASE_BIN: ${{ github.workspace }}/query-regression-bins/base/greptime CANDIDATE_BIN: ${{ github.workspace }}/query-regression-bins/candidate/greptime FIXTURE_GENERATOR: ${{ github.workspace }}/query-regression-bins/candidate/query_perf_fixture - REMOTE_WRITE_GENERATOR: ${{ github.workspace }}/query-regression-bins/candidate/prom_remote_write_fixture run: >- uv run --no-project python src/.github/scripts/query-regression-run.py --base-src src @@ -171,9 +173,13 @@ jobs: with: name: query-regression-report path: | - query-regression-work/** + query-regression-work/**/query-regression-report.json + query-regression-work/**/base/report.json + query-regression-work/**/candidate/report.json + query-regression-work/**/logs/** query-regression-summary.md if-no-files-found: warn + retention-days: 7 - name: Upload trusted comment artifact if: ${{ always() && github.event_name == 'pull_request' }} diff --git a/Cargo.lock b/Cargo.lock index e00e30e28a6..75ba5ce410b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11038,9 +11038,9 @@ dependencies = [ [[package]] name = "promql-parser" -version = "0.7.3" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f514ab521bb2b4ce26aca29bf133498b2a2651b785d03a262bbf5772329c279f" +checksum = "9e8c90c956be4237a93be6f2298302b56974ef862ab11e0f98869d1f27c69f88" dependencies = [ "cfgrammar", "chrono", @@ -11183,7 +11183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -11196,7 +11196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425" dependencies = [ "anyhow", - "itertools 0.11.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", diff --git a/Cargo.toml b/Cargo.toml index 5ee60c450e8..fd878c7c0a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -198,7 +198,7 @@ paste = "1.0" pin-project = "1.0" pretty_assertions = "1.4.0" prometheus = { version = "0.14", features = ["process"] } -promql-parser = { version = "0.7.3", features = ["ser"] } +promql-parser = { version = "0.10.0", features = ["ser"] } prost = { version = "0.14", features = ["no-recursion-limit"] } prost-types = "0.14" raft-engine = { version = "0.4.1", default-features = false } diff --git a/README.md b/README.md index 3ba52bc5266..99c78814609 100644 --- a/README.md +++ b/README.md @@ -209,7 +209,13 @@ Read the [v1.0 highlights](https://greptime.com/blogs/2025-11-05-greptimedb-v1-h If GreptimeDB is useful to you, please star the repo. -[![Star History Chart](https://api.star-history.com/svg?repos=GreptimeTeam/GreptimeDB&type=Date)](https://www.star-history.com/#GreptimeTeam/GreptimeDB&Date) + + + + + Star History Chart + + Known Users diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml index a8adfa01a48..b06da6e517a 100644 --- a/src/cmd/Cargo.toml +++ b/src/cmd/Cargo.toml @@ -12,10 +12,7 @@ path = "src/bin/greptime.rs" [[bin]] name = "query_perf_fixture" path = "src/bin/query_perf_fixture.rs" - -[[bin]] -name = "prom_remote_write_fixture" -path = "src/bin/prom_remote_write_fixture.rs" +required-features = ["dev-tools"] [features] default = [ @@ -25,7 +22,7 @@ default = [ "meta-srv/mysql_kvbackend", ] enterprise = ["common-meta/enterprise", "frontend/enterprise", "meta-srv/enterprise"] -# Developer-only tools (benchmarks, diagnostics) gating the `datanode parquetbench` subcommand. +# Developer-only helper binary gate for `query_perf_fixture`. # Kept out of `default` so normal/release builds don't compile them. dev-tools = [] tokio-console = ["common-telemetry/tokio-console"] diff --git a/src/cmd/src/bin/greptime.rs b/src/cmd/src/bin/greptime.rs index e0f326908a2..957e227355a 100644 --- a/src/cmd/src/bin/greptime.rs +++ b/src/cmd/src/bin/greptime.rs @@ -117,7 +117,6 @@ async fn start(cli: Command) -> Result<()> { } datanode::SubCommand::Objbench(ref bench) => bench.run().await, datanode::SubCommand::Scanbench(ref bench) => bench.run().await, - #[cfg(feature = "dev-tools")] datanode::SubCommand::Parquetbench(ref bench) => bench.run().await, }, SubCommand::Flownode(cmd) => { diff --git a/src/cmd/src/bin/prom_remote_write_fixture.rs b/src/cmd/src/bin/prom_remote_write_fixture.rs deleted file mode 100644 index 0bfdf954823..00000000000 --- a/src/cmd/src/bin/prom_remote_write_fixture.rs +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#![allow(clippy::print_stdout)] - -use std::time::{Duration, Instant}; - -use api::prom_store::remote::{Label, Sample, TimeSeries, WriteRequest}; -use clap::Parser; -use prost::Message; -use serde_json::json; -use servers::prom_store::snappy_compress; - -#[derive(Debug, Parser)] -#[command(about = "Generate a deterministic Prometheus remote-write fixture")] -struct Args { - #[arg(long, default_value = "http://127.0.0.1:4000/v1/prometheus/write")] - endpoint: String, - #[arg(long, default_value = "public")] - database: String, - #[arg(long)] - metric: String, - #[arg(long, default_value = "greptime_physical_table")] - physical_table: String, - #[arg(long, default_value_t = 8)] - series_count: u64, - #[arg(long, default_value_t = 30)] - samples_per_series: u64, - #[arg(long, default_value_t = 1_704_067_200_000)] - start_unix_millis: i64, - #[arg(long, default_value_t = 15_000)] - step_millis: i64, - #[arg(long, alias = "batch-size", default_value_t = 8)] - chunk_series_count: u64, - #[arg(long, default_value_t = 60)] - timeout_seconds: u64, -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - let args = Args::parse(); - let started = Instant::now(); - let client = reqwest::Client::builder() - .timeout(Duration::from_secs(args.timeout_seconds)) - .build()?; - let mut batches = 0_u64; - let mut rows = 0_u64; - let mut http_statuses = Vec::new(); - - let chunk = args.chunk_series_count.max(1); - for first in (0..args.series_count).step_by(chunk as usize) { - let last = (first + chunk).min(args.series_count); - let timeseries = (first..last) - .map(|series_idx| make_series(&args, series_idx)) - .collect::>(); - rows += timeseries - .iter() - .map(|ts| ts.samples.len() as u64) - .sum::(); - let body = snappy_compress( - &WriteRequest { - timeseries, - ..Default::default() - } - .encode_to_vec(), - )?; - let response = client - .post(&args.endpoint) - .header("Content-Encoding", "snappy") - .body(body) - .send() - .await?; - let status = response.status(); - http_statuses.push(status.as_u16()); - if !status.is_success() { - let body_text = response.text().await.unwrap_or_default(); - return Err(format!("remote-write failed with status {status}: {body_text}").into()); - } - batches += 1; - } - - println!( - "{}", - json!({ - "status": "ok", - "endpoint": args.endpoint, - "database": args.database, - "metric": args.metric, - "physical_table": args.physical_table, - "series_count": args.series_count, - "samples_per_series": args.samples_per_series, - "rows": rows, - "samples_written": rows, - "batches": batches, - "elapsed_seconds": started.elapsed().as_secs_f64(), - "http_statuses": http_statuses, - }) - ); - Ok(()) -} - -fn make_series(args: &Args, series_idx: u64) -> TimeSeries { - let mut labels = vec![ - label("__name__", &args.metric), - label("x_greptime_database", &args.database), - label("x_greptime_physical_table", &args.physical_table), - label("host", &format!("host{:04}", series_idx % 1024)), - label("instance", &format!("instance{:06}", series_idx)), - ]; - labels.sort_unstable_by(|left, right| left.name.cmp(&right.name)); - let samples = (0..args.samples_per_series) - .map(|sample_idx| Sample { - value: deterministic_value(series_idx, sample_idx), - timestamp: args.start_unix_millis + (sample_idx as i64 * args.step_millis), - }) - .collect(); - TimeSeries { - labels, - samples, - ..Default::default() - } -} - -fn label(name: &str, value: &str) -> Label { - Label { - name: name.to_string(), - value: value.to_string(), - } -} - -fn deterministic_value(series_idx: u64, sample_idx: u64) -> f64 { - ((series_idx % 97) as f64) + (sample_idx as f64 * 0.125) -} diff --git a/src/cmd/src/bin/query_perf_fixture.rs b/src/cmd/src/bin/query_perf_fixture.rs index dfc3a664a03..d0e185038fa 100644 --- a/src/cmd/src/bin/query_perf_fixture.rs +++ b/src/cmd/src/bin/query_perf_fixture.rs @@ -14,656 +14,10 @@ #![allow(clippy::print_stderr, clippy::print_stdout)] -//! Generic direct-readable-SST fixture generator for query performance labs. -//! -//! This is a bounded MVP: it reads one table/one region case TOML, builds -//! synthetic region metadata, writes real Mito readable SST parquet files, and -//! emits a manifest checkpoint plus validation metadata. The first case uses -//! `sst_format = "flat"`; primary-key output is also accepted for early lab use -//! because the writer can convert the same flat input batch to primary-key SSTs. -//! -//! TODO(query-perf): add multi-region generation and seed-manifest/catalog DB -//! integration so the generated fixture can be mounted directly by a full -//! GreptimeDB open/query path without manual metadata wiring. - -use std::collections::HashMap; -use std::fs; -use std::io::{BufWriter, Write}; -use std::num::{NonZeroU64, NonZeroUsize}; -use std::path::{Path, PathBuf}; -use std::sync::Arc; - -use api::v1::{OpType, SemanticType}; -use async_trait::async_trait; -use clap::Parser; -use datatypes::arrow::array::{ - ArrayRef, BinaryDictionaryBuilder, Float64Array, RecordBatch, StringDictionaryBuilder, - TimestampMillisecondArray, TimestampNanosecondArray, UInt8Array, UInt64Array, -}; -use datatypes::arrow::datatypes::UInt32Type; -use datatypes::prelude::ConcreteDataType; -use datatypes::schema::{ColumnSchema, SkippingIndexOptions}; -use mito_codec::row_converter::{DensePrimaryKeyCodec, PrimaryKeyCodecExt, SortField}; -use mito2::access_layer::{FilePathProvider, Metrics, WriteType}; -use mito2::config::IndexConfig; -use mito2::manifest::action::{RegionCheckpoint, RegionManifest, RemovedFilesRecord}; -use mito2::read::FlatSource; -use mito2::sst::file::{FileMeta, RegionFileId}; -use mito2::sst::index::{Indexer, IndexerBuilder}; -use mito2::sst::parquet::writer::ParquetWriter; -use mito2::sst::parquet::{SstInfo, WriteOptions}; -use mito2::sst::{ - DEFAULT_WRITE_BUFFER_SIZE, FlatSchemaOptions, FormatType, to_flat_sst_arrow_schema, -}; -use object_store::ObjectStore; -use object_store::services::Fs as FsBuilder; -use serde::Deserialize; -use store_api::codec::PrimaryKeyEncoding; -use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataRef}; -use store_api::path_utils::region_name; -use store_api::region_request::PathType; -use store_api::storage::{FileId, RegionId}; - -#[derive(Parser, Debug)] -#[command(name = "query_perf_fixture")] -#[command(about = "Generate direct-readable SST fixtures for query performance cases")] -struct Args { - /// TOML case file, for example tests/perf/query_cases/.../case.toml. - #[arg(long, value_name = "PATH")] - case: PathBuf, - - /// Output directory (creates object-store/, manifest/, files.jsonl, summary.json). - #[arg(long, value_name = "DIR")] - out_dir: PathBuf, - - /// Region ID to synthesize. Defaults to table id 1024, region 0. - #[arg(long, default_value = "4398046511104")] - region_id: u64, - - /// Table directory relative to object-store root. Defaults to data/{database}/{table}/. - #[arg(long, value_name = "DIR")] - table_dir: Option, - - /// Table name to generate when the case contains multiple [[tables]]. - #[arg(long, value_name = "NAME")] - table: Option, - - /// Manifest/checkpoint version. - #[arg(long, default_value = "1000000")] - checkpoint_version: u64, - - /// Safety flag when sst_count exceeds 1000. - #[arg(long)] - allow_large: bool, - - /// Print plan only. - #[arg(long)] - dry_run: bool, -} - -#[derive(Debug, Deserialize)] -struct CaseFile { - scenario: Scenario, -} - -#[derive(Debug, Deserialize)] -#[serde(tag = "kind")] -enum Scenario { - #[serde(rename = "direct_readable_sst")] - DirectReadableSst(DirectReadableSstScenario), -} - -#[derive(Debug, Deserialize)] -struct DirectReadableSstScenario { - #[serde(default)] - seed: Option, - tables: Vec, - layout: LayoutConfig, -} - -impl Scenario { - fn kind(&self) -> &'static str { - match self { - Scenario::DirectReadableSst(_) => "direct_readable_sst", - } - } - - fn direct_readable_sst(&self) -> &DirectReadableSstScenario { - match self { - Scenario::DirectReadableSst(scenario) => scenario, - } - } -} - -#[derive(Debug, Deserialize)] -struct TableConfig { - database: String, - name: String, - engine: String, - #[serde(default)] - append_mode: Option, - #[serde(default)] - sst_format: Option, - primary_key: Vec, - time_index: String, - columns: Vec, -} - -#[derive(Debug, Deserialize)] -struct ColumnConfig { - name: String, - #[serde(rename = "type")] - ty: String, - semantic: String, - distribution: Option, -} - -#[derive(Debug, Deserialize)] -#[serde(tag = "kind")] -enum Distribution { - #[serde(rename = "cardinality")] - Cardinality { - values: NonZeroUsize, - prefix: String, - }, - #[serde(rename = "deterministic_wave")] - DeterministicWave { min: f64, max: f64 }, -} - -#[derive(Debug, Deserialize)] -struct LayoutConfig { - regions: usize, - sst_count: usize, - rows_per_sst: usize, - row_group_size: usize, - series_count: NonZeroUsize, - start_unix_nanos: i64, - step_nanos: i64, - time_range_layout: String, - series_layout: String, -} - -struct NoopIndexBuilder; - -#[async_trait] -impl IndexerBuilder for NoopIndexBuilder { - async fn build(&self, _file_id: FileId, _index_version: u64) -> Indexer { - Indexer::default() - } -} - -#[derive(Clone)] -struct FixedPathProvider { - table_dir: String, -} - -impl FilePathProvider for FixedPathProvider { - fn build_index_file_path(&self, file_id: RegionFileId) -> String { - mito2::sst::location::index_file_path_legacy(&self.table_dir, file_id, PathType::Bare) - } - fn build_index_file_path_with_version( - &self, - index_id: mito2::sst::file::RegionIndexId, - ) -> String { - mito2::sst::location::index_file_path(&self.table_dir, index_id, PathType::Bare) - } - fn build_sst_file_path(&self, file_id: RegionFileId) -> String { - mito2::sst::location::sst_file_path(&self.table_dir, file_id, PathType::Bare) - } -} - -fn semantic_type(s: &str) -> SemanticType { - match s.to_ascii_lowercase().as_str() { - "tag" => SemanticType::Tag, - "field" => SemanticType::Field, - "timestamp" => SemanticType::Timestamp, - other => panic!("unsupported semantic {other}"), - } -} - -fn concrete_type(ty: &str) -> ConcreteDataType { - match ty.to_ascii_uppercase().as_str() { - "STRING" => ConcreteDataType::string_datatype(), - "DOUBLE" => ConcreteDataType::float64_datatype(), - "UINT64" => ConcreteDataType::uint64_datatype(), - "TIMESTAMP(9)" => ConcreteDataType::timestamp_nanosecond_datatype(), - "TIMESTAMP(3)" => ConcreteDataType::timestamp_millisecond_datatype(), - other => panic!("unsupported column type {other}"), - } -} - -fn build_region_metadata(table: &TableConfig, region_id: RegionId) -> RegionMetadata { - let mut builder = store_api::metadata::RegionMetadataBuilder::new(region_id); - for (idx, col) in table.columns.iter().enumerate() { - let mut schema = ColumnSchema::new( - &col.name, - concrete_type(&col.ty), - col.semantic != "timestamp", - ); - if col.semantic == "tag" { - schema = schema.with_inverted_index(true); - } - if col.semantic == "tag" || col.semantic == "field" { - schema = schema - .with_skipping_options(SkippingIndexOptions { - granularity: 1, - ..Default::default() - }) - .expect("valid skipping index options for query perf fixture"); - } - if col.name == table.time_index { - schema = schema.with_time_index(true); - } - builder.push_column_metadata(ColumnMetadata { - column_schema: schema, - semantic_type: semantic_type(&col.semantic), - column_id: idx as u32, - }); - } - let pk_ids = table - .primary_key - .iter() - .map(|name| { - table - .columns - .iter() - .position(|c| c.name == *name) - .unwrap_or_else(|| panic!("primary key column {name} not found")) as u32 - }) - .collect(); - builder.primary_key(pk_ids); - builder.primary_key_encoding(PrimaryKeyEncoding::Dense); - builder - .build() - .expect("region metadata should be valid for fixture table config") -} - -fn encode_dense_primary_key(table: &TableConfig, values: &HashMap) -> Vec { - let fields = table - .primary_key - .iter() - .enumerate() - .map(|(idx, _)| { - ( - idx as u32, - SortField::new(ConcreteDataType::string_datatype()), - ) - }) - .collect(); - let converter = DensePrimaryKeyCodec::with_fields(fields); - converter - .encode( - table - .primary_key - .iter() - .map(|name| datatypes::value::ValueRef::String(values[name].as_str())), - ) - .expect("dense primary key encoding should match fixture primary key fields") -} - -fn tag_value(col: &ColumnConfig, series: usize) -> String { - match col - .distribution - .as_ref() - .expect("tag distribution is required") - { - Distribution::Cardinality { values, prefix } => { - format!("{}{}", prefix, series % values.get()) - } - _ => panic!("tag column {} requires cardinality distribution", col.name), - } -} - -fn series_for_row(layout: &LayoutConfig, sst_idx: usize, base_row: usize, row: usize) -> usize { - match layout.series_layout.as_str() { - "round_robin" | "timestamp_major" => (base_row + row) % layout.series_count.get(), - "per_sst" => sst_idx % layout.series_count.get(), - other => panic!("unsupported series_layout {other}"), - } -} - -fn timestamp_for_row(layout: &LayoutConfig, base_row: usize, row: usize) -> i64 { - let logical_row = base_row + row; - let timestamp_step = if layout.series_layout == "timestamp_major" { - logical_row / layout.series_count.get() - } else { - logical_row - }; - - layout.start_unix_nanos + (timestamp_step as i64 * layout.step_nanos) -} - -fn wave_value(min: f64, max: f64, row: usize) -> f64 { - let span = max - min; - let phase = (row % 1024) as f64 / 1023.0; - min + span * (0.5 - 0.5 * (std::f64::consts::TAU * phase).cos()) -} - -fn generate_record_batch( - table: &TableConfig, - metadata: &RegionMetadataRef, - layout: &LayoutConfig, - sst_idx: usize, - sequence: u64, -) -> RecordBatch { - let flat_schema = to_flat_sst_arrow_schema(metadata, &FlatSchemaOptions::default()); - let rows = layout.rows_per_sst; - let mut columns = Vec::new(); - let base_row = sst_idx * rows; - - for col in &table.columns { - match (col.semantic.as_str(), col.ty.to_ascii_uppercase().as_str()) { - ("tag", "STRING") => { - let mut b = StringDictionaryBuilder::::new(); - for row in 0..rows { - let series = series_for_row(layout, sst_idx, base_row, row); - let value = tag_value(col, series); - b.append_value(value); - } - columns.push(Arc::new(b.finish()) as ArrayRef); - } - ("field", "DOUBLE") => { - let (min, max) = match col.distribution.as_ref() { - Some(Distribution::DeterministicWave { min, max }) => (*min, *max), - _ => (0.0, 1.0), - }; - columns.push(Arc::new(Float64Array::from( - (0..rows) - .map(|r| wave_value(min, max, base_row + r)) - .collect::>(), - )) as ArrayRef); - } - ("field", "UINT64") => columns.push(Arc::new(UInt64Array::from( - (0..rows).map(|r| (base_row + r) as u64).collect::>(), - )) as ArrayRef), - ("timestamp", "TIMESTAMP(9)") => columns.push(Arc::new(TimestampNanosecondArray::from( - (0..rows) - .map(|r| timestamp_for_row(layout, base_row, r)) - .collect::>(), - )) as ArrayRef), - ("timestamp", "TIMESTAMP(3)") => { - columns.push(Arc::new(TimestampMillisecondArray::from( - (0..rows) - .map(|r| timestamp_for_row(layout, base_row, r) / 1_000_000) - .collect::>(), - )) as ArrayRef) - } - other => panic!("unsupported column combination {other:?}"), - } - } - - let mut pk_builder = BinaryDictionaryBuilder::::new(); - for row in 0..rows { - let series = series_for_row(layout, sst_idx, base_row, row); - let values = table - .columns - .iter() - .filter(|c| c.semantic == "tag") - .map(|c| (c.name.clone(), tag_value(c, series))) - .collect(); - pk_builder - .append(encode_dense_primary_key(table, &values)) - .expect("append generated dense primary key to Arrow dictionary builder"); - } - columns.push(Arc::new(pk_builder.finish())); - columns.push(Arc::new(UInt64Array::from_value(sequence, rows))); - columns.push(Arc::new(UInt8Array::from_value(OpType::Put as u8, rows))); - RecordBatch::try_new(flat_schema, columns) - .expect("generated fixture columns should match flat SST Arrow schema") -} - -fn file_meta_from_sst_info( - info: &SstInfo, - region_id: RegionId, - file_id: FileId, - sequence: u64, -) -> FileMeta { - FileMeta { - region_id, - file_id, - time_range: info.time_range, - level: 0, - file_size: info.file_size, - max_row_group_uncompressed_size: info.max_row_group_uncompressed_size, - available_indexes: Default::default(), - indexes: Default::default(), - index_file_size: 0, - index_version: 0, - num_rows: info.num_rows as u64, - num_row_groups: info.num_row_groups, - sequence: NonZeroU64::new(sequence), - partition_expr: None, - num_series: info.num_series, - primary_key_min: None, - primary_key_max: None, - } -} - -fn deterministic_file_id(seed: u64, index: usize) -> FileId { - let seed_hi = ((seed >> 16) & 0xffff) as u16; - let seed_lo = (seed & 0xffff) as u16; - FileId::parse_str(&format!( - "00000000-0000-{seed_hi:04x}-{seed_lo:04x}-{index:012x}" - )) - .expect("deterministic file id must be a valid UUID") -} - -fn remap_sst_file( - object_store_dir: &Path, - table_dir: &str, - region_id: RegionId, - old_file_id: FileId, - new_file_id: FileId, -) { - if old_file_id == new_file_id { - return; - } - let old_path = object_store_dir.join(mito2::sst::location::sst_file_path( - table_dir, - RegionFileId::new(region_id, old_file_id), - PathType::Bare, - )); - let new_path = object_store_dir.join(mito2::sst::location::sst_file_path( - table_dir, - RegionFileId::new(region_id, new_file_id), - PathType::Bare, - )); - if let Some(parent) = new_path.parent() { - fs::create_dir_all(parent).expect("failed to create remapped SST parent directory"); - } - fs::rename(&old_path, &new_path).unwrap_or_else(|err| { - panic!( - "failed to remap SST file {} -> {}: {err}", - old_path.display(), - new_path.display() - ) - }); -} - -fn case_name_from_path(path: &Path) -> String { - path.parent() - .and_then(Path::file_name) - .and_then(|name| name.to_str()) - .unwrap_or("query_perf_case") - .to_string() -} +#[path = "query_perf_fixture/mod.rs"] +mod query_perf_fixture; #[tokio::main] async fn main() { - let args = Args::parse(); - let case_text = fs::read_to_string(&args.case).expect("failed to read query perf case TOML"); - let case: CaseFile = toml::from_str(&case_text).expect("failed to parse query perf case TOML"); - let case_name = case_name_from_path(&args.case); - let scenario = case.scenario.direct_readable_sst(); - if scenario.tables.is_empty() || scenario.layout.regions != 1 { - panic!("fixture generator supports one or more tables and exactly one region per table"); - } - if scenario.layout.time_range_layout != "non_overlapping_per_sst" { - panic!("MVP supports time_range_layout=non_overlapping_per_sst"); - } - if scenario.layout.series_layout == "timestamp_major" - && !scenario - .layout - .rows_per_sst - .is_multiple_of(scenario.layout.series_count.get()) - { - panic!( - "series_layout=timestamp_major requires rows_per_sst to be divisible by series_count" - ); - } - if scenario.layout.sst_count > 1000 && !args.allow_large { - panic!("sst_count exceeds 1000; pass --allow-large"); - } - let seed = scenario.seed.unwrap_or(0); - let table_index = match args.table.as_deref() { - Some(name) => scenario - .tables - .iter() - .position(|table| table.name == name) - .unwrap_or_else(|| panic!("--table {name} was not found in case {case_name}")), - None if scenario.tables.len() == 1 => 0, - None => panic!( - "case {} contains {} tables; pass --table to choose one", - case_name, - scenario.tables.len() - ), - }; - let table = &scenario.tables[table_index]; - if table.engine != "mito" { - panic!("MVP supports engine=mito"); - } - let region_id = RegionId::from(args.region_id); - let table_dir = args - .table_dir - .unwrap_or_else(|| format!("data/{}/{}/", table.database, table.name)); - if Path::new(&table_dir).is_absolute() { - panic!("--table-dir must be relative"); - } - if table_dir.trim_end_matches('/').rsplit('/').next() - == Some(region_name(region_id.table_id(), region_id.region_sequence()).as_str()) - { - panic!("--table-dir must be table-level directory"); - } - let region_dir = - mito2::sst::location::region_dir_from_table_dir(&table_dir, region_id, PathType::Bare); - let format = match table.sst_format.as_deref().unwrap_or("flat") { - "flat" => FormatType::Flat, - "primary_key" | "primary-key" => FormatType::PrimaryKey, - other => panic!("unsupported sst_format {other}"), - }; - println!( - "query_perf_fixture case={} scenario={} table={}.{} ssts={} rows_per_sst={} format={format:?}", - case_name, - case.scenario.kind(), - table.database, - table.name, - scenario.layout.sst_count, - scenario.layout.rows_per_sst - ); - println!( - "out_dir={} table_dir={} region_dir={}", - args.out_dir.display(), - table_dir, - region_dir - ); - if args.dry_run { - return; - } - - let obj_store_dir = args.out_dir.join("object-store"); - let manifest_dir = args.out_dir.join("manifest"); - fs::create_dir_all(&obj_store_dir).expect("failed to create fixture object-store directory"); - fs::create_dir_all(&manifest_dir).expect("failed to create fixture manifest directory"); - let ostorage = ObjectStore::new(FsBuilder::default().root(&obj_store_dir.to_string_lossy())) - .expect("failed to create filesystem object store for fixture output") - .finish(); - let metadata: RegionMetadataRef = Arc::new(build_region_metadata(table, region_id)); - let mut files = HashMap::with_capacity(scenario.layout.sst_count); - let mut next_file_index = 1; - for i in 0..scenario.layout.sst_count { - let sequence = 1000 + i as u64; - let batch = generate_record_batch(table, &metadata, &scenario.layout, i, sequence); - let source = - FlatSource::new_iter(batch.schema(), Box::new(vec![batch].into_iter().map(Ok))); - let mut metrics = Metrics::new(WriteType::Flush); - let mut writer = ParquetWriter::new_with_object_store( - ostorage.clone(), - metadata.clone(), - IndexConfig::default(), - NoopIndexBuilder, - FixedPathProvider { - table_dir: table_dir.clone(), - }, - &mut metrics, - ) - .await; - let opts = WriteOptions { - write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE, - row_group_size: scenario.layout.row_group_size, - max_file_size: None, - }; - let infos = match format { - FormatType::Flat => writer.write_all_flat(source, Some(sequence), &opts).await, - FormatType::PrimaryKey => { - writer - .write_all_flat_as_primary_key(source, Some(sequence), &opts) - .await - } - } - .expect("failed to write fixture SST through Mito parquet writer"); - for info in infos { - let file_id = deterministic_file_id( - seed.wrapping_add((table_index as u64) << 16), - next_file_index, - ); - next_file_index += 1; - remap_sst_file(&obj_store_dir, &table_dir, region_id, info.file_id, file_id); - files.insert( - file_id, - file_meta_from_sst_info(&info, region_id, file_id, sequence), - ); - } - } - let overall_seq = files - .values() - .filter_map(|m| m.sequence) - .map(|s| s.get()) - .max() - .unwrap_or(0); - let manifest = RegionManifest { - metadata, - files, - removed_files: RemovedFilesRecord::default(), - flushed_entry_id: overall_seq, - flushed_sequence: overall_seq, - committed_sequence: Some(overall_seq), - manifest_version: args.checkpoint_version, - truncated_entry_id: None, - compaction_time_window: None, - sst_format: format, - append_mode: table.append_mode, - }; - let checkpoint = RegionCheckpoint { - last_version: args.checkpoint_version, - compacted_actions: manifest.files.len(), - checkpoint: Some(manifest.clone()), - }; - let checkpoint_bytes = checkpoint - .encode() - .expect("failed to encode fixture region checkpoint"); - let checkpoint_path = manifest_dir.join(format!("{:020}.checkpoint", args.checkpoint_version)); - fs::write(&checkpoint_path, &checkpoint_bytes).expect("failed to write checkpoint file"); - fs::write(manifest_dir.join("_last_checkpoint"), serde_json::to_vec_pretty(&serde_json::json!({ "size": checkpoint_bytes.len(), "version": args.checkpoint_version, "checksum": null, "extend_metadata": {} })).expect("failed to serialize _last_checkpoint metadata")).expect("failed to write _last_checkpoint file"); - let files_jsonl_path = args.out_dir.join("files.jsonl"); - let mut jsonl = BufWriter::new( - fs::File::create(&files_jsonl_path).expect("failed to create fixture files.jsonl"), - ); - let mut entries: Vec<_> = manifest.files.iter().collect(); - entries.sort_by_key(|(id, _)| id.to_string()); - for (file_id, meta) in entries { - writeln!(jsonl, "{}", serde_json::to_string(&serde_json::json!({ "file_id": file_id.to_string(), "region_id": meta.region_id.as_u64(), "object_path": mito2::sst::location::sst_file_path(&table_dir, RegionFileId::new(meta.region_id, *file_id), PathType::Bare), "time_range_start": meta.time_range.0.value(), "time_range_end": meta.time_range.1.value(), "num_rows": meta.num_rows, "num_row_groups": meta.num_row_groups, "file_size": meta.file_size, "num_series": meta.num_series, "sequence": meta.sequence.map(|s| s.get()) })).expect("failed to serialize fixture SST metadata JSONL entry")).expect("failed to write fixture SST metadata JSONL entry"); - } - jsonl.flush().expect("failed to flush fixture files.jsonl"); - fs::write(args.out_dir.join("summary.json"), serde_json::to_vec_pretty(&serde_json::json!({ "case": case_name, "seed": seed, "table_index": table_index, "table": table.name, "database": table.database, "region_id": region_id.as_u64(), "table_dir": table_dir, "region_dir": region_dir, "sst_format": format!("{format:?}"), "sst_count": scenario.layout.sst_count, "rows_per_sst": scenario.layout.rows_per_sst, "row_group_size": scenario.layout.row_group_size, "total_rows": scenario.layout.sst_count * scenario.layout.rows_per_sst, "checkpoint_path": checkpoint_path, "files_jsonl_path": files_jsonl_path, "readback_validated": false, "metadata_source": "synthetic" })).expect("failed to serialize fixture summary")).expect("failed to write fixture summary.json"); - println!("Done. wrote {} SST file entries", manifest.files.len()); + query_perf_fixture::run().await; } diff --git a/src/cmd/src/bin/query_perf_fixture/case.rs b/src/cmd/src/bin/query_perf_fixture/case.rs new file mode 100644 index 00000000000..7dce735d7b9 --- /dev/null +++ b/src/cmd/src/bin/query_perf_fixture/case.rs @@ -0,0 +1,470 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::num::NonZeroUsize; + +use clap::ValueEnum; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct CaseFile { + pub(super) scenario: Scenario, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(tag = "kind")] +pub(super) enum Scenario { + #[serde(rename = "direct_readable_sst")] + DirectReadableSst(DirectReadableSstScenario), + #[serde(rename = "prom_remote_write_then_query")] + PromRemoteWriteThenQuery(PromRemoteWriteThenQueryScenario), +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct DirectReadableSstScenario { + #[serde(default)] + pub(super) seed: Option, + pub(super) tables: Vec, + pub(super) layout: LayoutConfig, +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct PromRemoteWriteThenQueryScenario { + #[serde(default)] + pub(super) queries: Vec, + pub(super) remote_write: PromRemoteWritePlan, +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct PromRemoteWritePlan { + #[serde(default = "default_database")] + pub(super) database: String, + #[serde(alias = "metric_name")] + pub(super) metric: String, + #[serde(default = "default_physical_table")] + pub(super) physical_table: String, + #[serde(default = "default_series_count")] + pub(super) series_count: u64, + #[serde(default = "default_samples_per_series")] + pub(super) samples_per_series: u64, + #[serde(default = "default_start_unix_millis")] + pub(super) start_unix_millis: i64, + #[serde(default = "default_step_millis")] + pub(super) step_millis: i64, + #[serde(default = "default_chunk_series_count", alias = "batch_size")] + pub(super) chunk_series_count: u64, + #[serde(default = "default_timeout_seconds")] + pub(super) timeout_seconds: u64, + #[serde(default)] + pub(super) sample_chunk_size: Option, + #[serde(default = "default_flush_every_sample_chunks")] + pub(super) flush_every_sample_chunks: u64, + #[serde(default = "default_visibility_timeout_seconds")] + pub(super) visibility_timeout_seconds: u64, + #[serde(default)] + pub(super) prom_store: PromStoreConfig, + #[serde(default)] + pub(super) value: ValueConfig, + #[serde(default)] + pub(super) storage: Option, + #[serde(default)] + pub(super) read_bench: Option, +} + +pub(super) fn default_database() -> String { + "public".to_string() +} +pub(super) fn default_physical_table() -> String { + "greptime_physical_table".to_string() +} +pub(super) fn default_series_count() -> u64 { + 8 +} +pub(super) fn default_samples_per_series() -> u64 { + 30 +} +pub(super) fn default_start_unix_millis() -> i64 { + 1_704_067_200_000 +} +pub(super) fn default_step_millis() -> i64 { + 15_000 +} +pub(super) fn default_chunk_series_count() -> u64 { + 8 +} +pub(super) fn default_timeout_seconds() -> u64 { + 60 +} +pub(super) fn default_flush_every_sample_chunks() -> u64 { + 1 +} +pub(super) fn default_visibility_timeout_seconds() -> u64 { + 30 +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct PromStoreConfig { + #[serde(default = "default_pending_rows_flush_interval")] + pub(super) pending_rows_flush_interval: String, + #[serde(default = "default_max_batch_rows")] + pub(super) max_batch_rows: u64, + #[serde(default = "default_max_concurrent_flushes")] + pub(super) max_concurrent_flushes: u64, + #[serde(default = "default_worker_channel_capacity")] + pub(super) worker_channel_capacity: u64, + #[serde(default = "default_max_inflight_requests")] + pub(super) max_inflight_requests: u64, +} + +impl Default for PromStoreConfig { + fn default() -> Self { + Self { + pending_rows_flush_interval: default_pending_rows_flush_interval(), + max_batch_rows: default_max_batch_rows(), + max_concurrent_flushes: default_max_concurrent_flushes(), + worker_channel_capacity: default_worker_channel_capacity(), + max_inflight_requests: default_max_inflight_requests(), + } + } +} + +pub(super) fn default_pending_rows_flush_interval() -> String { + "1s".to_string() +} +pub(super) fn default_max_batch_rows() -> u64 { + 100000 +} +pub(super) fn default_max_concurrent_flushes() -> u64 { + 256 +} +pub(super) fn default_worker_channel_capacity() -> u64 { + 65526 +} +pub(super) fn default_max_inflight_requests() -> u64 { + 3000 +} + +#[derive(Debug, Clone, Copy, Deserialize, Serialize, ValueEnum, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +#[value(rename_all = "snake_case")] +pub(super) enum ValuePattern { + Linear, + Constant, + Modulo, + Unique, + SeededRandom, + RunLength, + QuantizedSignal, + SignalWithSporadicStalls, + MixedSignalRepeated, +} + +impl std::fmt::Display for ValuePattern { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + serde_json::to_value(self).unwrap().as_str().unwrap() + ) + } +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct ValueConfig { + #[serde(default = "default_value_pattern")] + pub(super) pattern: ValuePattern, + #[serde(default)] + pub(super) base: f64, + #[serde(default = "default_value_step")] + pub(super) step: f64, + #[serde(default = "default_value_cardinality")] + pub(super) cardinality: u64, + #[serde(default)] + pub(super) seed: u64, + #[serde(default = "default_value_run_length")] + pub(super) run_length: u64, + #[serde(default = "default_value_stall_every")] + pub(super) stall_every: u64, + #[serde(default = "default_value_stall_length")] + pub(super) stall_length: u64, + #[serde(default = "default_value_mixed_every")] + pub(super) mixed_every: u64, +} +impl Default for ValueConfig { + fn default() -> Self { + Self { + pattern: default_value_pattern(), + base: 0.0, + step: default_value_step(), + cardinality: default_value_cardinality(), + seed: 0, + run_length: default_value_run_length(), + stall_every: default_value_stall_every(), + stall_length: default_value_stall_length(), + mixed_every: default_value_mixed_every(), + } + } +} +pub(super) fn default_value_pattern() -> ValuePattern { + ValuePattern::Linear +} +pub(super) fn default_value_step() -> f64 { + 0.125 +} +pub(super) fn default_value_cardinality() -> u64 { + 97 +} +pub(super) fn default_value_run_length() -> u64 { + 8 +} +pub(super) fn default_value_stall_every() -> u64 { + 100 +} +pub(super) fn default_value_stall_length() -> u64 { + 16 +} +pub(super) fn default_value_mixed_every() -> u64 { + 5 +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct StorageConfig { + #[serde(default = "default_true")] + pub(super) inspect: bool, + #[serde(default = "default_storage_column")] + pub(super) column: String, + #[serde(default)] + pub(super) root_suffix: Option, + #[serde(default)] + pub(super) include_metadata_files: bool, + #[serde(default = "default_min_files")] + pub(super) min_files: u64, + #[serde(default = "default_min_files")] + pub(super) min_files_with_column: u64, + #[serde(default)] + pub(super) require_encodings: Vec, + #[serde(default)] + pub(super) forbid_encodings: Vec, + #[serde(default)] + pub(super) max_total_file_size_bytes: Option, + #[serde(default)] + pub(super) max_column_compressed_size_bytes: Option, + #[serde(default)] + pub(super) max_column_uncompressed_size_bytes: Option, + #[serde(default)] + pub(super) max_candidate_total_file_size_regression_pct: Option, + #[serde(default)] + pub(super) max_candidate_column_compressed_size_regression_pct: Option, + #[serde(default)] + pub(super) max_candidate_column_uncompressed_size_regression_pct: Option, + #[serde(skip_deserializing, default)] + pub(super) planned_thresholds: Vec, +} + +#[derive(Debug, Serialize)] +pub(super) struct StorageThresholdPlan { + pub(super) threshold: String, + pub(super) status: String, + pub(super) value: serde_json::Value, +} + +impl StorageConfig { + pub(super) fn populate_planned_thresholds(&mut self) { + let mut planned = Vec::new(); + planned.push(StorageThresholdPlan::new("min_files", self.min_files)); + planned.push(StorageThresholdPlan::new( + "min_files_with_column", + self.min_files_with_column, + )); + if !self.require_encodings.is_empty() { + planned.push(StorageThresholdPlan::new( + "require_encodings", + &self.require_encodings, + )); + } + if !self.forbid_encodings.is_empty() { + planned.push(StorageThresholdPlan::new( + "forbid_encodings", + &self.forbid_encodings, + )); + } + macro_rules! push_optional { + ($name:literal, $value:expr) => { + if let Some(value) = $value { + planned.push(StorageThresholdPlan::new($name, value)); + } + }; + } + push_optional!("max_total_file_size_bytes", self.max_total_file_size_bytes); + push_optional!( + "max_column_compressed_size_bytes", + self.max_column_compressed_size_bytes + ); + push_optional!( + "max_column_uncompressed_size_bytes", + self.max_column_uncompressed_size_bytes + ); + push_optional!( + "max_candidate_total_file_size_regression_pct", + self.max_candidate_total_file_size_regression_pct + ); + push_optional!( + "max_candidate_column_compressed_size_regression_pct", + self.max_candidate_column_compressed_size_regression_pct + ); + push_optional!( + "max_candidate_column_uncompressed_size_regression_pct", + self.max_candidate_column_uncompressed_size_regression_pct + ); + self.planned_thresholds = planned; + } +} + +impl StorageThresholdPlan { + fn new(threshold: &str, value: T) -> Self { + Self { + threshold: threshold.to_string(), + status: "planned".to_string(), + value: serde_json::to_value(value).expect("storage threshold value must serialize"), + } + } +} +pub(super) fn default_true() -> bool { + true +} +pub(super) fn default_storage_column() -> String { + "greptime_value".to_string() +} +pub(super) fn default_min_files() -> u64 { + 1 +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub(super) struct ReadBenchConfig { + #[serde(default = "default_true")] + pub(super) enabled: bool, + #[serde(default = "default_true")] + pub(super) parquetbench: bool, + #[serde(default = "default_true")] + pub(super) scanbench: bool, + #[serde(default = "default_iterations")] + pub(super) iterations: u64, + #[serde(default)] + pub(super) projection: Vec, + #[serde(default = "default_parquet_reader")] + pub(super) parquet_reader: String, + #[serde(default = "default_scan_scanner")] + pub(super) scan_scanner: String, + #[serde(default = "default_parallelism")] + pub(super) parallelism: u64, + #[serde(default)] + pub(super) max_files: Option, + #[serde(flatten)] + pub(super) thresholds: HashMap, +} + +impl Default for ReadBenchConfig { + fn default() -> Self { + Self { + enabled: true, + parquetbench: true, + scanbench: true, + iterations: default_iterations(), + projection: vec![], + parquet_reader: default_parquet_reader(), + scan_scanner: default_scan_scanner(), + parallelism: default_parallelism(), + max_files: None, + thresholds: HashMap::new(), + } + } +} + +pub(super) fn default_iterations() -> u64 { + 7 +} +pub(super) fn default_parquet_reader() -> String { + "direct".to_string() +} +pub(super) fn default_scan_scanner() -> String { + "seq".to_string() +} +pub(super) fn default_parallelism() -> u64 { + 1 +} + +impl Scenario { + pub(super) fn kind(&self) -> &'static str { + match self { + Scenario::DirectReadableSst(_) => "direct_readable_sst", + Scenario::PromRemoteWriteThenQuery(_) => "prom_remote_write_then_query", + } + } + + pub(super) fn direct_readable_sst(&self) -> &DirectReadableSstScenario { + match self { + Scenario::DirectReadableSst(scenario) => scenario, + _ => panic!("scenario is not direct_readable_sst"), + } + } +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct TableConfig { + pub(super) database: String, + pub(super) name: String, + pub(super) engine: String, + #[serde(default)] + pub(super) append_mode: Option, + #[serde(default)] + pub(super) sst_format: Option, + pub(super) primary_key: Vec, + pub(super) time_index: String, + pub(super) columns: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct ColumnConfig { + pub(super) name: String, + #[serde(rename = "type")] + pub(super) ty: String, + pub(super) semantic: String, + pub(super) distribution: Option, +} + +#[derive(Debug, Deserialize, Serialize)] +#[serde(tag = "kind")] +pub(super) enum Distribution { + #[serde(rename = "cardinality")] + Cardinality { + values: NonZeroUsize, + prefix: String, + }, + #[serde(rename = "deterministic_wave")] + DeterministicWave { min: f64, max: f64 }, +} + +#[derive(Debug, Deserialize, Serialize)] +pub(super) struct LayoutConfig { + pub(super) regions: usize, + pub(super) sst_count: usize, + pub(super) rows_per_sst: usize, + pub(super) row_group_size: usize, + pub(super) series_count: NonZeroUsize, + pub(super) start_unix_nanos: i64, + pub(super) step_nanos: i64, + pub(super) time_range_layout: String, + pub(super) series_layout: String, +} diff --git a/src/cmd/src/bin/query_perf_fixture/direct_sst.rs b/src/cmd/src/bin/query_perf_fixture/direct_sst.rs new file mode 100644 index 00000000000..0f2e3b34876 --- /dev/null +++ b/src/cmd/src/bin/query_perf_fixture/direct_sst.rs @@ -0,0 +1,529 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::fs::{self}; +use std::io::{BufWriter, Write}; +use std::num::NonZeroU64; +use std::path::Path; +use std::sync::Arc; + +use api::v1::{OpType, SemanticType}; +use async_trait::async_trait; +use datatypes::arrow::array::{ + ArrayRef, BinaryDictionaryBuilder, Float64Array, RecordBatch, StringDictionaryBuilder, + TimestampMillisecondArray, TimestampNanosecondArray, UInt8Array, UInt64Array, +}; +use datatypes::arrow::datatypes::UInt32Type; +use datatypes::prelude::ConcreteDataType; +use datatypes::schema::{ColumnSchema, SkippingIndexOptions}; +use mito_codec::row_converter::{DensePrimaryKeyCodec, PrimaryKeyCodecExt, SortField}; +use mito2::access_layer::{FilePathProvider, Metrics, WriteType}; +use mito2::config::IndexConfig; +use mito2::manifest::action::{RegionCheckpoint, RegionManifest, RemovedFilesRecord}; +use mito2::read::FlatSource; +use mito2::sst::file::{FileMeta, RegionFileId}; +use mito2::sst::index::{Indexer, IndexerBuilder}; +use mito2::sst::parquet::writer::ParquetWriter; +use mito2::sst::parquet::{SstInfo, WriteOptions}; +use mito2::sst::{ + DEFAULT_WRITE_BUFFER_SIZE, FlatSchemaOptions, FormatType, to_flat_sst_arrow_schema, +}; +use object_store::ObjectStore; +use object_store::services::Fs as FsBuilder; +use store_api::codec::PrimaryKeyEncoding; +use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataRef}; +use store_api::path_utils::region_name; +use store_api::region_request::PathType; +use store_api::storage::{FileId, RegionId}; + +use crate::query_perf_fixture::DirectArgs; +use crate::query_perf_fixture::case::*; +use crate::query_perf_fixture::util::case_name_from_path; + +struct NoopIndexBuilder; + +#[async_trait] +impl IndexerBuilder for NoopIndexBuilder { + async fn build(&self, _file_id: FileId, _index_version: u64) -> Indexer { + Indexer::default() + } +} + +#[derive(Clone)] +struct FixedPathProvider { + table_dir: String, +} + +impl FilePathProvider for FixedPathProvider { + fn build_index_file_path(&self, file_id: RegionFileId) -> String { + mito2::sst::location::index_file_path_legacy(&self.table_dir, file_id, PathType::Bare) + } + fn build_index_file_path_with_version( + &self, + index_id: mito2::sst::file::RegionIndexId, + ) -> String { + mito2::sst::location::index_file_path(&self.table_dir, index_id, PathType::Bare) + } + fn build_sst_file_path(&self, file_id: RegionFileId) -> String { + mito2::sst::location::sst_file_path(&self.table_dir, file_id, PathType::Bare) + } +} + +fn semantic_type(s: &str) -> SemanticType { + match s.to_ascii_lowercase().as_str() { + "tag" => SemanticType::Tag, + "field" => SemanticType::Field, + "timestamp" => SemanticType::Timestamp, + other => panic!("unsupported semantic {other}"), + } +} + +fn concrete_type(ty: &str) -> ConcreteDataType { + match ty.to_ascii_uppercase().as_str() { + "STRING" => ConcreteDataType::string_datatype(), + "DOUBLE" => ConcreteDataType::float64_datatype(), + "UINT64" => ConcreteDataType::uint64_datatype(), + "TIMESTAMP(9)" => ConcreteDataType::timestamp_nanosecond_datatype(), + "TIMESTAMP(3)" => ConcreteDataType::timestamp_millisecond_datatype(), + other => panic!("unsupported column type {other}"), + } +} + +fn build_region_metadata(table: &TableConfig, region_id: RegionId) -> RegionMetadata { + let mut builder = store_api::metadata::RegionMetadataBuilder::new(region_id); + for (idx, col) in table.columns.iter().enumerate() { + let mut schema = ColumnSchema::new( + &col.name, + concrete_type(&col.ty), + col.semantic != "timestamp", + ); + if col.semantic == "tag" { + schema = schema.with_inverted_index(true); + } + if col.semantic == "tag" || col.semantic == "field" { + schema = schema + .with_skipping_options(SkippingIndexOptions { + granularity: 1, + ..Default::default() + }) + .expect("valid skipping index options for query perf fixture"); + } + if col.name == table.time_index { + schema = schema.with_time_index(true); + } + builder.push_column_metadata(ColumnMetadata { + column_schema: schema, + semantic_type: semantic_type(&col.semantic), + column_id: idx as u32, + }); + } + let pk_ids = table + .primary_key + .iter() + .map(|name| { + table + .columns + .iter() + .position(|c| c.name == *name) + .unwrap_or_else(|| panic!("primary key column {name} not found")) as u32 + }) + .collect(); + builder.primary_key(pk_ids); + builder.primary_key_encoding(PrimaryKeyEncoding::Dense); + builder + .build() + .expect("region metadata should be valid for fixture table config") +} + +fn encode_dense_primary_key(table: &TableConfig, values: &HashMap) -> Vec { + let fields = table + .primary_key + .iter() + .enumerate() + .map(|(idx, _)| { + ( + idx as u32, + SortField::new(ConcreteDataType::string_datatype()), + ) + }) + .collect(); + let converter = DensePrimaryKeyCodec::with_fields(fields); + converter + .encode( + table + .primary_key + .iter() + .map(|name| datatypes::value::ValueRef::String(values[name].as_str())), + ) + .expect("dense primary key encoding should match fixture primary key fields") +} + +fn tag_value(col: &ColumnConfig, series: usize) -> String { + match col + .distribution + .as_ref() + .expect("tag distribution is required") + { + Distribution::Cardinality { values, prefix } => { + format!("{}{}", prefix, series % values.get()) + } + _ => panic!("tag column {} requires cardinality distribution", col.name), + } +} + +fn series_for_row(layout: &LayoutConfig, sst_idx: usize, base_row: usize, row: usize) -> usize { + match layout.series_layout.as_str() { + "round_robin" | "timestamp_major" => (base_row + row) % layout.series_count.get(), + "per_sst" => sst_idx % layout.series_count.get(), + other => panic!("unsupported series_layout {other}"), + } +} + +fn timestamp_for_row(layout: &LayoutConfig, base_row: usize, row: usize) -> i64 { + let logical_row = base_row + row; + let timestamp_step = if layout.series_layout == "timestamp_major" { + logical_row / layout.series_count.get() + } else { + logical_row + }; + + layout.start_unix_nanos + (timestamp_step as i64 * layout.step_nanos) +} + +fn wave_value(min: f64, max: f64, row: usize) -> f64 { + let span = max - min; + let phase = (row % 1024) as f64 / 1023.0; + min + span * (0.5 - 0.5 * (std::f64::consts::TAU * phase).cos()) +} + +fn generate_record_batch( + table: &TableConfig, + metadata: &RegionMetadataRef, + layout: &LayoutConfig, + sst_idx: usize, + sequence: u64, +) -> RecordBatch { + let flat_schema = to_flat_sst_arrow_schema(metadata, &FlatSchemaOptions::default()); + let rows = layout.rows_per_sst; + let mut columns = Vec::new(); + let base_row = sst_idx * rows; + + for col in &table.columns { + match (col.semantic.as_str(), col.ty.to_ascii_uppercase().as_str()) { + ("tag", "STRING") => { + let mut b = StringDictionaryBuilder::::new(); + for row in 0..rows { + let series = series_for_row(layout, sst_idx, base_row, row); + let value = tag_value(col, series); + b.append_value(value); + } + columns.push(Arc::new(b.finish()) as ArrayRef); + } + ("field", "DOUBLE") => { + let (min, max) = match col.distribution.as_ref() { + Some(Distribution::DeterministicWave { min, max }) => (*min, *max), + _ => (0.0, 1.0), + }; + columns.push(Arc::new(Float64Array::from( + (0..rows) + .map(|r| wave_value(min, max, base_row + r)) + .collect::>(), + )) as ArrayRef); + } + ("field", "UINT64") => columns.push(Arc::new(UInt64Array::from( + (0..rows).map(|r| (base_row + r) as u64).collect::>(), + )) as ArrayRef), + ("timestamp", "TIMESTAMP(9)") => columns.push(Arc::new(TimestampNanosecondArray::from( + (0..rows) + .map(|r| timestamp_for_row(layout, base_row, r)) + .collect::>(), + )) as ArrayRef), + ("timestamp", "TIMESTAMP(3)") => { + columns.push(Arc::new(TimestampMillisecondArray::from( + (0..rows) + .map(|r| timestamp_for_row(layout, base_row, r) / 1_000_000) + .collect::>(), + )) as ArrayRef) + } + other => panic!("unsupported column combination {other:?}"), + } + } + + let mut pk_builder = BinaryDictionaryBuilder::::new(); + for row in 0..rows { + let series = series_for_row(layout, sst_idx, base_row, row); + let values = table + .columns + .iter() + .filter(|c| c.semantic == "tag") + .map(|c| (c.name.clone(), tag_value(c, series))) + .collect(); + pk_builder + .append(encode_dense_primary_key(table, &values)) + .expect("append generated dense primary key to Arrow dictionary builder"); + } + columns.push(Arc::new(pk_builder.finish())); + columns.push(Arc::new(UInt64Array::from_value(sequence, rows))); + columns.push(Arc::new(UInt8Array::from_value(OpType::Put as u8, rows))); + RecordBatch::try_new(flat_schema, columns) + .expect("generated fixture columns should match flat SST Arrow schema") +} + +fn file_meta_from_sst_info( + info: &SstInfo, + region_id: RegionId, + file_id: FileId, + sequence: u64, +) -> FileMeta { + FileMeta { + region_id, + file_id, + time_range: info.time_range, + level: 0, + file_size: info.file_size, + max_row_group_uncompressed_size: info.max_row_group_uncompressed_size, + available_indexes: Default::default(), + indexes: Default::default(), + index_file_size: 0, + index_version: 0, + num_rows: info.num_rows as u64, + num_row_groups: info.num_row_groups, + sequence: NonZeroU64::new(sequence), + partition_expr: None, + num_series: info.num_series, + primary_key_min: None, + primary_key_max: None, + } +} + +fn deterministic_file_id(seed: u64, index: usize) -> FileId { + let seed_hi = ((seed >> 16) & 0xffff) as u16; + let seed_lo = (seed & 0xffff) as u16; + FileId::parse_str(&format!( + "00000000-0000-{seed_hi:04x}-{seed_lo:04x}-{index:012x}" + )) + .expect("deterministic file id must be a valid UUID") +} + +fn remap_sst_file( + object_store_dir: &Path, + table_dir: &str, + region_id: RegionId, + old_file_id: FileId, + new_file_id: FileId, +) { + if old_file_id == new_file_id { + return; + } + let old_path = object_store_dir.join(mito2::sst::location::sst_file_path( + table_dir, + RegionFileId::new(region_id, old_file_id), + PathType::Bare, + )); + let new_path = object_store_dir.join(mito2::sst::location::sst_file_path( + table_dir, + RegionFileId::new(region_id, new_file_id), + PathType::Bare, + )); + if let Some(parent) = new_path.parent() { + fs::create_dir_all(parent).expect("failed to create remapped SST parent directory"); + } + fs::rename(&old_path, &new_path).unwrap_or_else(|err| { + panic!( + "failed to remap SST file {} -> {}: {err}", + old_path.display(), + new_path.display() + ) + }); +} +pub(super) async fn run_direct_sst(args: DirectArgs) { + let case_path = args.case.as_ref().expect("--case is required"); + let out_dir = args.out_dir.as_ref().expect("--out-dir is required"); + let case_text = fs::read_to_string(case_path).expect("failed to read query perf case TOML"); + let case: CaseFile = toml::from_str(&case_text).expect("failed to parse query perf case TOML"); + let case_name = case_name_from_path(case_path); + let scenario = case.scenario.direct_readable_sst(); + if scenario.tables.is_empty() || scenario.layout.regions != 1 { + panic!("fixture generator supports one or more tables and exactly one region per table"); + } + if scenario.layout.time_range_layout != "non_overlapping_per_sst" { + panic!("MVP supports time_range_layout=non_overlapping_per_sst"); + } + if scenario.layout.series_layout == "timestamp_major" + && !scenario + .layout + .rows_per_sst + .is_multiple_of(scenario.layout.series_count.get()) + { + panic!( + "series_layout=timestamp_major requires rows_per_sst to be divisible by series_count" + ); + } + if scenario.layout.sst_count > 1000 && !args.allow_large { + panic!("sst_count exceeds 1000; pass --allow-large"); + } + let seed = scenario.seed.unwrap_or(0); + let table_index = match args.table.as_deref() { + Some(name) => scenario + .tables + .iter() + .position(|table| table.name == name) + .unwrap_or_else(|| panic!("--table {name} was not found in case {case_name}")), + None if scenario.tables.len() == 1 => 0, + None => panic!( + "case {} contains {} tables; pass --table to choose one", + case_name, + scenario.tables.len() + ), + }; + let table = &scenario.tables[table_index]; + if table.engine != "mito" { + panic!("MVP supports engine=mito"); + } + let region_id = RegionId::from(args.region_id); + let table_dir = args + .table_dir + .unwrap_or_else(|| format!("data/{}/{}/", table.database, table.name)); + if Path::new(&table_dir).is_absolute() { + panic!("--table-dir must be relative"); + } + if table_dir.trim_end_matches('/').rsplit('/').next() + == Some(region_name(region_id.table_id(), region_id.region_sequence()).as_str()) + { + panic!("--table-dir must be table-level directory"); + } + let region_dir = + mito2::sst::location::region_dir_from_table_dir(&table_dir, region_id, PathType::Bare); + let format = match table.sst_format.as_deref().unwrap_or("flat") { + "flat" => FormatType::Flat, + "primary_key" | "primary-key" => FormatType::PrimaryKey, + other => panic!("unsupported sst_format {other}"), + }; + println!( + "query_perf_fixture case={} scenario={} table={}.{} ssts={} rows_per_sst={} format={format:?}", + case_name, + case.scenario.kind(), + table.database, + table.name, + scenario.layout.sst_count, + scenario.layout.rows_per_sst + ); + println!( + "out_dir={} table_dir={} region_dir={}", + out_dir.display(), + table_dir, + region_dir + ); + if args.dry_run { + return; + } + + let obj_store_dir = out_dir.join("object-store"); + let manifest_dir = out_dir.join("manifest"); + fs::create_dir_all(&obj_store_dir).expect("failed to create fixture object-store directory"); + fs::create_dir_all(&manifest_dir).expect("failed to create fixture manifest directory"); + let ostorage = ObjectStore::new(FsBuilder::default().root(&obj_store_dir.to_string_lossy())) + .expect("failed to create filesystem object store for fixture output") + .finish(); + let metadata: RegionMetadataRef = Arc::new(build_region_metadata(table, region_id)); + let mut files = HashMap::with_capacity(scenario.layout.sst_count); + let mut next_file_index = 1; + for i in 0..scenario.layout.sst_count { + let sequence = 1000 + i as u64; + let batch = generate_record_batch(table, &metadata, &scenario.layout, i, sequence); + let source = + FlatSource::new_iter(batch.schema(), Box::new(vec![batch].into_iter().map(Ok))); + let mut metrics = Metrics::new(WriteType::Flush); + let mut writer = ParquetWriter::new_with_object_store( + ostorage.clone(), + metadata.clone(), + IndexConfig::default(), + NoopIndexBuilder, + FixedPathProvider { + table_dir: table_dir.clone(), + }, + &mut metrics, + ) + .await; + let opts = WriteOptions { + write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE, + row_group_size: scenario.layout.row_group_size, + max_file_size: None, + }; + let infos = match format { + FormatType::Flat => writer.write_all_flat(source, Some(sequence), &opts).await, + FormatType::PrimaryKey => { + writer + .write_all_flat_as_primary_key(source, Some(sequence), &opts) + .await + } + } + .expect("failed to write fixture SST through Mito parquet writer"); + for info in infos { + let file_id = deterministic_file_id( + seed.wrapping_add((table_index as u64) << 16), + next_file_index, + ); + next_file_index += 1; + remap_sst_file(&obj_store_dir, &table_dir, region_id, info.file_id, file_id); + files.insert( + file_id, + file_meta_from_sst_info(&info, region_id, file_id, sequence), + ); + } + } + let overall_seq = files + .values() + .filter_map(|m| m.sequence) + .map(|s| s.get()) + .max() + .unwrap_or(0); + let manifest = RegionManifest { + metadata, + files, + removed_files: RemovedFilesRecord::default(), + flushed_entry_id: overall_seq, + flushed_sequence: overall_seq, + committed_sequence: Some(overall_seq), + manifest_version: args.checkpoint_version, + truncated_entry_id: None, + compaction_time_window: None, + sst_format: format, + append_mode: table.append_mode, + }; + let checkpoint = RegionCheckpoint { + last_version: args.checkpoint_version, + compacted_actions: manifest.files.len(), + checkpoint: Some(manifest.clone()), + }; + let checkpoint_bytes = checkpoint + .encode() + .expect("failed to encode fixture region checkpoint"); + let checkpoint_path = manifest_dir.join(format!("{:020}.checkpoint", args.checkpoint_version)); + fs::write(&checkpoint_path, &checkpoint_bytes).expect("failed to write checkpoint file"); + fs::write(manifest_dir.join("_last_checkpoint"), serde_json::to_vec_pretty(&serde_json::json!({ "size": checkpoint_bytes.len(), "version": args.checkpoint_version, "checksum": null, "extend_metadata": {} })).expect("failed to serialize _last_checkpoint metadata")).expect("failed to write _last_checkpoint file"); + let files_jsonl_path = out_dir.join("files.jsonl"); + let mut jsonl = BufWriter::new( + fs::File::create(&files_jsonl_path).expect("failed to create fixture files.jsonl"), + ); + let mut entries: Vec<_> = manifest.files.iter().collect(); + entries.sort_by_key(|(id, _)| id.to_string()); + for (file_id, meta) in entries { + writeln!(jsonl, "{}", serde_json::to_string(&serde_json::json!({ "file_id": file_id.to_string(), "region_id": meta.region_id.as_u64(), "object_path": mito2::sst::location::sst_file_path(&table_dir, RegionFileId::new(meta.region_id, *file_id), PathType::Bare), "time_range_start": meta.time_range.0.value(), "time_range_end": meta.time_range.1.value(), "num_rows": meta.num_rows, "num_row_groups": meta.num_row_groups, "file_size": meta.file_size, "num_series": meta.num_series, "sequence": meta.sequence.map(|s| s.get()) })).expect("failed to serialize fixture SST metadata JSONL entry")).expect("failed to write fixture SST metadata JSONL entry"); + } + jsonl.flush().expect("failed to flush fixture files.jsonl"); + fs::write(out_dir.join("summary.json"), serde_json::to_vec_pretty(&serde_json::json!({ "case": case_name, "seed": seed, "table_index": table_index, "table": table.name, "database": table.database, "region_id": region_id.as_u64(), "table_dir": table_dir, "region_dir": region_dir, "sst_format": format!("{format:?}"), "sst_count": scenario.layout.sst_count, "rows_per_sst": scenario.layout.rows_per_sst, "row_group_size": scenario.layout.row_group_size, "total_rows": scenario.layout.sst_count * scenario.layout.rows_per_sst, "checkpoint_path": checkpoint_path, "files_jsonl_path": files_jsonl_path, "readback_validated": false, "metadata_source": "synthetic" })).expect("failed to serialize fixture summary")).expect("failed to write fixture summary.json"); + println!("Done. wrote {} SST file entries", manifest.files.len()); +} diff --git a/src/cmd/src/bin/query_perf_fixture/inspect_footer.rs b/src/cmd/src/bin/query_perf_fixture/inspect_footer.rs new file mode 100644 index 00000000000..a52005c4e30 --- /dev/null +++ b/src/cmd/src/bin/query_perf_fixture/inspect_footer.rs @@ -0,0 +1,181 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::BTreeSet; +use std::fs::{self, File}; +use std::path::{Path, PathBuf}; + +use clap::Args as ClapArgs; +use parquet::file::reader::{FileReader, SerializedFileReader}; +use serde::Serialize; + +#[derive(Debug, ClapArgs)] +pub(super) struct InspectFooterArgs { + #[arg(long)] + root: PathBuf, + #[arg(long, default_value = "greptime_value")] + column: String, + #[arg(long)] + include_metadata_files: bool, +} + +#[derive(Debug, Serialize)] +struct FooterReport { + root: String, + summary: FooterSummary, + files: Vec, +} +#[derive(Debug, Default, Serialize)] +struct FooterSummary { + column: String, + file_count: usize, + files_with_column: usize, + total_file_size: u64, + total_rows: i64, + column_compressed_size: i64, + column_uncompressed_size: i64, + unique_encodings: Vec, +} +#[derive(Debug, Serialize)] +struct FooterFileReport { + path: String, + relative_path: String, + file_size: u64, + num_rows: i64, + num_row_groups: usize, + columns: Vec, +} +#[derive(Debug, Serialize)] +struct FooterColumnChunkReport { + row_group_index: usize, + column_index: usize, + column_path: String, + encodings: Vec, + compression: String, + compressed_size: i64, + uncompressed_size: i64, + num_values: i64, +} + +pub(super) fn run_inspect_footer( + args: InspectFooterArgs, +) -> Result<(), Box> { + let files = collect_parquet_files(&args.root, args.include_metadata_files)?; + let reports = files + .iter() + .map(|p| inspect_file(&args.root, p, &args.column)) + .collect::, _>>()?; + println!( + "{}", + serde_json::to_string_pretty(&FooterReport { + root: args.root.display().to_string(), + summary: summarize_footer(&args.column, &reports), + files: reports + })? + ); + Ok(()) +} +fn collect_parquet_files( + root: &Path, + include_metadata_files: bool, +) -> Result, Box> { + let mut files = Vec::new(); + collect_parquet_files_inner(root, root, include_metadata_files, &mut files)?; + files.sort(); + Ok(files) +} +fn collect_parquet_files_inner( + root: &Path, + path: &Path, + include_metadata_files: bool, + files: &mut Vec, +) -> Result<(), Box> { + if path.is_dir() { + for entry in fs::read_dir(path)? { + collect_parquet_files_inner(root, &entry?.path(), include_metadata_files, files)?; + } + } else if path.extension().is_some_and(|ext| ext == "parquet") + && (include_metadata_files + || !path + .strip_prefix(root) + .unwrap_or(path) + .components() + .any(|c| c.as_os_str() == "metadata")) + { + files.push(path.to_path_buf()); + } + Ok(()) +} +fn inspect_file( + root: &Path, + path: &Path, + column: &str, +) -> Result> { + let file_size = fs::metadata(path)?.len(); + let reader = SerializedFileReader::new(File::open(path)?)?; + let metadata = reader.metadata().file_metadata(); + let row_groups = reader.metadata().row_groups(); + let mut columns = Vec::new(); + for (row_group_index, rg) in row_groups.iter().enumerate() { + for (column_index, chunk) in rg.columns().iter().enumerate() { + let column_path = chunk.column_path().string(); + if column_path == column { + columns.push(FooterColumnChunkReport { + row_group_index, + column_index, + column_path, + encodings: chunk.encodings().map(|e| format!("{e:?}")).collect(), + compression: format!("{:?}", chunk.compression()), + compressed_size: chunk.compressed_size(), + uncompressed_size: chunk.uncompressed_size(), + num_values: chunk.num_values(), + }); + } + } + } + Ok(FooterFileReport { + path: path.display().to_string(), + relative_path: path + .strip_prefix(root) + .unwrap_or(path) + .display() + .to_string(), + file_size, + num_rows: metadata.num_rows(), + num_row_groups: row_groups.len(), + columns, + }) +} +fn summarize_footer(column: &str, files: &[FooterFileReport]) -> FooterSummary { + let mut encodings = BTreeSet::new(); + let mut s = FooterSummary { + column: column.to_string(), + file_count: files.len(), + ..Default::default() + }; + for file in files { + s.total_file_size += file.file_size; + s.total_rows += file.num_rows; + if !file.columns.is_empty() { + s.files_with_column += 1; + } + for c in &file.columns { + s.column_compressed_size += c.compressed_size; + s.column_uncompressed_size += c.uncompressed_size; + encodings.extend(c.encodings.iter().cloned()); + } + } + s.unique_encodings = encodings.into_iter().collect(); + s +} diff --git a/src/cmd/src/bin/query_perf_fixture/mod.rs b/src/cmd/src/bin/query_perf_fixture/mod.rs new file mode 100644 index 00000000000..00cef327c78 --- /dev/null +++ b/src/cmd/src/bin/query_perf_fixture/mod.rs @@ -0,0 +1,133 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod case; +mod direct_sst; +mod inspect_footer; +mod prom_remote_write; +mod util; + +use std::fs; +use std::path::PathBuf; + +use case::*; +use clap::{Args as ClapArgs, Parser, Subcommand}; +use direct_sst::run_direct_sst; +use inspect_footer::{InspectFooterArgs, run_inspect_footer}; +use prom_remote_write::{PromRemoteWriteArgs, run_prom_remote_write}; +use serde_json::json; + +#[derive(Parser, Debug)] +#[command(name = "query_perf_fixture")] +#[command(about = "Query performance fixture helper")] +struct Args { + #[command(subcommand)] + command: Option, + + #[command(flatten)] + legacy: DirectArgs, +} + +#[derive(Subcommand, Debug)] +enum Command { + DirectSst(DirectArgs), + PromRemoteWrite(PromRemoteWriteArgs), + InspectFooter(InspectFooterArgs), + Plan(PlanArgs), +} + +#[derive(ClapArgs, Debug, Clone)] +struct DirectArgs { + /// TOML case file, for example tests/perf/query_cases/.../case.toml. + #[arg(long, value_name = "PATH")] + case: Option, + + /// Output directory (creates object-store/, manifest/, files.jsonl, summary.json). + #[arg(long, value_name = "DIR")] + out_dir: Option, + + /// Region ID to synthesize. Defaults to table id 1024, region 0. + #[arg(long, default_value = "4398046511104")] + region_id: u64, + + /// Table directory relative to object-store root. Defaults to data/{database}/{table}/. + #[arg(long, value_name = "DIR")] + table_dir: Option, + + /// Table name to generate when the case contains multiple [[tables]]. + #[arg(long, value_name = "NAME")] + table: Option, + + /// Manifest/checkpoint version. + #[arg(long, default_value = "1000000")] + checkpoint_version: u64, + + /// Safety flag when sst_count exceeds 1000. + #[arg(long)] + allow_large: bool, + + /// Print plan only. + #[arg(long)] + dry_run: bool, +} + +#[derive(ClapArgs, Debug)] +struct PlanArgs { + #[arg(long, value_name = "PATH")] + case: PathBuf, +} + +pub async fn run() { + let args = Args::parse(); + match args.command { + Some(Command::DirectSst(direct)) => run_direct_sst(direct).await, + Some(Command::PromRemoteWrite(rw)) => run_prom_remote_write(rw) + .await + .expect("prom remote write failed"), + Some(Command::InspectFooter(inspect)) => { + run_inspect_footer(inspect).expect("inspect footer failed") + } + Some(Command::Plan(plan)) => run_plan(plan).expect("plan failed"), + None => run_direct_sst(args.legacy).await, + } +} + +fn run_plan(args: PlanArgs) -> Result<(), Box> { + let case_text = fs::read_to_string(&args.case)?; + let mut case: CaseFile = toml::from_str(&case_text)?; + if let Scenario::PromRemoteWriteThenQuery(s) = &mut case.scenario + && let Some(storage) = &mut s.remote_write.storage + { + storage.populate_planned_thresholds(); + if storage.inspect { + let mut rb = s.remote_write.read_bench.clone().unwrap_or_default(); + if rb.enabled && rb.projection.is_empty() { + rb.projection = vec![storage.column.clone()]; + } + s.remote_write.read_bench = Some(rb); + } else if s + .remote_write + .read_bench + .as_ref() + .is_some_and(|rb| rb.enabled) + { + return Err("scenario.remote_write.read_bench requires scenario.remote_write.storage.inspect = true".into()); + } + } + println!( + "{}", + serde_json::to_string_pretty(&json!({"schema_version": 1, "scenario": case.scenario}))? + ); + Ok(()) +} diff --git a/src/cmd/src/bin/query_perf_fixture/prom_remote_write.rs b/src/cmd/src/bin/query_perf_fixture/prom_remote_write.rs new file mode 100644 index 00000000000..1f36b0c2d23 --- /dev/null +++ b/src/cmd/src/bin/query_perf_fixture/prom_remote_write.rs @@ -0,0 +1,333 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use clap::Args as ClapArgs; +use prost::Message; +use serde_json::json; +use servers::prom_store::snappy_compress; + +use crate::query_perf_fixture::case::ValuePattern; + +#[derive(Debug, ClapArgs)] +pub(super) struct PromRemoteWriteArgs { + #[arg(long, default_value = "http://127.0.0.1:4000/v1/prometheus/write")] + endpoint: String, + #[arg(long, default_value = "public")] + database: String, + #[arg(long)] + metric: String, + #[arg(long, default_value = "greptime_physical_table")] + physical_table: String, + #[arg(long, default_value_t = 8)] + series_count: u64, + #[arg(long, default_value_t = 30)] + samples_per_series: u64, + #[arg(long, default_value_t = 1_704_067_200_000)] + start_unix_millis: i64, + #[arg(long, default_value_t = 15_000)] + step_millis: i64, + #[arg(long, alias = "batch-size", default_value_t = 8)] + chunk_series_count: u64, + #[arg(long, default_value_t = 60)] + timeout_seconds: u64, + #[arg(long, default_value_t = ValuePattern::Linear)] + value_pattern: ValuePattern, + #[arg(long, default_value_t = 0.0)] + value_base: f64, + #[arg(long, default_value_t = 0.125)] + value_step: f64, + #[arg(long, default_value_t = 97)] + value_cardinality: u64, + #[arg(long, default_value_t = 0)] + value_seed: u64, + #[arg(long, default_value_t = 8)] + value_run_length: u64, + #[arg(long, default_value_t = 100)] + value_stall_every: u64, + #[arg(long, default_value_t = 16)] + value_stall_length: u64, + #[arg(long, default_value_t = 5)] + value_mixed_every: u64, + #[arg(long, default_value_t = 0)] + value_sample_offset: u64, + #[arg(long)] + value_total_samples_per_series: Option, +} + +pub(super) async fn run_prom_remote_write( + args: PromRemoteWriteArgs, +) -> Result<(), Box> { + use api::prom_store::remote::WriteRequest; + let started = std::time::Instant::now(); + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(args.timeout_seconds)) + .build()?; + let mut batches = 0_u64; + let mut rows = 0_u64; + let mut http_statuses = Vec::new(); + for first in (0..args.series_count).step_by(args.chunk_series_count.max(1) as usize) { + let last = (first + args.chunk_series_count.max(1)).min(args.series_count); + let timeseries = (first..last) + .map(|series_idx| make_prom_series(&args, series_idx)) + .collect::>(); + rows += timeseries + .iter() + .map(|ts| ts.samples.len() as u64) + .sum::(); + let body = snappy_compress( + &WriteRequest { + timeseries, + ..Default::default() + } + .encode_to_vec(), + )?; + let response = client + .post(&args.endpoint) + .header("Content-Encoding", "snappy") + .body(body) + .send() + .await?; + let status = response.status(); + http_statuses.push(status.as_u16()); + if !status.is_success() { + return Err(format!( + "remote-write failed with status {status}: {}", + response.text().await.unwrap_or_default() + ) + .into()); + } + batches += 1; + } + println!( + "{}", + json!({"status":"ok","endpoint":args.endpoint,"database":args.database,"metric":args.metric,"physical_table":args.physical_table,"series_count":args.series_count,"samples_per_series":args.samples_per_series,"rows":rows,"samples_written":rows,"batches":batches,"elapsed_seconds":started.elapsed().as_secs_f64(),"http_statuses":http_statuses,"value":{"pattern":args.value_pattern.to_string(),"base":args.value_base,"step":args.value_step,"cardinality":args.value_cardinality,"seed":args.value_seed,"run_length":args.value_run_length,"stall_every":args.value_stall_every,"stall_length":args.value_stall_length,"mixed_every":args.value_mixed_every,"sample_offset":args.value_sample_offset,"total_samples_per_series":args.value_total_samples_per_series.unwrap_or(args.samples_per_series)}}) + ); + Ok(()) +} + +fn make_prom_series( + args: &PromRemoteWriteArgs, + series_idx: u64, +) -> api::prom_store::remote::TimeSeries { + let mut labels = vec![ + prom_label("__name__", &args.metric), + prom_label("x_greptime_database", &args.database), + prom_label("x_greptime_physical_table", &args.physical_table), + prom_label("host", &format!("host{:04}", series_idx % 1024)), + prom_label("instance", &format!("instance{:06}", series_idx)), + ]; + labels.sort_unstable_by(|l, r| l.name.cmp(&r.name)); + let samples = (0..args.samples_per_series) + .map(|sample_idx| api::prom_store::remote::Sample { + value: deterministic_prom_value(args, series_idx, sample_idx), + timestamp: args.start_unix_millis + (sample_idx as i64 * args.step_millis), + }) + .collect(); + api::prom_store::remote::TimeSeries { + labels, + samples, + ..Default::default() + } +} +fn prom_label(name: &str, value: &str) -> api::prom_store::remote::Label { + api::prom_store::remote::Label { + name: name.to_string(), + value: value.to_string(), + } +} +fn deterministic_prom_value(args: &PromRemoteWriteArgs, series_idx: u64, sample_idx: u64) -> f64 { + let ordinal = series_idx + * args + .value_total_samples_per_series + .unwrap_or(args.samples_per_series) + + args.value_sample_offset + + sample_idx; + let local = args.value_sample_offset + sample_idx; + match args.value_pattern { + ValuePattern::Linear => { + args.value_base + (series_idx % 97) as f64 + local as f64 * args.value_step + } + ValuePattern::Constant => args.value_base, + ValuePattern::Modulo => { + args.value_base + (ordinal % args.value_cardinality.max(1)) as f64 * args.value_step + } + ValuePattern::Unique => args.value_base + ordinal as f64 * args.value_step, + ValuePattern::SeededRandom => { + args.value_base + + (splitmix64(ordinal ^ args.value_seed) % args.value_cardinality.max(1)) as f64 + * args.value_step + } + ValuePattern::RunLength => { + args.value_base + + ((ordinal / args.value_run_length.max(1)) % args.value_cardinality.max(1)) as f64 + * args.value_step + } + ValuePattern::QuantizedSignal => { + args.value_base + + (((series_idx % args.value_cardinality.max(1)) + + (local / args.value_run_length.max(1))) + % args.value_cardinality.max(1)) as f64 + * args.value_step + } + ValuePattern::SignalWithSporadicStalls => { + let every = args.value_stall_every.max(1); + let phase = local % every; + let effective = if phase < args.value_stall_length.min(every) { + local - phase + } else { + local + }; + args.value_base + (series_idx % 97) as f64 + effective as f64 * args.value_step + } + ValuePattern::MixedSignalRepeated => { + if local.is_multiple_of(args.value_mixed_every.max(1)) { + args.value_base + } else { + args.value_base + (series_idx % 97) as f64 + local as f64 * args.value_step + } + } + } +} +fn splitmix64(mut value: u64) -> u64 { + value = value.wrapping_add(0x9e3779b97f4a7c15); + let mut mixed = value; + mixed = (mixed ^ (mixed >> 30)).wrapping_mul(0xbf58476d1ce4e5b9); + mixed = (mixed ^ (mixed >> 27)).wrapping_mul(0x94d049bb133111eb); + mixed ^ (mixed >> 31) +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use super::*; + + fn args(pattern: ValuePattern) -> PromRemoteWriteArgs { + PromRemoteWriteArgs { + endpoint: "http://127.0.0.1:4000/v1/prometheus/write".to_string(), + database: "public".to_string(), + metric: "test_metric".to_string(), + physical_table: "greptime_physical_table".to_string(), + series_count: 2, + samples_per_series: 64, + start_unix_millis: 1_704_067_200_000, + step_millis: 60_000, + chunk_series_count: 2, + timeout_seconds: 60, + value_pattern: pattern, + value_base: 0.0, + value_step: 1.0, + value_cardinality: 1_024, + value_seed: 8_444, + value_run_length: 16, + value_stall_every: 100, + value_stall_length: 16, + value_mixed_every: 5, + value_sample_offset: 0, + value_total_samples_per_series: None, + } + } + + #[test] + fn seeded_random_is_deterministic_in_range_and_highly_distinct() { + let mut args = args(ValuePattern::SeededRandom); + args.samples_per_series = 4_096; + args.value_cardinality = 29_491_200; + + let values = (0..args.samples_per_series) + .map(|sample_idx| deterministic_prom_value(&args, 0, sample_idx)) + .collect::>(); + let repeated = (0..args.samples_per_series) + .map(|sample_idx| deterministic_prom_value(&args, 0, sample_idx)) + .collect::>(); + + assert_eq!(values, repeated); + assert!(values.iter().all(|value| { + (args.value_base..=args.value_base + (args.value_cardinality - 1) as f64) + .contains(value) + })); + assert!( + values + .iter() + .map(|value| value.to_bits()) + .collect::>() + .len() + > 4_000 + ); + } + + #[test] + fn run_length_values_have_exact_sixteen_sample_runs() { + let args = args(ValuePattern::RunLength); + + for sample_idx in 0..64 { + assert_eq!( + deterministic_prom_value(&args, 0, sample_idx), + (sample_idx / 16) as f64 + ); + } + } + + #[test] + fn mixed_every_five_repeats_only_the_expected_positions() { + let mut args = args(ValuePattern::MixedSignalRepeated); + args.value_step = 0.125; + + for sample_idx in 0..20 { + let expected = if sample_idx % 5 == 0 { + 0.0 + } else { + 1.0 + sample_idx as f64 * 0.125 + }; + assert_eq!(deterministic_prom_value(&args, 1, sample_idx), expected); + } + } + + #[test] + fn chunked_values_match_monolithic_values_through_offsets() { + let mut monolithic = args(ValuePattern::SeededRandom); + monolithic.samples_per_series = 14_400; + monolithic.value_total_samples_per_series = Some(14_400); + + for series_idx in 0..2 { + let expected = (0..monolithic.samples_per_series) + .map(|sample_idx| deterministic_prom_value(&monolithic, series_idx, sample_idx)) + .collect::>(); + for chunk_idx in 0..10 { + let mut chunk = args(ValuePattern::SeededRandom); + chunk.samples_per_series = 1_440; + chunk.value_sample_offset = chunk_idx * 1_440; + chunk.value_total_samples_per_series = Some(14_400); + for sample_idx in 0..chunk.samples_per_series { + assert_eq!( + deterministic_prom_value(&chunk, series_idx, sample_idx), + expected[(chunk.value_sample_offset + sample_idx) as usize] + ); + } + } + } + } + + #[test] + fn values_continue_across_series_boundaries() { + let mut args = args(ValuePattern::Unique); + args.samples_per_series = 16; + args.value_total_samples_per_series = Some(16); + args.value_base = 1.0; + args.value_step = 0.5; + + assert_eq!(deterministic_prom_value(&args, 0, 15), 8.5); + assert_eq!(deterministic_prom_value(&args, 1, 0), 9.0); + } +} diff --git a/src/cmd/src/bin/query_perf_fixture/util.rs b/src/cmd/src/bin/query_perf_fixture/util.rs new file mode 100644 index 00000000000..2c86baa6839 --- /dev/null +++ b/src/cmd/src/bin/query_perf_fixture/util.rs @@ -0,0 +1,23 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::path::Path; + +pub(super) fn case_name_from_path(path: &Path) -> String { + path.parent() + .and_then(Path::file_name) + .and_then(|name| name.to_str()) + .unwrap_or("query_perf_case") + .to_string() +} diff --git a/src/cmd/src/datanode.rs b/src/cmd/src/datanode.rs index 744624a5084..c33be6c6f33 100644 --- a/src/cmd/src/datanode.rs +++ b/src/cmd/src/datanode.rs @@ -15,11 +15,10 @@ pub mod builder; #[allow(clippy::print_stdout)] pub(crate) mod objbench; -#[cfg(feature = "dev-tools")] #[allow(clippy::print_stdout)] -mod parquetbench; +pub mod parquetbench; #[allow(clippy::print_stdout)] -mod scanbench; +pub mod scanbench; use std::path::Path; use std::time::Duration; @@ -40,7 +39,6 @@ use tracing_appender::non_blocking::WorkerGuard; use crate::App; use crate::datanode::builder::InstanceBuilder; use crate::datanode::objbench::ObjbenchCommand; -#[cfg(feature = "dev-tools")] use crate::datanode::parquetbench::ParquetbenchCommand; use crate::datanode::scanbench::ScanbenchCommand; use crate::error::{ @@ -115,7 +113,6 @@ impl Command { SubCommand::Start(cmd) => cmd.load_options(global_options), // Bench commands are standalone utilities and don't need to load DatanodeOptions. SubCommand::Objbench(_) | SubCommand::Scanbench(_) => Self::default_bench_options(), - #[cfg(feature = "dev-tools")] SubCommand::Parquetbench(_) => Self::default_bench_options(), } } @@ -141,7 +138,6 @@ pub enum SubCommand { /// Scan benchmark tool - benchmarks scanning a region directly from storage Scanbench(ScanbenchCommand), /// Benchmark scanning a single parquet SST. - #[cfg(feature = "dev-tools")] Parquetbench(ParquetbenchCommand), } @@ -160,7 +156,6 @@ impl SubCommand { cmd.run().await?; std::process::exit(0); } - #[cfg(feature = "dev-tools")] SubCommand::Parquetbench(cmd) => { cmd.run().await?; std::process::exit(0); diff --git a/src/mito2/src/engine/close_test.rs b/src/mito2/src/engine/close_test.rs index 4c06583b0b8..0092634ec74 100644 --- a/src/mito2/src/engine/close_test.rs +++ b/src/mito2/src/engine/close_test.rs @@ -12,11 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::sync::Arc; +use std::sync::atomic::Ordering; + +use common_base::Plugins; use store_api::region_engine::RegionEngine; use store_api::region_request::{RegionCloseRequest, RegionRequest}; use store_api::storage::RegionId; use crate::config::MitoConfig; +use crate::engine::flush_test::MockRegionHook; +use crate::engine::region_hook::RegionHookRef; use crate::test_util::{CreateRequestBuilder, TestEnv}; #[tokio::test] @@ -60,3 +66,40 @@ async fn test_engine_close_region_with_format(flat_format: bool) { .await .unwrap(); } + +#[tokio::test] +async fn test_region_hook_on_close() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::new().await; + + let hook = Arc::new(MockRegionHook::new()); + let plugins = Plugins::new(); + plugins.insert(hook.clone() as RegionHookRef); + + let engine = env + .create_engine_with_plugins(MitoConfig::default(), plugins) + .await; + + let region_id = RegionId::new(1, 1); + let request = CreateRequestBuilder::new().build(); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + + // Sanity: no lifecycle events before closing besides the create open. + assert_eq!(hook.opened_count.load(Ordering::Relaxed), 1); + assert_eq!(hook.closed_count.load(Ordering::Relaxed), 0); + assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 0); + + engine + .handle_request(region_id, RegionRequest::Close(RegionCloseRequest {})) + .await + .unwrap(); + + // Closing fires on_region_closed exactly once. + assert_eq!(hook.closed_count.load(Ordering::Relaxed), 1); + // Close must not be confused with drop. + assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 0); + assert_eq!(hook.files_removed_count.load(Ordering::Relaxed), 0); +} diff --git a/src/mito2/src/engine/drop_test.rs b/src/mito2/src/engine/drop_test.rs index a34a5d11727..90afd66ba32 100644 --- a/src/mito2/src/engine/drop_test.rs +++ b/src/mito2/src/engine/drop_test.rs @@ -13,9 +13,11 @@ // limitations under the License. use std::sync::Arc; +use std::sync::atomic::Ordering; use std::time::Duration; use api::v1::Rows; +use common_base::Plugins; use common_meta::key::SchemaMetadataManager; use common_meta::kv_backend::KvBackendRef; use object_store::util::join_path; @@ -25,7 +27,9 @@ use store_api::storage::RegionId; use crate::config::MitoConfig; use crate::engine::MitoEngine; +use crate::engine::flush_test::MockRegionHook; use crate::engine::listener::DropListener; +use crate::engine::region_hook::RegionHookRef; use crate::test_util::{ CreateRequestBuilder, TestEnv, build_rows_for_key, flush_region, put_rows, rows_schema, }; @@ -302,3 +306,75 @@ async fn test_engine_drop_region_for_custom_store_with_format(flat_format: bool) .unwrap() ); } + +#[tokio::test] +async fn test_region_hook_on_drop() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::with_prefix("drop_hook").await; + + let hook = Arc::new(MockRegionHook::new()); + let plugins = Plugins::new(); + plugins.insert(hook.clone() as RegionHookRef); + + let engine = env + .create_engine_with_plugins(MitoConfig::default(), plugins) + .await; + + let region_id = RegionId::new(1, 1); + env.get_schema_metadata_manager() + .register_region_table_info( + region_id.table_id(), + "test_table", + "test_catalog", + "test_schema", + None, + env.get_kv_backend(), + ) + .await; + + let request = CreateRequestBuilder::new().build(); + let column_schemas = rows_schema(&request); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + + // Put some rows so the region has active data (kept in memtable; we + // intentionally do not flush so the drop GC worker finds no parquet files + // and removes the directory immediately, avoiding the multi-minute GC wait). + let rows = Rows { + schema: column_schemas, + rows: build_rows_for_key("a", 0, 2, 0), + }; + put_rows(&engine, region_id, rows).await; + + // Sanity: nothing fired yet besides the open from create. + assert_eq!(hook.opened_count.load(Ordering::Relaxed), 1); + assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 0); + assert_eq!(hook.closed_count.load(Ordering::Relaxed), 0); + + // full drop (partial_drop = false) so the GC worker physically deletes the + // directory and fires on_region_files_removed. + engine + .handle_request( + region_id, + RegionRequest::Drop(RegionDropRequest { + fast_path: false, + force: false, + partial_drop: false, + }), + ) + .await + .unwrap(); + + // on_region_dropped fires inline while handling the drop request. + hook.wait_for_dropped().await; + assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 1); + // Drop must not be confused with close. + assert_eq!(hook.closed_count.load(Ordering::Relaxed), 0); + + // on_region_files_removed fires from the background GC worker once the + // directory is physically deleted. + hook.wait_for_files_removed().await; + assert_eq!(hook.files_removed_count.load(Ordering::Relaxed), 1); +} diff --git a/src/mito2/src/engine/flush_test.rs b/src/mito2/src/engine/flush_test.rs index 4870e8c2bbe..dc0063a7e63 100644 --- a/src/mito2/src/engine/flush_test.rs +++ b/src/mito2/src/engine/flush_test.rs @@ -895,24 +895,46 @@ async fn test_update_topic_latest_entry_id(factory: Option) { } #[derive(Debug)] -struct MockRegionHook { - sst_written_count: AtomicUsize, - manifest_updated_count: AtomicUsize, +pub(super) struct MockRegionHook { + pub(super) sst_written_count: AtomicUsize, + pub(super) manifest_updated_count: AtomicUsize, + pub(super) opened_count: AtomicUsize, + pub(super) closed_count: AtomicUsize, + pub(super) dropped_count: AtomicUsize, + pub(super) files_removed_count: AtomicUsize, notify: Notify, + dropped_notify: Notify, + files_removed_notify: Notify, } impl MockRegionHook { - fn new() -> Self { + pub(super) fn new() -> Self { Self { sst_written_count: AtomicUsize::new(0), manifest_updated_count: AtomicUsize::new(0), + opened_count: AtomicUsize::new(0), + closed_count: AtomicUsize::new(0), + dropped_count: AtomicUsize::new(0), + files_removed_count: AtomicUsize::new(0), notify: Notify::new(), + dropped_notify: Notify::new(), + files_removed_notify: Notify::new(), } } async fn wait_for_manifest_update(&self) { self.notify.notified().await; } + + /// Waits until `on_region_dropped` has fired at least once. + pub(super) async fn wait_for_dropped(&self) { + self.dropped_notify.notified().await; + } + + /// Waits until `on_region_files_removed` has fired at least once. + pub(super) async fn wait_for_files_removed(&self) { + self.files_removed_notify.notified().await; + } } #[async_trait] @@ -966,6 +988,35 @@ impl RegionHook for MockRegionHook { ); self.notify.notify_one(); } + + async fn on_region_opened(&self, region_id: RegionId, _region_metadata: &RegionMetadataRef) { + self.opened_count.fetch_add(1, Ordering::Relaxed); + common_telemetry::info!("MockRegionHook::on_region_opened: region={}", region_id); + } + + async fn on_region_closed(&self, region_id: RegionId, _region_metadata: &RegionMetadataRef) { + self.closed_count.fetch_add(1, Ordering::Relaxed); + common_telemetry::info!("MockRegionHook::on_region_closed: region={}", region_id); + } + + async fn on_region_dropped(&self, region_id: RegionId, _region_metadata: &RegionMetadataRef) { + self.dropped_count.fetch_add(1, Ordering::Relaxed); + common_telemetry::info!("MockRegionHook::on_region_dropped: region={}", region_id); + self.dropped_notify.notify_one(); + } + + async fn on_region_files_removed( + &self, + region_id: RegionId, + _region_metadata: &RegionMetadataRef, + ) { + self.files_removed_count.fetch_add(1, Ordering::Relaxed); + common_telemetry::info!( + "MockRegionHook::on_region_files_removed: region={}", + region_id + ); + self.files_removed_notify.notify_one(); + } } #[tokio::test] diff --git a/src/mito2/src/engine/open_test.rs b/src/mito2/src/engine/open_test.rs index cf4bfb1ff1d..ee0d9560093 100644 --- a/src/mito2/src/engine/open_test.rs +++ b/src/mito2/src/engine/open_test.rs @@ -32,6 +32,8 @@ use tokio::sync::oneshot; use crate::compaction::compactor::{OpenCompactionRegionRequest, open_compaction_region}; use crate::config::MitoConfig; +use crate::engine::flush_test::MockRegionHook; +use crate::engine::region_hook::RegionHookRef; use crate::error; use crate::region::opener::{PartitionExprFetcher, PartitionExprFetcherRef}; use crate::region::options::RegionOptions; @@ -805,3 +807,36 @@ async fn test_open_keeps_none_without_fetcher() { let meta = engine.get_region(region_id).unwrap().metadata(); assert!(meta.partition_expr.is_none()); } + +#[tokio::test] +async fn test_region_hook_on_open() { + common_telemetry::init_default_ut_logging(); + let mut env = TestEnv::with_prefix("open_hook").await; + + let hook = Arc::new(MockRegionHook::new()); + let plugins = Plugins::new(); + plugins.insert(hook.clone() as RegionHookRef); + + let engine = env + .create_engine_with_plugins(MitoConfig::default(), plugins) + .await; + + let region_id = RegionId::new(1, 1); + let request = CreateRequestBuilder::new().build(); + let table_dir = request.table_dir.clone(); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + + // Creating a region fires on_region_opened exactly once. + assert_eq!(hook.opened_count.load(Ordering::Relaxed), 1); + assert_eq!(hook.closed_count.load(Ordering::Relaxed), 0); + + // `reopen_region` closes then opens; the open must fire the hook again. + reopen_region(&engine, region_id, table_dir, false, Default::default()).await; + + assert_eq!(hook.opened_count.load(Ordering::Relaxed), 2); + assert_eq!(hook.closed_count.load(Ordering::Relaxed), 1); + assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 0); +} diff --git a/src/mito2/src/engine/region_hook.rs b/src/mito2/src/engine/region_hook.rs index d1b28f6ebae..9c5a7dfc95d 100644 --- a/src/mito2/src/engine/region_hook.rs +++ b/src/mito2/src/engine/region_hook.rs @@ -16,15 +16,24 @@ //! //! ## Design //! -//! The [`RegionHook`] trait provides two methods with clear separation of concerns: +//! The [`RegionHook`] trait observes region activity through two categories of +//! callbacks — manifest/file observation and region lifecycle: //! //! - [`on_sst_files_written`]: Fires when mito2 physically writes SST **data files**. //! Provides per-file [`SstInfo`] + [`FileMeta`]; metadata richness varies by path //! (see [`SstFileInfo`] and the coverage footnote). //! -//! - [`on_manifest_updated`]: Fires after **any** manifest write is successfully committed. -//! Receives the full [`RegionMetaActionList`] so consumers can inspect what changed -//! (file additions/removals, schema changes, truncation, partition expression changes, etc.). +//! - [`on_manifest_updated`]: Fires after a manifest write is committed to the **live** +//! (normal) manifest directory. Writes to the staging directory (enter staging, +//! operations during staging, the intermediate apply-staging edit) are suppressed — +//! their effects are accumulated and delivered in a single notification when the +//! staged actions are promoted to the live manifest. Receives the full +//! [`RegionMetaActionList`] so consumers can inspect what changed (file additions/ +//! removals, schema changes, truncation, partition expression changes, etc.). +//! +//! - [`on_region_opened`] / [`on_region_closed`] / [`on_region_dropped`] / [`on_region_files_removed`]: +//! Region **lifecycle** callbacks for open, close, logical drop, and physical file removal. +//! See [Region lifecycle](#region-lifecycle) below. //! //! Hook implementations are registered via the [`Plugins`](common_base::Plugins) system: //! ```ignore @@ -33,6 +42,12 @@ //! //! ## Coverage //! +//! Only manifest writes to the **normal** (live) manifest directory trigger +//! `on_manifest_updated`. Writes to the staging manifest directory (operations +//! that happen while the region is in staging mode) are intentionally suppressed: +//! their effects are accumulated and delivered in a single notification when +//! the staged actions are promoted to the live manifest via `exit_staging_on_success`. +//! //! | Scenario | `on_sst_files_written` | `on_manifest_updated` | //! |------------------------------|:----------------------:|:---------------------:| //! | Flush (memtable → SST) | ✅ Yes | ✅ Yes | @@ -40,23 +55,57 @@ //! | Remote compaction | ✅ (compactor node) ¹ | ✅ (compactor node) ¹ | //! | RegionEdit / bulk ingestion | ❌ (files pre-written) | ✅ Yes | //! | Copy region | ❌ (object-store copy) | ✅ Yes | -//! | Apply staging | ❌ (delegates to edit) | ✅ Yes ² | +//! | Apply staging (promote) | ❌ (delegates to edit) | ✅ Yes ² | //! | Alter (schema change) | ❌ (no SST files) | ✅ Yes | //! | Truncate | ❌ (removes files) | ✅ Yes | -//! | Enter staging | ❌ (no SST files) | ✅ Yes | +//! | Enter staging | ❌ (no SST files) | ❌ (staging dir) | +//! | Operations during staging | N/A | ❌ (staging dir) | //! | Async index build | ❌ (index files only) | ✅ Yes | //! //! ¹ Remote compaction runs on a dedicated compactor node via `open_compaction_region()`; //! pass plugins via `OpenCompactionRegionRequest` to enable hooks there. `sst_infos` is //! `#[serde(skip)]` over the wire, so each [`SstInfo`] is rebuilt from [`FileMeta`] with //! empty footer/index — see [`SstFileInfo`] for field-level detail. -//! ² Apply staging fires `on_manifest_updated` twice: once when the staging SST files are -//! committed via `RegionEdit`, and once when `exit_staging_on_success` merges all staged -//! manifest actions into the live manifest. +//! ² Apply staging fires `on_manifest_updated` once when `exit_staging_on_success` promotes +//! all staged manifest actions (including the SST file additions) into the live manifest. +//! The intermediate staging `RegionEdit` is written to the staging directory and does not +//! fire the hook — its file list is included in the promote notification. //! //! The following paths do **not** trigger any hook: //! - Follower region sync / catchup (manifest read-only; followers don't author changes) -//! - GC / checkpoint / drop / remap (internal bookkeeping, not logical state changes) +//! - GC / checkpoint / remap (internal bookkeeping, not logical state changes) +//! +//! An explicit region **drop** does fire lifecycle hooks — see +//! [Region lifecycle](#region-lifecycle). +//! +//! ## Region lifecycle +//! +//! Beyond manifest/SST observation, the hook observes the high-level lifecycle of an +//! active region: +//! +//! | Event | Method | When | +//! |-------|--------|------| +//! | Open | [`on_region_opened`] | A create or open request registers the region as active (the counterpart to close/drop). Does not fire for the compactor's transient regions or the catch-up reopen. | +//! | Close | [`on_region_closed`] | A close request (or a close-after-flush) removes the region from the active set. Data files, manifest and WAL state are **preserved**; the region may be reopened. | +//! | Logical drop | [`on_region_dropped`] | A drop request has been handled: the region leaves the active set and its WAL entries are marked obsolete. Data files are **not yet deleted**. | +//! | Physical file removal | [`on_region_files_removed`] | The drop GC worker has deleted the region directory. Terminal file-lifecycle event. | +//! +//! Notes: +//! - `on_region_closed` / `on_region_dropped` run **inline in the region worker loop**, +//! so implementations must be fast (same contract as `on_manifest_updated`). +//! - `on_region_opened` runs inline in the worker loop on the **create** path, but on the +//! **open** path it fires inside the spawned open task (`common_runtime::spawn_global`), +//! i.e. concurrently with the worker loop — after WAL replay, before the region is +//! registered and its open request is acknowledged. Implementations must still be fast +//! and must not assume worker-loop-thread affinity or strict ordering against concurrent +//! requests to other regions. +//! - `on_region_files_removed` runs on the background drop GC task, outside the worker loop. +//! - When global GC is enabled and a normal table region is dropped with `partial_drop`, its +//! directory is left for global reclamation and `on_region_files_removed` is **not** fired +//! by the drop worker (observe it via the global GC path instead). +//! - Logical file removal (compaction, region edit, truncate) is already observable via +//! [`on_manifest_updated`] (`Edit.files_to_remove` / `Truncate` action); only the drop +//! worker's physical directory deletion needs a dedicated file hook. //! //! ## Invocation points //! @@ -77,11 +126,18 @@ //! //! ## Future work //! -//! A future `on_files_removed` hook may be added to observe file lifecycle end -//! (GC, drop, truncate, compaction removal). This is not yet implemented. +//! `on_region_files_removed` currently covers only the **drop** GC worker's physical +//! directory removal. A broader per-file `on_files_removed` hook covering compaction +//! removal, truncate, and the global GC reclamation path is not yet implemented +//! (though logical file removal is already observable via `on_manifest_updated`). +//! Role/leadership transitions (`on_region_role_changed`) are also not hooked. //! //! [`on_sst_files_written`]: RegionHook::on_sst_files_written //! [`on_manifest_updated`]: RegionHook::on_manifest_updated +//! [`on_region_opened`]: RegionHook::on_region_opened +//! [`on_region_closed`]: RegionHook::on_region_closed +//! [`on_region_dropped`]: RegionHook::on_region_dropped +//! [`on_region_files_removed`]: RegionHook::on_region_files_removed //! [`RegionManifestManager::update`]: crate::manifest::manager::RegionManifestManager::update //! [`ManifestContext::update_locked`]: crate::region::ManifestContext::update_locked //! [`ManifestContext::update_manifest`]: crate::region::ManifestContext::update_manifest @@ -103,6 +159,13 @@ use crate::sst::parquet::SstInfo; /// /// Must be [`fire`](Self::fire)d **after** the manifest write lock is released /// (the hook may read the manifest). `#[must_use]` so a forgotten receipt warns. +/// +/// ## Staging suppression +/// +/// When `is_staging` is `true`, [`fire`](Self::fire) is a no-op — the write went to +/// the staging manifest directory. The hook only observes writes to the live (normal) +/// manifest directory. Staging actions are accumulated and delivered in a single +/// notification when `exit_staging_on_success` promotes them (`is_staging = false`). #[must_use = "the region hook must be fired after releasing the manifest write lock"] pub(crate) struct PendingManifestHook { region_id: RegionId, @@ -110,6 +173,9 @@ pub(crate) struct PendingManifestHook { action_list: Option, version: ManifestVersion, hook: Option, + /// Whether the manifest write went to the staging directory. + /// When `true`, `fire()` is suppressed — the hook only observes live manifest writes. + is_staging: bool, } impl PendingManifestHook { @@ -118,12 +184,14 @@ impl PendingManifestHook { action_list: Option, version: ManifestVersion, hook: Option, + is_staging: bool, ) -> Self { Self { region_id, action_list, version, hook, + is_staging, } } @@ -132,9 +200,13 @@ impl PendingManifestHook { self.version } - /// Fires the hook if one is registered. Safe to call unconditionally: it is - /// a no-op when no hook is registered. + /// Fires the hook if one is registered, **unless** the write went to the staging + /// manifest directory (`is_staging = true`). Safe to call unconditionally: + /// it is a no-op when no hook is registered or when the write is staging-only. pub(crate) async fn fire(self) { + if self.is_staging { + return; + } if let (Some(hook), Some(action_list)) = (self.hook, self.action_list) { hook.on_manifest_updated(self.region_id, &action_list, self.version) .await; @@ -146,6 +218,9 @@ impl PendingManifestHook { /// keeps `self`'s actions followed by `other`'s, and the *later* manifest /// version wins. Used when a sequence of writes (e.g. staging-exit followed /// by metadata backfill) should notify the hook exactly once. + /// + /// `is_staging` is `true` only if **both** sides are staging; if either side + /// is a live write, the merged result is also live. pub(crate) fn merge(self, other: PendingManifestHook) -> PendingManifestHook { debug_assert_eq!( self.region_id, other.region_id, @@ -164,6 +239,7 @@ impl PendingManifestHook { }, version: self.version.max(other.version), hook: self.hook.or(other.hook), + is_staging: self.is_staging && other.is_staging, } } } @@ -211,10 +287,14 @@ pub trait RegionHook: Send + Sync + Debug { let _ = (region_id, region_metadata, files); } - /// Called after the region manifest is successfully committed. + /// Called after the region manifest is successfully committed to the **live** + /// (normal) manifest directory. /// - /// Fires for **all** manifest write paths: flush, compaction, region edit, - /// copy region, alter, truncate, enter staging, index build, etc. + /// Fires for: flush, compaction, region edit, copy region, alter, truncate, + /// async index build, and apply-staging promote. Does **not** fire for writes + /// to the staging manifest directory (enter staging, operations during staging, + /// the intermediate apply-staging edit) — those are suppressed because their + /// effects are accumulated and delivered in a single promote notification. /// /// Does **not** fire for: /// - Manifest reads / follower sync (no write) @@ -228,6 +308,80 @@ pub trait RegionHook: Send + Sync + Debug { ) { let _ = (region_id, action_list, manifest_version); } + + /// Called once a region **open** or **create** succeeds, but **before** the + /// region is registered in the engine's active set (`insert_region` runs + /// immediately afterwards). + /// + /// Fires once when a region becomes active via a create or open request — + /// the natural counterpart to [`on_region_closed`] / [`on_region_dropped`]. + /// It does **not** fire for the compactor's transient compaction regions + /// (`open_compaction_region`), nor for the internal reopen performed during + /// follower catch-up / leadership promotion. + /// + /// On the **create** path it runs inline in the region worker loop; on the + /// **open** path it runs inside the spawned open task + /// (`common_runtime::spawn_global`), concurrently with the worker loop + /// (after WAL replay, before the region is registered/acknowledged). + /// Implementations must be fast and must **not** assume worker-loop-thread + /// affinity or strict ordering against concurrent requests to other regions. + /// + /// [`on_region_closed`]: RegionHook::on_region_closed + /// [`on_region_dropped`]: RegionHook::on_region_dropped + async fn on_region_opened(&self, region_id: RegionId, region_metadata: &RegionMetadataRef) { + let _ = (region_id, region_metadata); + } + + /// Called after a region is **closed** via a close request. + /// + /// The region is removed from the engine's active set, but its data files, + /// manifest, and WAL state are **preserved**; the region may be reopened + /// later. Fires once per successful close, after the region's background + /// tasks (flush/compaction) have been stopped. + /// + /// Fires for a region of **any** role (leader or follower) that is closed. + /// Does **not** fire when a region is dropped (see [`on_region_dropped`]). + /// + /// Runs inline in the region worker loop; implementations should be fast. + /// + /// [`on_region_dropped`]: RegionHook::on_region_dropped + async fn on_region_closed(&self, region_id: RegionId, region_metadata: &RegionMetadataRef) { + let _ = (region_id, region_metadata); + } + + /// Called after a region is **logically dropped** (a drop request has been + /// handled). + /// + /// The region is removed from the active set and its WAL entries are marked + /// obsolete. Its data files are **not yet deleted** — they are scheduled for + /// asynchronous removal by the GC worker. Observe physical deletion via + /// [`on_region_files_removed`]. + /// + /// Runs inline in the region worker loop; implementations should be fast. + /// + /// [`on_region_files_removed`]: RegionHook::on_region_files_removed + async fn on_region_dropped(&self, region_id: RegionId, region_metadata: &RegionMetadataRef) { + let _ = (region_id, region_metadata); + } + + /// Called after a dropped region's data files are **physically removed** by + /// the drop GC worker (the region directory has been deleted). + /// + /// This is the terminal event in a region's file lifecycle; no further + /// callbacks fire for this region id afterwards. Fires only when the drop + /// worker itself deletes the directory. When global GC is enabled and the + /// region is a normal table region dropped with `partial_drop`, the + /// directory is left for global reclamation and this hook is **not** fired + /// by the drop worker. + /// + /// Runs on a background task, outside the region worker loop. + async fn on_region_files_removed( + &self, + region_id: RegionId, + region_metadata: &RegionMetadataRef, + ) { + let _ = (region_id, region_metadata); + } } pub type RegionHookRef = Arc; diff --git a/src/mito2/src/region.rs b/src/mito2/src/region.rs index 1e173ab62d2..c566dd47c04 100644 --- a/src/mito2/src/region.rs +++ b/src/mito2/src/region.rs @@ -1236,6 +1236,7 @@ impl ManifestContext { action_list_for_hook, version, self.hook.clone(), + is_staging, )) } diff --git a/src/mito2/src/worker/handle_close.rs b/src/mito2/src/worker/handle_close.rs index a616a0e32dc..b582c276927 100644 --- a/src/mito2/src/worker/handle_close.rs +++ b/src/mito2/src/worker/handle_close.rs @@ -89,5 +89,13 @@ impl RegionWorkerLoop { // clean index build status. self.index_build_scheduler.on_region_closed(region_id).await; self.region_count.dec(); + + // Notify the region hook that the region has been closed. The region is + // fully stopped and unregistered, but its files/manifest are preserved. + // Runs inline; the hook contract requires it to be fast. + if let Some(hook) = region.manifest_ctx.hook() { + let metadata = region.metadata(); + hook.on_region_closed(region_id, &metadata).await; + } } } diff --git a/src/mito2/src/worker/handle_create.rs b/src/mito2/src/worker/handle_create.rs index 6e43e74a0e4..bf254dc1e6b 100644 --- a/src/mito2/src/worker/handle_create.rs +++ b/src/mito2/src/worker/handle_create.rs @@ -86,6 +86,12 @@ impl RegionWorkerLoop { self.region_count.inc(); + // Notify the region hook that the region has been opened (created). + // Fires before registration; allocates nothing when no hook is registered. + if let Some(hook) = region.manifest_ctx.hook() { + hook.on_region_opened(region_id, ®ion.metadata()).await; + } + // Insert the MitoRegion into the RegionMap. self.regions.insert_region(region); diff --git a/src/mito2/src/worker/handle_drop.rs b/src/mito2/src/worker/handle_drop.rs index 00a96b0d166..3d111776b82 100644 --- a/src/mito2/src/worker/handle_drop.rs +++ b/src/mito2/src/worker/handle_drop.rs @@ -23,10 +23,12 @@ use object_store::util::join_path; use object_store::{EntryMode, ObjectStore}; use snafu::ResultExt; use store_api::logstore::LogStore; +use store_api::metadata::RegionMetadataRef; use store_api::region_request::{AffectedRows, PathType}; use store_api::storage::RegionId; use tokio::time::sleep; +use crate::engine::region_hook::RegionHookRef; use crate::error::{OpenDalSnafu, Result}; use crate::region::{RegionLeaderState, RegionMapRef}; use crate::worker::{DROPPING_MARKER_FILE, RegionWorkerLoop}; @@ -106,6 +108,22 @@ where self.region_count.dec(); + // Notify registered hooks that the region has been logically dropped, and + // prepare a payload for the background GC task to fire the terminal + // `on_region_files_removed`. When no hook is registered this allocates + // nothing — no metadata snapshot is taken and no payload is built. + let hook_payload = match region.manifest_ctx.hook() { + Some(hook) => { + let region_metadata = region.metadata(); + hook.on_region_dropped(region_id, ®ion_metadata).await; + Some(DropHookPayload { + hook, + metadata: region_metadata, + }) + } + None => None, + }; + let object_store = region.access_layer.object_store().clone(); let dropping_regions = self.dropping_regions.clone(); let listener = self.listener.clone(); @@ -122,6 +140,7 @@ where object_store, dropping_regions, partial_drop, + hook_payload.as_ref(), ) .await } else { @@ -135,6 +154,7 @@ where object_store, dropping_regions, gc_duration, + hook_payload.as_ref(), ) .await }; @@ -156,6 +176,19 @@ where } } +/// Carries the region hook and region metadata into the background GC task so +/// it can fire [`RegionHook::on_region_files_removed`] once the dropped region's +/// directory is physically deleted. +/// +/// Only constructed when a hook is registered; the task receives it as +/// `Option<&DropHookPayload>` so the no-hook path allocates nothing. +/// +/// [`RegionHook::on_region_files_removed`]: crate::engine::region_hook::RegionHook::on_region_files_removed +struct DropHookPayload { + hook: RegionHookRef, + metadata: RegionMetadataRef, +} + /// Background GC task to remove the entire region path once one of the following /// conditions is true: /// - It finds there is no parquet file left. @@ -172,6 +205,7 @@ async fn later_drop_task_without_global_gc( object_store: ObjectStore, dropping_regions: RegionMapRef, gc_duration: Duration, + hook_payload: Option<&DropHookPayload>, ) -> bool { remove_region_with_retry( region_id, @@ -180,6 +214,7 @@ async fn later_drop_task_without_global_gc( dropping_regions, Some(gc_duration), false, + hook_payload, ) .await } @@ -191,6 +226,7 @@ async fn remove_region_with_retry( dropping_regions: std::sync::Arc, gc_duration: Option, mut force: bool, + hook_payload: Option<&DropHookPayload>, ) -> bool { for _ in 0..MAX_RETRY_TIMES { let result = remove_region_dir_once(®ion_path, &object_store, force).await; @@ -204,6 +240,15 @@ async fn remove_region_with_retry( Ok(true) => { dropping_regions.remove_region(region_id); info!("Region {} is dropped, force: {}", region_path, force); + // The region directory has been physically deleted; fire the + // terminal file-lifecycle event. The partial-drop/global-GC path + // never reaches here (it does not delete the directory itself). + if let Some(hook_payload) = hook_payload { + hook_payload + .hook + .on_region_files_removed(region_id, &hook_payload.metadata) + .await; + } return true; } Ok(false) => (), @@ -230,6 +275,7 @@ async fn later_drop_task_with_global_gc( object_store: ObjectStore, dropping_regions: RegionMapRef, partial_drop: bool, + hook_payload: Option<&DropHookPayload>, ) -> bool { // For metadata regions or regions marked for full deletion (such as when dropping a table) // the region directory is forcefully removed immediately. @@ -243,6 +289,7 @@ async fn later_drop_task_with_global_gc( dropping_regions, None, true, + hook_payload, ) .await } else { diff --git a/src/mito2/src/worker/handle_open.rs b/src/mito2/src/worker/handle_open.rs index 3d3520b0cc8..48fef5ddd6f 100644 --- a/src/mito2/src/worker/handle_open.rs +++ b/src/mito2/src/worker/handle_open.rs @@ -149,6 +149,13 @@ impl RegionWorkerLoop { ); region_count.inc(); + // Notify the region hook that the region has been opened. + // Fires before registration; allocates nothing when no hook + // is registered. + if let Some(hook) = region.manifest_ctx.hook() { + hook.on_region_opened(region_id, ®ion.metadata()).await; + } + // Insert the Region into the RegionMap. regions.insert_region(region); diff --git a/src/query/src/parser.rs b/src/query/src/parser.rs index 53f24557c73..87b827e0cc1 100644 --- a/src/query/src/parser.rs +++ b/src/query/src/parser.rs @@ -52,45 +52,37 @@ pub enum QueryStatement { } impl QueryStatement { - pub fn post_process(&self, params: HashMap) -> Result { + pub fn post_process(self, params: HashMap) -> Result { match self { QueryStatement::Sql(_) => UnimplementedSnafu { operation: "sql post process", } .fail(), - QueryStatement::Promql(eval_stmt, alias) => { + QueryStatement::Promql(mut eval_stmt, alias) => { let node_name = match params.get("name") { Some(name) => name.as_str(), None => "", }; - let extension_node = Self::create_extension_node(node_name, &eval_stmt.expr); - Ok(QueryStatement::Promql( - EvalStmt { - expr: Extension(extension_node.unwrap()), - start: eval_stmt.start, - end: eval_stmt.end, - interval: eval_stmt.interval, - lookback_delta: eval_stmt.lookback_delta, - }, - alias.clone(), - )) + let extension_node = Self::create_extension_node(node_name, eval_stmt.expr); + eval_stmt.expr = Extension(extension_node.unwrap()); + Ok(QueryStatement::Promql(eval_stmt, alias)) } } } - fn create_extension_node(node_name: &str, expr: &Expr) -> Option { + fn create_extension_node(node_name: &str, expr: Expr) -> Option { match node_name { ANALYZE_NODE_NAME => Some(NodeExtension { - expr: Arc::new(AnalyzeExpr { expr: expr.clone() }), + expr: Arc::new(AnalyzeExpr { expr }), }), ANALYZE_VERBOSE_NODE_NAME => Some(NodeExtension { - expr: Arc::new(AnalyzeVerboseExpr { expr: expr.clone() }), + expr: Arc::new(AnalyzeVerboseExpr { expr }), }), EXPLAIN_NODE_NAME => Some(NodeExtension { - expr: Arc::new(ExplainExpr { expr: expr.clone() }), + expr: Arc::new(ExplainExpr { expr }), }), EXPLAIN_VERBOSE_NODE_NAME => Some(NodeExtension { - expr: Arc::new(ExplainVerboseExpr { expr: expr.clone() }), + expr: Arc::new(ExplainVerboseExpr { expr }), }), _ => None, } @@ -204,9 +196,10 @@ impl QueryLanguageParser { } pub(crate) fn apply_alias_extension(mut eval_stmt: EvalStmt, alias: &str) -> EvalStmt { + let expr = eval_stmt.expr; eval_stmt.expr = Extension(NodeExtension { expr: Arc::new(AliasExpr { - expr: eval_stmt.expr.clone(), + expr, alias: alias.to_string(), }), }); @@ -263,6 +256,14 @@ macro_rules! define_node_ast_extension { fn children(&self) -> &[Expr] { std::slice::from_ref(&self.expr) } + + fn with_new_children(&self, children: Vec) -> Arc { + let mut iter = children.into_iter(); + match (iter.next(), iter.next()) { + (Some(expr), None) => Arc::new($name_expr { expr }), + _ => Arc::new(self.clone()), + } + } } #[allow(rustdoc::broken_intra_doc_links)] @@ -313,6 +314,16 @@ impl ExtensionExpr for AliasExpr { fn children(&self) -> &[Expr] { std::slice::from_ref(&self.expr) } + fn with_new_children(&self, children: Vec) -> Arc { + let mut iter = children.into_iter(); + match (iter.next(), iter.next()) { + (Some(expr), None) => Arc::new(Self { + expr, + alias: self.alias.clone(), + }), + _ => Arc::new(self.clone()), + } + } } #[derive(Debug, Clone)] pub struct Alias { diff --git a/src/query/src/promql/planner.rs b/src/query/src/promql/planner.rs index c2cd6316ada..61913156cf8 100644 --- a/src/query/src/promql/planner.rs +++ b/src/query/src/promql/planner.rs @@ -229,6 +229,8 @@ impl IslandExpr { !modifier.return_bool && modifier.matching.is_none() && matches!(modifier.card, VectorMatchCardinality::OneToOne) + && modifier.fill_values.lhs.is_none() + && modifier.fill_values.rhs.is_none() }) => { let lhs = Self::try_new(lhs, env)?; @@ -1093,6 +1095,18 @@ impl PromPlanner { query_engine_state: &QueryEngineState, binary_expr: &PromBinaryExpr, ) -> Result { + // promql-parser accepts fill modifiers, but Greptime does not implement the + // required outer joins and missing-value substitution. Reject them before the + // binary-island fast path so they cannot silently behave like normal inner joins. + if let Some(modifier) = &binary_expr.modifier { + ensure!( + modifier.fill_values.lhs.is_none() && modifier.fill_values.rhs.is_none(), + UnsupportedExprSnafu { + name: "PromQL fill modifiers" + } + ); + } + if let Some(plan) = self.try_plan_binary_island(binary_expr).await? { return Ok(plan); } @@ -5542,6 +5556,33 @@ mod test { ); } + #[tokio::test] + async fn reject_binary_fill_modifiers() { + let state = build_query_engine_state(); + + for query in [ + "some_metric + fill(0) some_alt_metric", + "some_metric + fill_left(0) some_alt_metric", + "some_metric + fill_right(0) some_alt_metric", + "(some_metric + fill(0) some_alt_metric) + some_metric", + ] { + let eval_stmt = build_eval_stmt(query); + let table_provider = build_test_table_provider(&[], 0, 0).await; + let err = PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &state) + .await + .unwrap_err(); + + assert!( + matches!( + &err, + crate::promql::error::Error::UnsupportedExpr { name, .. } + if name == "PromQL fill modifiers" + ), + "{err}" + ); + } + } + #[tokio::test] async fn timestamp_binary_join_falls_back_when_tsid_is_projected_out() { for query in [ diff --git a/tests-integration/src/tests/instance_test.rs b/tests-integration/src/tests/instance_test.rs index 628f97de96a..8572e9ce5b2 100644 --- a/tests-integration/src/tests/instance_test.rs +++ b/tests-integration/src/tests/instance_test.rs @@ -1098,6 +1098,10 @@ async fn test_execute_copy_from_csv_strict_headers(instance: Arcv1.1.1"] +features = ["json", "json2", "table", "read"] +owner = "query" diff --git a/tests/compatibility/cases/legacy_json2_variant_payload/setup.sql b/tests/compatibility/cases/legacy_json2_variant_payload/setup.sql new file mode 100644 index 00000000000..cef3a913c93 --- /dev/null +++ b/tests/compatibility/cases/legacy_json2_variant_payload/setup.sql @@ -0,0 +1,29 @@ +CREATE TABLE t_legacy_json2_variant_payload ( + ts TIMESTAMP TIME INDEX, + j JSON2 +) WITH ( + 'append_mode' = 'true' +); + +INSERT INTO t_legacy_json2_variant_payload (ts, j) VALUES + ('2026-07-08 00:00:00+0000', '{"a": {"b": 1}, "c": "s1", "d": [{"e": {"f": 0.1}}]}'), + ('2026-07-08 00:01:00+0000', '{"a": {"b": -2}, "c": [1], "d": [{"e": {"f": 0.2}}]}'), + ('2026-07-08 00:02:00+0000', '{"a": {"b": "s3"}, "c": "s3", "d": [{"e": {"g": -0.3}}]}'), + ('2026-07-08 00:03:00+0000', '{"a": {"b": null}, "c": null}'), + ('2026-07-08 00:04:00+0000', '{"c": "s5", "d": {"e": true}}'), + ('2026-07-08 00:05:00+0000', '{"a": {"b": 6}}'); + +ADMIN FLUSH_TABLE('t_legacy_json2_variant_payload'); + +CREATE TABLE t_legacy_json2_variant_payload_object_d ( + ts TIMESTAMP TIME INDEX, + j JSON2 +) WITH ( + 'append_mode' = 'true' +); + +INSERT INTO t_legacy_json2_variant_payload_object_d (ts, j) VALUES + ('2026-07-08 01:00:00+0000', '{"d": {"e": true}}'), + ('2026-07-08 01:01:00+0000', '{"d": {"e": false}}'); + +ADMIN FLUSH_TABLE('t_legacy_json2_variant_payload_object_d'); diff --git a/tests/compatibility/cases/legacy_json2_variant_payload/verify.result b/tests/compatibility/cases/legacy_json2_variant_payload/verify.result new file mode 100644 index 00000000000..a474cc84345 --- /dev/null +++ b/tests/compatibility/cases/legacy_json2_variant_payload/verify.result @@ -0,0 +1,60 @@ +SELECT ts, j.a.b AS a_b, j.c AS c +FROM t_legacy_json2_variant_payload +ORDER BY ts; + ++---------------------+-----+-----+ +| ts | a_b | c | ++---------------------+-----+-----+ +| 2026-07-08T00:00:00 | 1 | s1 | +| 2026-07-08T00:01:00 | -2 | [1] | +| 2026-07-08T00:02:00 | s3 | s3 | +| 2026-07-08T00:03:00 | | | +| 2026-07-08T00:04:00 | | s5 | +| 2026-07-08T00:05:00 | 6 | | ++---------------------+-----+-----+ + +SELECT + ts, + j.d.e AS d_e +FROM t_legacy_json2_variant_payload_object_d +ORDER BY ts; + ++---------------------+-------+ +| ts | d_e | ++---------------------+-------+ +| 2026-07-08T01:00:00 | true | +| 2026-07-08T01:01:00 | false | ++---------------------+-------+ + +-- JSON2 projection rendering is a user-visible compatibility contract. +SELECT ts, j.d AS d +FROM t_legacy_json2_variant_payload +ORDER BY ts; + ++---------------------+--------------------+ +| ts | d | ++---------------------+--------------------+ +| 2026-07-08T00:00:00 | [{"e":{"f":0.1}}] | +| 2026-07-08T00:01:00 | [{"e":{"f":0.2}}] | +| 2026-07-08T00:02:00 | [{"e":{"g":-0.3}}] | +| 2026-07-08T00:03:00 | | +| 2026-07-08T00:04:00 | {"e":true} | +| 2026-07-08T00:05:00 | | ++---------------------+--------------------+ + +SHOW CREATE TABLE t_legacy_json2_variant_payload; + ++--------------------------------+---------------------------------------------------------------+ +| Table | Create Table | ++--------------------------------+---------------------------------------------------------------+ +| t_legacy_json2_variant_payload | CREATE TABLE IF NOT EXISTS "t_legacy_json2_variant_payload" ( | +| | "ts" TIMESTAMP(3) NOT NULL, | +| | "j" JSON2 NULL, | +| | TIME INDEX ("ts") | +| | ) | +| | | +| | ENGINE=mito | +| | WITH( | +| | append_mode = 'true' | +| | ) | ++--------------------------------+---------------------------------------------------------------+ diff --git a/tests/compatibility/cases/legacy_json2_variant_payload/verify.sql b/tests/compatibility/cases/legacy_json2_variant_payload/verify.sql new file mode 100644 index 00000000000..a3c2cab1b92 --- /dev/null +++ b/tests/compatibility/cases/legacy_json2_variant_payload/verify.sql @@ -0,0 +1,16 @@ +SELECT ts, j.a.b AS a_b, j.c AS c +FROM t_legacy_json2_variant_payload +ORDER BY ts; + +SELECT + ts, + j.d.e AS d_e +FROM t_legacy_json2_variant_payload_object_d +ORDER BY ts; + +-- JSON2 projection rendering is a user-visible compatibility contract. +SELECT ts, j.d AS d +FROM t_legacy_json2_variant_payload +ORDER BY ts; + +SHOW CREATE TABLE t_legacy_json2_variant_payload; diff --git a/tests/perf/AGENTS.md b/tests/perf/AGENTS.md index eb95544d22d..7f64b283bdc 100644 --- a/tests/perf/AGENTS.md +++ b/tests/perf/AGENTS.md @@ -11,14 +11,14 @@ docs together. - `[case]` is report metadata only. `[scenario]` is the executable regression configuration and must include `kind`, data layout, tables, queries, and - thresholds. Keep scenario parsing explicit in both - `tests/perf/query_regression_runner.py` and - `src/cmd/src/bin/query_perf_fixture.rs`. + thresholds. Rust owns case schema, defaults, validation, and normalized plan + output through `query_perf_fixture plan`; `tests/perf/query_regression_runner.py` + should only orchestrate from that normalized JSON. - Keep the direct-SST generator generic. Issue-specific behavior belongs in case files and thresholds, not in Rust generator logic. - Before pushing perf harness changes, run at least: - `uv run --no-project python -m py_compile .github/scripts/query-regression-run.py .github/scripts/query-regression-summary.py .github/scripts/query-regression-pr-metadata.py tests/perf/query_regression_runner.py` - `cargo fmt --all -- --check` - - `cargo build -p cmd --bin query_perf_fixture` + - `cargo build -p cmd --bin query_perf_fixture --features dev-tools` - dry-run the Python runner and Rust fixture generator against all built-in cases when the DSL or workflow case selection changes. diff --git a/tests/perf/README.md b/tests/perf/README.md index a59081b5681..91bc6c1637e 100644 --- a/tests/perf/README.md +++ b/tests/perf/README.md @@ -4,6 +4,11 @@ This directory is for query performance cases that compare a base build with a candidate build. It is not a replacement for sqlness: the goal is to measure the effect of optimizer/query-engine changes on realistic scan work. +The Prometheus remote-write cases are end-to-end regression coverage for the +automatic write, flush, SST, and query path. They do not replace controlled +encoding experiments that explicitly compare `plain`/dictionary, +`no_dictionary`, BYTE_STREAM_SPLIT, or Auto policies. + ## Phase 1: direct readable SST fixtures Phase 1 should generate data by writing readable Mito SST files and matching @@ -40,8 +45,10 @@ The runner also supports `scenario.kind = "prom_remote_write_then_query"` for a bounded write-path smoke/regression flow. This path is explicit and separate from the direct-SST fixture path: it starts the base and candidate distributed clusters, writes deterministic Prometheus remote-write v1 samples through -`/v1/prometheus/write`, flushes the configured physical metric table, then runs -the configured SQL/TQL queries against the logical metric table. +`/v1/prometheus/write`, flushes the configured physical metric table, checks +visibility, runs the configured SQL/TQL queries, stops and awaits the datanode, +then optionally inspects the landed SST/Parquet footer, encoding, and size and +runs the read-bench against the quiescent data directory. Remote-write cases configure one database, one logical metric, and one physical table under `[scenario.remote_write]`: @@ -52,34 +59,121 @@ kind = "prom_remote_write_then_query" [scenario.remote_write] database = "public" -metric = "prom_remote_write_smoke" +metric = "prom_remote_write_seeded_random" physical_table = "greptime_physical_table" -series_count = 8 -samples_per_series = 30 +series_count = 512 +samples_per_series = 1440 +sample_chunk_size = 480 +flush_every_sample_chunks = 1 start_unix_millis = 1_704_067_200_000 -step_millis = 15_000 -chunk_series_count = 4 -visibility_timeout_seconds = 30 +step_millis = 60_000 +chunk_series_count = 128 +timeout_seconds = 180 +visibility_timeout_seconds = 120 # Optional: split by time so each helper invocation writes only this many samples # per series, then periodically flush the physical metric table to produce # multiple time-interval SSTs. -# sample_chunk_size = 300 -# flush_every_sample_chunks = 1 [scenario.remote_write.prom_store] pending_rows_flush_interval = "1s" +max_batch_rows = 100000 ``` +Cases can optionally add `[scenario.remote_write.value]` to control the generated +sample values without changing label cardinality or the runner lifecycle: + +```toml +[scenario.remote_write.value] +pattern = "quantized_signal" # linear, constant, modulo, unique, seeded_random, + # run_length, quantized_signal, + # signal_with_sporadic_stalls, mixed_signal_repeated +base = 0.0 +step = 0.125 +cardinality = 4096 # buckets for modulo/seeded_random/quantized_signal/run_length +seed = 12345 # deterministic seeded_random input +run_length = 8 # adjacent samples per bucket for run_length/quantized_signal +stall_every = 100 # interval for signal_with_sporadic_stalls +stall_length = 16 # held samples inside each stall interval +mixed_every = 5 # every Nth sample becomes the repeated base value +``` + +The default `linear` pattern preserves the helper's historical formula. Use +`constant` or low-cardinality `modulo`/`seeded_random` values for repeated-value +data shapes, `run_length` for run-heavy low-cardinality series, `quantized_signal` +for signal-like values collapsed into a finite bucket set, `signal_with_sporadic_stalls` +for mostly continuous signals with periodic flat spots, and `mixed_signal_repeated` +for signal-plus-periodic-default mixtures. `unique` or high-cardinality buckets +still work for broad sample-value distributions. This is a generic sample-value +control for query/ingestion cases; it does not inspect or assert storage +encoding, Parquet footers, or storage policy choices. For chunked remote-write +ingestion, the runner passes the sample offset and total sample count to the +helper so non-linear value patterns use a stable global/per-series ordinal across +chunks. + +Case schema, value distribution defaults, storage defaults, and read-bench +defaults are owned by Rust. The Python runner calls +`query_perf_fixture plan --case ` and orchestrates the normalized JSON. +The same helper exposes `direct-sst`, `prom-remote-write`, and `inspect-footer` +subcommands; the old direct invocation (`query_perf_fixture --case ... --out-dir +...`) remains compatible. + +Remote-write cases that need to validate storage output can add +`[scenario.remote_write.storage]`. When present, the scenario body becomes: +remote-write → flush → visibility check → query measurement → stop and await +datanode → Parquet footer/size/encoding inspection → optional read-bench. Footer +inspection is handled by `query_perf_fixture inspect-footer`, which reads local +Parquet footers directly; it does not require Python `pyarrow`. + +```toml +[scenario.remote_write.storage] +inspect = true +column = "greptime_value" +include_metadata_files = false +# Optional: inspect below datanode data home instead of the whole fresh data dir. +# root_suffix = "greptime/public/" +min_files = 1 +min_files_with_column = 1 +require_encodings = ["BYTE_STREAM_SPLIT"] +forbid_encodings = ["PLAIN_DICTIONARY"] +max_total_file_size_bytes = 104857600 +max_column_compressed_size_bytes = 52428800 +max_column_uncompressed_size_bytes = 209715200 +max_candidate_total_file_size_regression_pct = 10.0 +max_candidate_column_compressed_size_regression_pct = 10.0 +max_candidate_column_uncompressed_size_regression_pct = 10.0 +``` + +The `*_pct` storage thresholds are percent regression limits comparing candidate +against base; `pct` means percent, not percentile. Per-target storage checks +(`min_files`, `min_files_with_column`, required/forbidden encodings, and absolute +`max_*_bytes`) run for both base and candidate. Comparative +`max_candidate_*_regression_pct` checks remain base-vs-candidate. These checks +are generic footer and byte-size assertions and do not encode product-specific +heuristics. When storage inspection is enabled, `min_files` and +`min_files_with_column` default to `1` even if omitted, so dry-runs show these +planned checks and an empty flush or missing target column fails the scenario. +By default the inspector root is the target datanode data home, which is suitable +for a fresh single-case data directory. For reused or more complex data homes, set +`root_suffix` to a path relative to the datanode data home to narrow inspection. + +When storage inspection is enabled, `[scenario.remote_write.read_bench]` defaults +to enabled with both datanode `parquetbench` and `scanbench`. Disable it with +`enabled = false`. `parquetbench` measures per-SST reader cost, `scanbench` +measures region scan cost, and query measurements still exercise the SQL/TQL +frontend path. Treat all performance conclusions as release-only; debug builds +are suitable only for command wiring and correctness checks. + The runner creates the configured database if needed, writes a per-target frontend config enabling `[prom_store]` with metric engine storage and a non-zero `pending_rows_flush_interval`, and validates that the logical metric table reaches `series_count * samples_per_series` rows before trusting the query measurements. -Use `--remote-write-generator /path/to/prom_remote_write_fixture` to provide the -Rust payload helper. `--fixture-only` is rejected for remote-write cases; use -`--dry-run` for planning. +Use `--fixture-generator /path/to/query_perf_fixture` to provide the Rust helper. +Deprecated `--remote-write-generator` and `--storage-inspector` options are kept +only for CLI compatibility. `--fixture-only` is rejected for remote-write cases; +use `--dry-run` for planning. Large manual remote-write cases can set `sample_chunk_size` to split ingestion by -time. For each chunk, the runner invokes `prom_remote_write_fixture` with the +time. For each chunk, the runner invokes `query_perf_fixture prom-remote-write` with the same series cardinality but a shorter `--samples-per-series` and an advanced `--start-unix-millis`. `flush_every_sample_chunks` controls periodic `ADMIN FLUSH_TABLE('')` calls; with `flush_every_sample_chunks = 1`, @@ -89,6 +183,25 @@ need multi-window file distribution should span multiple compaction windows. If `sample_chunk_size` is omitted, the runner keeps the older single-helper-invocation behavior and flushes once at the end. +The default remote-write coverage set contains four cases with the same 2048 +series × 14,400 samples (29,491,200 rows per target) shape. Each writes ten +one-day chunks, flushes every chunk, requires at least two visible SSTs with the +value column, runs seven read-bench iterations with `parquetbench` capped at +four SSTs while `scanbench` covers the landed region, and measures one ten-day +TQL selector with two warmups and 15 iterations: + +- `prom_remote_write_seeded_random`: high-distinct seeded-random values with + cardinality 29,491,200 and seed 8444. +- `prom_remote_write_run_heavy`: low-cardinality values in exact 16-sample runs. +- `prom_remote_write_mixed_every`: continuous signal values with every fifth + sample repeated at the base value. +- `prom_remote_write_integer_counter`: strictly increasing integer counter + values, with disjoint ranges for each series. + +These cases record normal storage/footer and read-bench results but do not gate +specific value encodings or storage-size outcomes. Use the controlled encoding +experiment matrix for those policy comparisons. + `tests/perf/query_cases/prom_remote_write_7913/case.toml` is a larger manual case for issue #7913. It writes 8192 series × 20160 samples through remote-write in 1440-sample daily time chunks, flushing after each chunk before running 1d/7d/14d @@ -246,7 +359,8 @@ as a fallback. Manifest files are reflinked or copied, not hardlinked. Fixture generator smoke test: ```bash -cargo run -p cmd --bin query_perf_fixture -- \ +cargo run -p cmd --bin query_perf_fixture --features dev-tools -- \ + direct-sst \ --case tests/perf/query_cases/smoke_direct_sst/case.toml \ --out-dir /tmp/query-perf-smoke ``` @@ -271,10 +385,10 @@ Remote-write runner dry-run: ```bash uv run --no-project python tests/perf/query_regression_runner.py \ - --case tests/perf/query_cases/prom_remote_write_smoke/case.toml \ + --case tests/perf/query_cases/prom_remote_write_seeded_random/case.toml \ --base-bin /path/to/base/greptime \ --candidate-bin /path/to/candidate/greptime \ - --remote-write-generator /path/to/prom_remote_write_fixture \ + --fixture-generator /path/to/query_perf_fixture \ --work-dir /tmp/query-perf-remote-write \ --dry-run ``` @@ -289,14 +403,22 @@ query regression runs. It builds its own binaries for now: candidate checkout - runner and summary formatter from the candidate checkout -The workflow runs automatically only for non-draft PRs labeled -`query-regression` (on label/ready-for-review/reopen events, not every push). +The workflow builds base and candidate `greptime` as normal release-equivalent +binaries. Candidate `query_perf_fixture` is the extra head-side helper binary; +the runner uses candidate `greptime datanode parquetbench/scanbench` as the +read-bench tool against each target's data directory. + +The workflow runs automatically only when `query-regression` is added to a +non-draft PR; it does not rerun on pushes, ready-for-review, or reopen events. PR runs build base/candidate once and then run the default case set with `--allow-large-fixture`. Manual `workflow_dispatch` runs can pass `all`, one case path, or a comma/whitespace-separated list of case paths, and can override refs. -It always uploads `query-regression-work/**` and `query-regression-summary.md`, -writes the Markdown summary to the workflow step summary, and updates a sticky PR -comment through the trusted follow-up workflow. +The main report artifact uploads only aggregate/per-target JSON reports, +component logs, and `query-regression-summary.md` with seven-day retention; +fixture data, SSTs, and cluster state are excluded. PR runs also upload a +separate trusted-comment artifact containing PR metadata and aggregate reports. +The workflow writes the Markdown summary to the workflow step summary and +updates a sticky PR comment through the trusted follow-up workflow. ## Built-in cases diff --git a/tests/perf/query_cases/prom_remote_write_integer_counter/case.toml b/tests/perf/query_cases/prom_remote_write_integer_counter/case.toml new file mode 100644 index 00000000000..3ac64a94b9d --- /dev/null +++ b/tests/perf/query_cases/prom_remote_write_integer_counter/case.toml @@ -0,0 +1,48 @@ +[case] +name = "prom_remote_write_integer_counter" +description = "End-to-end Prometheus remote-write regression with strictly increasing integer counter values" + +[scenario] +kind = "prom_remote_write_then_query" + +[scenario.remote_write] +database = "public" +metric = "prom_remote_write_integer_counter" +physical_table = "greptime_physical_table" +series_count = 2048 +samples_per_series = 14400 +sample_chunk_size = 1440 +flush_every_sample_chunks = 1 +start_unix_millis = 1704067200000 +step_millis = 60000 +chunk_series_count = 256 +timeout_seconds = 600 +visibility_timeout_seconds = 300 + +[scenario.remote_write.value] +pattern = "unique" +base = 0 +step = 1 + +[scenario.remote_write.prom_store] +pending_rows_flush_interval = "1s" +max_batch_rows = 100000 + +[scenario.remote_write.storage] +column = "greptime_value" +min_files = 2 +min_files_with_column = 2 + +[scenario.remote_write.read_bench] +iterations = 7 +max_files = 4 + +[[scenario.queries]] +name = "selector_10d_integer_counter" +kind = "tql" +query = "TQL ANALYZE VERBOSE (1704067200, 1704931200, '1h') prom_remote_write_integer_counter{host=~'host.*'}" +warmup = 2 +iterations = 15 + +[scenario.queries.thresholds] +max_candidate_latency_regression_pct = 75 diff --git a/tests/perf/query_cases/prom_remote_write_mixed_every/case.toml b/tests/perf/query_cases/prom_remote_write_mixed_every/case.toml new file mode 100644 index 00000000000..9ba39f2fbed --- /dev/null +++ b/tests/perf/query_cases/prom_remote_write_mixed_every/case.toml @@ -0,0 +1,49 @@ +[case] +name = "prom_remote_write_mixed_every" +description = "End-to-end Prometheus remote-write regression with periodic repeated signal values" + +[scenario] +kind = "prom_remote_write_then_query" + +[scenario.remote_write] +database = "public" +metric = "prom_remote_write_mixed_every" +physical_table = "greptime_physical_table" +series_count = 2048 +samples_per_series = 14400 +sample_chunk_size = 1440 +flush_every_sample_chunks = 1 +start_unix_millis = 1704067200000 +step_millis = 60000 +chunk_series_count = 256 +timeout_seconds = 600 +visibility_timeout_seconds = 300 + +[scenario.remote_write.value] +pattern = "mixed_signal_repeated" +base = 0 +step = 0.125 +mixed_every = 5 + +[scenario.remote_write.prom_store] +pending_rows_flush_interval = "1s" +max_batch_rows = 100000 + +[scenario.remote_write.storage] +column = "greptime_value" +min_files = 2 +min_files_with_column = 2 + +[scenario.remote_write.read_bench] +iterations = 7 +max_files = 4 + +[[scenario.queries]] +name = "selector_10d_mixed_every" +kind = "tql" +query = "TQL ANALYZE VERBOSE (1704067200, 1704931200, '1h') prom_remote_write_mixed_every{host=~'host.*'}" +warmup = 2 +iterations = 15 + +[scenario.queries.thresholds] +max_candidate_latency_regression_pct = 75 diff --git a/tests/perf/query_cases/prom_remote_write_run_heavy/case.toml b/tests/perf/query_cases/prom_remote_write_run_heavy/case.toml new file mode 100644 index 00000000000..243892e42b1 --- /dev/null +++ b/tests/perf/query_cases/prom_remote_write_run_heavy/case.toml @@ -0,0 +1,50 @@ +[case] +name = "prom_remote_write_run_heavy" +description = "End-to-end Prometheus remote-write regression with repeated 16-sample value runs" + +[scenario] +kind = "prom_remote_write_then_query" + +[scenario.remote_write] +database = "public" +metric = "prom_remote_write_run_heavy" +physical_table = "greptime_physical_table" +series_count = 2048 +samples_per_series = 14400 +sample_chunk_size = 1440 +flush_every_sample_chunks = 1 +start_unix_millis = 1704067200000 +step_millis = 60000 +chunk_series_count = 256 +timeout_seconds = 600 +visibility_timeout_seconds = 300 + +[scenario.remote_write.value] +pattern = "run_length" +base = 0 +step = 1 +cardinality = 1024 +run_length = 16 + +[scenario.remote_write.prom_store] +pending_rows_flush_interval = "1s" +max_batch_rows = 100000 + +[scenario.remote_write.storage] +column = "greptime_value" +min_files = 2 +min_files_with_column = 2 + +[scenario.remote_write.read_bench] +iterations = 7 +max_files = 4 + +[[scenario.queries]] +name = "selector_10d_run_heavy" +kind = "tql" +query = "TQL ANALYZE VERBOSE (1704067200, 1704931200, '1h') prom_remote_write_run_heavy{host=~'host.*'}" +warmup = 2 +iterations = 15 + +[scenario.queries.thresholds] +max_candidate_latency_regression_pct = 75 diff --git a/tests/perf/query_cases/prom_remote_write_seeded_random/case.toml b/tests/perf/query_cases/prom_remote_write_seeded_random/case.toml new file mode 100644 index 00000000000..6a63edbcbbf --- /dev/null +++ b/tests/perf/query_cases/prom_remote_write_seeded_random/case.toml @@ -0,0 +1,48 @@ +[case] +name = "prom_remote_write_seeded_random" +description = "End-to-end Prometheus remote-write regression with high-distinct seeded-random values" + +[scenario] +kind = "prom_remote_write_then_query" + +[scenario.remote_write] +database = "public" +metric = "prom_remote_write_seeded_random" +physical_table = "greptime_physical_table" +series_count = 2048 +samples_per_series = 14400 +sample_chunk_size = 1440 +flush_every_sample_chunks = 1 +start_unix_millis = 1704067200000 +step_millis = 60000 +chunk_series_count = 256 +timeout_seconds = 600 +visibility_timeout_seconds = 300 + +[scenario.remote_write.value] +pattern = "seeded_random" +cardinality = 29_491_200 +seed = 8444 + +[scenario.remote_write.prom_store] +pending_rows_flush_interval = "1s" +max_batch_rows = 100000 + +[scenario.remote_write.storage] +column = "greptime_value" +min_files = 2 +min_files_with_column = 2 + +[scenario.remote_write.read_bench] +iterations = 7 +max_files = 4 + +[[scenario.queries]] +name = "selector_10d_seeded_random" +kind = "tql" +query = "TQL ANALYZE VERBOSE (1704067200, 1704931200, '1h') prom_remote_write_seeded_random{host=~'host.*'}" +warmup = 2 +iterations = 15 + +[scenario.queries.thresholds] +max_candidate_latency_regression_pct = 75 diff --git a/tests/perf/query_cases/prom_remote_write_smoke/case.toml b/tests/perf/query_cases/prom_remote_write_smoke/case.toml deleted file mode 100644 index b946fbc8f51..00000000000 --- a/tests/perf/query_cases/prom_remote_write_smoke/case.toml +++ /dev/null @@ -1,31 +0,0 @@ -[case] -name = "prom_remote_write_smoke" -description = "Tiny Prometheus remote-write ingestion then TQL query smoke case" - -[scenario] -kind = "prom_remote_write_then_query" - -[scenario.remote_write] -database = "public" -metric = "prom_remote_write_smoke" -physical_table = "greptime_physical_table" -series_count = 8 -samples_per_series = 30 -start_unix_millis = 1_704_067_200_000 -step_millis = 15_000 -chunk_series_count = 4 -timeout_seconds = 30 - -[scenario.remote_write.prom_store] -pending_rows_flush_interval = "1s" -max_batch_rows = 10000 - -[[scenario.queries]] -name = "selector_5m_smoke" -kind = "tql" -query = "TQL ANALYZE VERBOSE (1704067200, 1704067650, '15s') prom_remote_write_smoke{host=~'host.*'}" -warmup = 0 -iterations = 2 - -[scenario.queries.thresholds] -max_candidate_latency_regression_pct = 100 diff --git a/tests/perf/query_regression_runner.py b/tests/perf/query_regression_runner.py index a7f2646ea8b..f5c9628b83d 100644 --- a/tests/perf/query_regression_runner.py +++ b/tests/perf/query_regression_runner.py @@ -66,7 +66,8 @@ def parse_args() -> argparse.Namespace: p.add_argument("--base-bin", required=True, type=Path) p.add_argument("--candidate-bin", required=True, type=Path) p.add_argument("--fixture-generator", type=Path) - p.add_argument("--remote-write-generator", type=Path, help="prom_remote_write_fixture helper binary") + p.add_argument("--remote-write-generator", type=Path, help="deprecated: use --fixture-generator query_perf_fixture") + p.add_argument("--storage-inspector", type=Path, help="deprecated: use --fixture-generator query_perf_fixture") p.add_argument("--work-dir", required=True, type=Path) p.add_argument("--fixture-cache-dir", type=Path, help="persistent directory for generated fixtures, keyed by case content") p.add_argument("--reuse-fixture", action="store_true") @@ -83,6 +84,16 @@ def load_case(path: Path) -> dict[str, Any]: return tomllib.load(f) +def load_normalized_case(case_path: Path, fixture_generator: Path | None) -> dict[str, Any]: + if fixture_generator is None: + raise ValueError("--fixture-generator query_perf_fixture is required for Rust-owned case planning") + result = run_command([str(fixture_generator), "plan", "--case", str(case_path)]) + if result["returncode"] != 0: + raise RuntimeError(f"query_perf_fixture plan failed: {result['stderr'][:2000]}") + plan = json.loads(result["stdout"]) + return {"case": load_case(case_path).get("case", {}), "scenario": plan["scenario"], "schema_version": plan.get("schema_version")} + + def require_binary(path: Path, name: str, *, dry_run: bool) -> None: if dry_run: return @@ -157,11 +168,9 @@ def scenario(case: dict[str, Any]) -> dict[str, Any]: def case_tables(case: dict[str, Any]) -> list[dict[str, Any]]: value = scenario(case) if value.get("kind") == "prom_remote_write_then_query": - remote = value.get("remote_write") or {} - metric = remote.get("metric") or remote.get("metric_name") - database = remote.get("database", "public") - if not metric: - raise ValueError("remote-write scenario requires scenario.remote_write.metric") + remote = value["remote_write"] + metric = remote["metric"] + database = remote["database"] return [{"database": database, "name": metric, "engine": "metric", "validate_show_create_engine": False}] tables = value.get("tables") or [] if not tables or value.get("layout", {}).get("regions") != 1: @@ -548,7 +557,7 @@ def generate_fixture(generator: Path | None, case_path: Path, fixture_dir: Path, if fixture_dir.exists() and not reuse_fixture: shutil.rmtree(fixture_dir) fixture_dir.mkdir(parents=True, exist_ok=True) - cmd = [str(generator), "--case", str(case_path), "--out-dir", str(fixture_dir)] + cmd = [str(generator), "direct-sst", "--case", str(case_path), "--out-dir", str(fixture_dir)] if table_name is not None: cmd += ["--table", table_name] if region_id is not None: @@ -777,6 +786,243 @@ def enforce_thresholds(case: dict[str, Any], base: dict[str, Any], candidate: di return results +def storage_config(remote: dict[str, Any]) -> dict[str, Any] | None: + value = remote["storage"] + if not isinstance(value, dict): + return None + if value["inspect"] is False: + return None + return value + + +def run_storage_inspection(helper: Path | None, target: RunTarget, storage: dict[str, Any], *, dry_run: bool) -> dict[str, Any]: + if helper is None: + if not dry_run: + raise ValueError("--fixture-generator query_perf_fixture is required when storage inspection is enabled") + helper = Path("query_perf_fixture") + root = target.datanode_data_dir + if storage.get("root_suffix"): + root = root / str(storage["root_suffix"]) + cmd = [ + str(helper), + "inspect-footer", + "--root", str(root), + "--column", str(storage["column"]), + ] + if storage["include_metadata_files"]: + cmd.append("--include-metadata-files") + if dry_run: + return {"status": "dry-run", "cmd": cmd, "root": str(root)} + result = run_command(cmd) + result["status"] = "ok" if result["returncode"] == 0 else "failed" + if result["returncode"] != 0: + raise RuntimeError(f"storage inspector failed for {target.name}: {result['stderr'][:2000]}") + try: + result["summary"] = json.loads(result["stdout"]) + except json.JSONDecodeError: + result["summary_parse_error"] = result["stdout"] + result["status"] = "failed" + return result + + +def storage_summary(inspection: dict[str, Any]) -> dict[str, Any]: + summary = inspection.get("summary") + if isinstance(summary, dict): + nested = summary.get("summary") + if isinstance(nested, dict): + return nested + return {} + + +def enforce_storage_thresholds(storage: dict[str, Any] | None, base: dict[str, Any] | None, candidate: dict[str, Any] | None) -> list[dict[str, Any]]: + if not storage: + return [] + results: list[dict[str, Any]] = [] + target_summaries = [("base", storage_summary(base or {})), ("candidate", storage_summary(candidate or {}))] + cand = target_summaries[1][1] + base_summary = storage_summary(base or {}) + + def add_abs(name: str, field: str) -> None: + limit = storage.get(name) + if limit is None: + return + for target_name, summary in target_summaries: + actual = summary.get(field) + ok = actual is not None and float(actual) <= float(limit) + results.append({"target": target_name, "threshold": name, "status": "passed" if ok else "failed", "actual": actual, "limit": limit}) + + def add_min(name: str, field: str) -> None: + limit = storage[name] + if limit is None: + return + for target_name, summary in target_summaries: + actual = summary.get(field) + ok = actual is not None and int(actual) >= int(limit) + results.append({"target": target_name, "threshold": name, "status": "passed" if ok else "failed", "actual": actual, "limit": limit}) + + def add_cmp(name: str, field: str) -> None: + limit = storage.get(name) + if limit is None: + return + base_value = base_summary.get(field) + candidate_value = cand.get(field) + if base_value in (None, 0) or candidate_value is None: + results.append({"threshold": name, "status": "failed", "reason": "missing or zero base/candidate value", "base": base_value, "candidate": candidate_value}) + return + actual = (float(candidate_value) - float(base_value)) / float(base_value) * 100.0 + results.append({"threshold": name, "status": "passed" if actual <= float(limit) else "failed", "actual_pct": actual, "limit_pct": limit, "base": base_value, "candidate": candidate_value}) + + add_min("min_files", "file_count") + add_min("min_files_with_column", "files_with_column") + add_abs("max_total_file_size_bytes", "total_file_size") + add_abs("max_column_compressed_size_bytes", "column_compressed_size") + add_abs("max_column_uncompressed_size_bytes", "column_uncompressed_size") + for target_name, summary in target_summaries: + encodings = set(summary.get("unique_encodings") or []) + for required in storage["require_encodings"]: + results.append({"target": target_name, "threshold": "require_encodings", "encoding": required, "status": "passed" if required in encodings else "failed"}) + for forbidden in storage["forbid_encodings"]: + results.append({"target": target_name, "threshold": "forbid_encodings", "encoding": forbidden, "status": "passed" if forbidden not in encodings else "failed"}) + add_cmp("max_candidate_total_file_size_regression_pct", "total_file_size") + add_cmp("max_candidate_column_compressed_size_regression_pct", "column_compressed_size") + add_cmp("max_candidate_column_uncompressed_size_regression_pct", "column_uncompressed_size") + return results + + +def planned_storage_thresholds(storage: dict[str, Any] | None) -> list[dict[str, Any]]: + if not storage: + return [] + return list(storage["planned_thresholds"]) + + +REGION_DIR_RE = re.compile(r"^(\d+)_(\d{10})$") + + +def inspection_report(inspection: dict[str, Any] | None) -> dict[str, Any]: + if not inspection: + return {} + report = inspection.get("summary") + return report if isinstance(report, dict) else {} + + +def inspected_relative_path(target: RunTarget, inspection: dict[str, Any], relative_path: str) -> Path: + report = inspection_report(inspection) + root_text = report.get("root") or inspection.get("root") + if not root_text: + return Path(relative_path) + root = Path(root_text).resolve(strict=False) + data_home = target.datanode_data_dir.resolve(strict=False) + try: + root_suffix = root.relative_to(data_home) + except ValueError as e: + raise RuntimeError(f"storage inspection root {root} is not under datanode data home {data_home}") from e + return root_suffix / relative_path + + +def parse_sst_bench_target(target: RunTarget, inspection: dict[str, Any], file: dict[str, Any], *, dry_run: bool) -> dict[str, str] | None: + relative_path = str(file.get("relative_path") or "") + if dry_run and (not relative_path or "<" in relative_path): + return { + "relative_path": relative_path or "/_/data/.parquet", + "table_dir": "/", + "region_id": ":", + "path_type": "data", + "file_id": "", + } + if not relative_path: + return None + full_relative = inspected_relative_path(target, inspection, relative_path) + parts = full_relative.parts + if not parts or not parts[-1].endswith(".parquet"): + return None + region_index = next((idx for idx, part in enumerate(parts) if REGION_DIR_RE.match(part)), None) + if region_index is None or region_index == 0: + if dry_run: + return { + "relative_path": relative_path, + "table_dir": "/", + "region_id": ":", + "path_type": "data", + "file_id": Path(parts[-1]).stem, + } + return None + region_match = REGION_DIR_RE.match(parts[region_index]) + assert region_match is not None + file_index = len(parts) - 1 + path_type = "bare" + if region_index + 1 < file_index and parts[region_index + 1] in ("data", "metadata"): + path_type = parts[region_index + 1] + if path_type == "metadata": + return None + table_dir = "/".join(parts[:region_index]).rstrip("/") + "/" + return { + "relative_path": str(full_relative), + "table_dir": table_dir, + "region_id": f"{int(region_match.group(1))}:{int(region_match.group(2))}", + "path_type": path_type, + "file_id": Path(parts[-1]).stem, + } + + +def run_read_bench(bench_binary: Path, target: RunTarget, read_bench: dict[str, Any] | None, storage_inspection: dict[str, Any] | None, *, dry_run: bool) -> dict[str, Any]: + if not read_bench or read_bench["enabled"] is False: + return {"status": "skipped", "reason": "read_bench disabled"} + run_parquetbench = read_bench["parquetbench"] + run_scanbench = read_bench["scanbench"] + if not run_parquetbench and not run_scanbench: + return {"status": "skipped", "reason": "parquetbench and scanbench disabled"} + if storage_inspection is None: + raise ValueError("read_bench requires storage inspection") + bench_dir = target.work_dir / "read_bench" + config_toml = bench_dir / "bench.toml" + scan_json = bench_dir / "scan.json" + files = inspection_report(storage_inspection).get("files") or [] + ssts = [f for f in files if f.get("relative_path", "").endswith(".parquet") and f.get("columns")] + if dry_run and not ssts: + ssts = [{"relative_path": ".parquet"}] + if read_bench["max_files"] is not None: + ssts = ssts[: int(read_bench["max_files"])] + bench_targets = [parsed for f in ssts if (parsed := parse_sst_bench_target(target, storage_inspection, f, dry_run=dry_run)) is not None] + if not bench_targets: + return {"status": "failed", "reason": "no inspected data SST files available for read_bench"} + config_text = f'[storage]\ndata_home = "{target.datanode_data_dir}"\ntype = "File"\n\n[[region_engine]]\n[region_engine.mito]\n' + commands = [] + parquet_runs = [] + scan_runs = [] + iterations = str(int(read_bench["iterations"])) + if run_parquetbench: + for bench_target in bench_targets: + cmd = [str(bench_binary), "datanode", "parquetbench", "--config", str(config_toml), "--region-id", bench_target["region_id"], "--table-dir", bench_target["table_dir"], "--file-id", bench_target["file_id"], "--scan-config", str(scan_json), "--path-type", bench_target["path_type"], "--iterations", iterations, "--reader", str(read_bench["parquet_reader"])] + commands.append(cmd) + parquet_runs.append({**bench_target, "cmd": cmd, "status": "dry-run" if dry_run else "planned"}) + regions: dict[tuple[str, str, str], list[str]] = {} + if run_scanbench: + for bench_target in bench_targets: + regions.setdefault((bench_target["table_dir"], bench_target["region_id"], bench_target["path_type"]), []).append(bench_target["relative_path"]) + for (table_dir, region_id, path_type), region_files in regions.items(): + cmd = [str(bench_binary), "datanode", "scanbench", "--config", str(config_toml), "--region-id", region_id, "--table-dir", table_dir, "--scan-config", str(scan_json), "--path-type", path_type, "--scanner", str(read_bench["scan_scanner"]), "--parallelism", str(int(read_bench["parallelism"])), "--iterations", iterations] + commands.append(cmd) + scan_runs.append({"table_dir": table_dir, "region_id": region_id, "path_type": path_type, "files": region_files, "cmd": cmd, "status": "dry-run" if dry_run else "planned"}) + if dry_run: + return {"status": "dry-run", "config_path": str(config_toml), "scan_config_path": str(scan_json), "commands": commands, "parquetbench": parquet_runs, "scanbench": scan_runs} + bench_dir.mkdir(parents=True, exist_ok=True) + config_toml.write_text(config_text) + scan_json.write_text(json.dumps({"projection_names": read_bench["projection"]}, indent=2) + "\n") + avg_re = re.compile(r"Average duration[^0-9]*([0-9.]+)\s*ms", re.I) + for runs in (parquet_runs, scan_runs): + for run in runs: + result = run_command(run["cmd"]) + run.update(result) + run["status"] = "ok" if result["returncode"] == 0 else "failed" + m = avg_re.search(result.get("stdout", "")) + if m: + run["average_ms"] = float(m.group(1)) + def median(runs: list[dict[str, Any]]) -> float | None: + vals = [r["average_ms"] for r in runs if "average_ms" in r] + return statistics.median(vals) if vals else None + return {"status": "ok" if all(r.get("status") == "ok" for r in parquet_runs + scan_runs) else "failed", "config_path": str(config_toml), "scan_config_path": str(scan_json), "parquetbench": parquet_runs, "scanbench": scan_runs, "aggregate": {"parquetbench_median_average_ms": median(parquet_runs), "scanbench_median_average_ms": median(scan_runs)}} + + def write_json(path: Path, payload: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") @@ -791,18 +1037,18 @@ def require_fresh_work_dirs(targets: list[RunTarget], *, reuse_work_dir: bool, d def write_frontend_prom_config(target: RunTarget, remote: dict[str, Any]) -> Path: - prom = remote.get("prom_store") or {} + prom = remote["prom_store"] path = target.work_dir / "frontend-prom-store.toml" path.parent.mkdir(parents=True, exist_ok=True) path.write_text( "[prom_store]\n" "enable = true\n" "with_metric_engine = true\n" - f"pending_rows_flush_interval = {json.dumps(str(prom.get('pending_rows_flush_interval', '1s')))}\n" - f"max_batch_rows = {int(prom.get('max_batch_rows', 100000))}\n" - f"max_concurrent_flushes = {int(prom.get('max_concurrent_flushes', 256))}\n" - f"worker_channel_capacity = {int(prom.get('worker_channel_capacity', 65526))}\n" - f"max_inflight_requests = {int(prom.get('max_inflight_requests', 3000))}\n" + f"pending_rows_flush_interval = {json.dumps(str(prom['pending_rows_flush_interval']))}\n" + f"max_batch_rows = {int(prom['max_batch_rows'])}\n" + f"max_concurrent_flushes = {int(prom['max_concurrent_flushes'])}\n" + f"worker_channel_capacity = {int(prom['worker_channel_capacity'])}\n" + f"max_inflight_requests = {int(prom['max_inflight_requests'])}\n" ) return path @@ -810,23 +1056,38 @@ def write_frontend_prom_config(target: RunTarget, remote: dict[str, Any]) -> Pat def run_remote_write(generator: Path | None, target: RunTarget, remote: dict[str, Any], *, dry_run: bool) -> dict[str, Any]: if generator is None: if not dry_run: - raise ValueError("--remote-write-generator is required for prom_remote_write_then_query unless --dry-run is set") - generator = Path("prom_remote_write_fixture") - metric = remote.get("metric") or remote.get("metric_name") - physical_table = remote.get("physical_table", "greptime_physical_table") + raise ValueError("--fixture-generator query_perf_fixture is required for prom_remote_write_then_query") + generator = Path("query_perf_fixture") + metric = remote["metric"] + physical_table = remote["physical_table"] cmd = [ str(generator), + "prom-remote-write", "--endpoint", f"http://127.0.0.1:{target.http_port}/v1/prometheus/write", - "--database", remote.get("database", "public"), + "--database", remote["database"], "--metric", metric, "--physical-table", physical_table, - "--series-count", str(int(remote.get("series_count", 8))), - "--samples-per-series", str(int(remote.get("samples_per_series", 30))), - "--start-unix-millis", str(int(remote.get("start_unix_millis", 1_704_067_200_000))), - "--step-millis", str(int(remote.get("step_millis", 15_000))), - "--chunk-series-count", str(int(remote.get("chunk_series_count", remote.get("batch_size", 8)))), - "--timeout-seconds", str(int(remote.get("timeout_seconds", 60))), + "--series-count", str(int(remote["series_count"])), + "--samples-per-series", str(int(remote["samples_per_series"])), + "--start-unix-millis", str(int(remote["start_unix_millis"])), + "--step-millis", str(int(remote["step_millis"])), + "--chunk-series-count", str(int(remote["chunk_series_count"])), + "--timeout-seconds", str(int(remote["timeout_seconds"])), ] + value = remote["value"] + cmd.extend(["--value-pattern", str(value["pattern"])]) + cmd.extend(["--value-base", str(value["base"])]) + cmd.extend(["--value-step", str(value["step"])]) + cmd.extend(["--value-cardinality", str(int(value["cardinality"]))]) + cmd.extend(["--value-seed", str(int(value["seed"]))]) + cmd.extend(["--value-run-length", str(int(value["run_length"]))]) + cmd.extend(["--value-stall-every", str(int(value["stall_every"]))]) + cmd.extend(["--value-stall-length", str(int(value["stall_length"]))]) + cmd.extend(["--value-mixed-every", str(int(value["mixed_every"]))]) + if "sample_offset" in remote: + cmd.extend(["--value-sample-offset", str(int(remote["sample_offset"]))]) + if "total_samples_per_series" in remote: + cmd.extend(["--value-total-samples-per-series", str(int(remote["total_samples_per_series"]))]) if dry_run: return {"status": "dry-run", "cmd": cmd} result = run_command(cmd) @@ -862,23 +1123,23 @@ def flush_remote_physical_table(target: RunTarget, db: str, physical_table: str, def run_remote_write_ingestion(generator: Path | None, target: RunTarget, remote: dict[str, Any], args: argparse.Namespace, *, dry_run: bool) -> tuple[dict[str, Any], list[dict[str, Any]]]: - sample_chunk_size = remote.get("sample_chunk_size") - db = remote.get("database", "public") - physical_table = remote.get("physical_table", "greptime_physical_table") + sample_chunk_size = remote["sample_chunk_size"] + db = remote["database"] + physical_table = remote["physical_table"] if sample_chunk_size is None: rw = run_remote_write(generator, target, remote, dry_run=dry_run) flush = flush_remote_physical_table(target, db, physical_table, args, dry_run=dry_run, reason="final") return rw, [flush] - total_samples = int(remote.get("samples_per_series", 30)) + total_samples = int(remote["samples_per_series"]) chunk_samples = int(sample_chunk_size) if chunk_samples <= 0: raise ValueError("scenario.remote_write.sample_chunk_size must be positive") - flush_every = int(remote.get("flush_every_sample_chunks", 1)) + flush_every = int(remote["flush_every_sample_chunks"]) if flush_every <= 0: raise ValueError("scenario.remote_write.flush_every_sample_chunks must be positive") - start = int(remote.get("start_unix_millis", 1_704_067_200_000)) - step = int(remote.get("step_millis", 15_000)) + start = int(remote["start_unix_millis"]) + step = int(remote["step_millis"]) chunks: list[dict[str, Any]] = [] flushes: list[dict[str, Any]] = [] chunk_index = 0 @@ -889,6 +1150,8 @@ def run_remote_write_ingestion(generator: Path | None, target: RunTarget, remote chunk_remote = dict(remote) chunk_remote["samples_per_series"] = current chunk_remote["start_unix_millis"] = start + offset * step + chunk_remote["sample_offset"] = offset + chunk_remote["total_samples_per_series"] = total_samples result = run_remote_write(generator, target, chunk_remote, dry_run=dry_run) result["sample_offset"] = offset result["samples_per_series"] = current @@ -901,8 +1164,8 @@ def run_remote_write_ingestion(generator: Path | None, target: RunTarget, remote flushes.append(flush_remote_physical_table(target, db, physical_table, args, dry_run=dry_run, reason="final", chunk_index=chunk_index)) aggregate = summarize_remote_write_chunks(chunks) if dry_run: - series_count = int(remote.get("series_count", 8)) - chunk_series_count = int(remote.get("chunk_series_count", remote.get("batch_size", 8))) + series_count = int(remote["series_count"]) + chunk_series_count = int(remote["chunk_series_count"]) aggregate["rows"] = series_count * total_samples aggregate["samples_written"] = aggregate["rows"] aggregate["batches"] = chunk_index * ((series_count + chunk_series_count - 1) // chunk_series_count) @@ -917,7 +1180,7 @@ def run_remote_write_ingestion(generator: Path | None, target: RunTarget, remote def expected_remote_write_rows(remote: dict[str, Any]) -> int: - return int(remote.get("series_count", 8)) * int(remote.get("samples_per_series", 30)) + return int(remote["series_count"]) * int(remote["samples_per_series"]) def extract_count_value(result: dict[str, Any]) -> int | None: @@ -966,16 +1229,21 @@ def poll_expected_count(target: RunTarget, table_name: str, db: str, expected_ro def run_remote_write_scenario(args: argparse.Namespace, case: dict[str, Any], case_path: Path, targets: list[RunTarget], report: dict[str, Any]) -> None: - remote = scenario(case).get("remote_write") or {} + remote = scenario(case)["remote_write"] tables = case_tables(case) if args.fixture_only: raise ValueError("--fixture-only is not supported for prom_remote_write_then_query; use --dry-run for planning") - if args.remote_write_generator is not None: - require_binary(args.remote_write_generator, "remote-write generator", dry_run=args.dry_run) + helper = args.fixture_generator or args.remote_write_generator + if helper is not None: + require_binary(helper, "query_perf_fixture", dry_run=args.dry_run) elif not args.dry_run: - raise ValueError("--remote-write-generator is required for prom_remote_write_then_query") + raise ValueError("--fixture-generator is required for prom_remote_write_then_query") + storage = storage_config(remote) + if storage and helper is None and not args.dry_run: + raise ValueError("--fixture-generator is required when scenario.remote_write.storage.inspect is enabled") clusters: list[DistributedCluster] = [] target_results = [] + storage_results = [] try: for target in targets: target.work_dir.mkdir(parents=True, exist_ok=True) @@ -984,28 +1252,40 @@ def run_remote_write_scenario(args: argparse.Namespace, case: dict[str, Any], ca clusters.append(cluster) if not args.dry_run: cluster.start_all(config_path) - db = remote.get("database", "public") + db = remote["database"] create_database = {"status": "dry-run", "database": db} if not args.dry_run: create_database = http_post_sql(target.http_port, f"CREATE DATABASE IF NOT EXISTS {sql_ident(db)}", "public", args.http_timeout) - rw, flushes = run_remote_write_ingestion(args.remote_write_generator, target, remote, args, dry_run=args.dry_run) - physical = remote.get("physical_table", "greptime_physical_table") + rw, flushes = run_remote_write_ingestion(helper, target, remote, args, dry_run=args.dry_run) visibility = {"status": "dry-run"} query_result = {"validation": [], "validation_errors": [], "measurements": [], "status": "planned"} if not args.dry_run: - visibility = poll_expected_count(target, tables[0]["name"], db, expected_remote_write_rows(remote), float(remote.get("visibility_timeout_seconds", 30)), args.http_timeout) + visibility = poll_expected_count(target, tables[0]["name"], db, expected_remote_write_rows(remote), float(remote["visibility_timeout_seconds"]), args.http_timeout) query_result = run_queries(target, case, tables, args.http_timeout) + cluster.stop_component("datanode") + storage_inspection = None + if storage: + storage_inspection = run_storage_inspection(helper or args.storage_inspector, target, storage, dry_run=args.dry_run) + storage_results.append(storage_inspection) + read_bench_result = run_read_bench(args.candidate_bin, target, remote["read_bench"], storage_inspection, dry_run=args.dry_run) tr = {"name": target.name, "binary": str(target.binary), "work_dir": str(target.work_dir), "components": cluster.component_report(), "frontend_config": str(config_path), "create_database": create_database, "remote_write": rw, "flushes": flushes, "flush": flushes[-1] if flushes else None, "visibility": visibility, **query_result} + if storage_inspection is not None: + tr["storage_inspection"] = storage_inspection + tr["read_bench"] = read_bench_result flushes_ok = all(flush.get("ok") for flush in flushes) - remote_checks_ok = args.dry_run or (create_database.get("ok") and flushes_ok and visibility.get("ok") and visibility.get("row_count_ok")) + storage_ok = storage_inspection is None or args.dry_run or storage_inspection.get("status") == "ok" + read_bench_ok = args.dry_run or read_bench_result.get("status") in ("ok", "skipped") + remote_checks_ok = args.dry_run or (create_database.get("ok") and flushes_ok and visibility.get("ok") and visibility.get("row_count_ok") and storage_ok and read_bench_ok) if not remote_checks_ok: - tr.setdefault("validation_errors", []).append({"phase": "remote_write_visibility", "create_database_ok": create_database.get("ok"), "flushes_ok": flushes_ok, "visibility_ok": visibility.get("ok"), "row_count_ok": visibility.get("row_count_ok"), "create_database": create_database, "flushes": flushes, "visibility": visibility}) + tr.setdefault("validation_errors", []).append({"phase": "remote_write_visibility_storage", "create_database_ok": create_database.get("ok"), "flushes_ok": flushes_ok, "visibility_ok": visibility.get("ok"), "row_count_ok": visibility.get("row_count_ok"), "storage_ok": storage_ok, "read_bench_ok": read_bench_ok, "create_database": create_database, "flushes": flushes, "visibility": visibility, "read_bench": read_bench_result}) tr["status"] = "planned" if args.dry_run else ("measured" if remote_checks_ok and query_result["status"] == "ok" else "failed") write_json(target.report_path, tr) report["targets"].append(tr) target_results.append(query_result) if not args.dry_run: - report["thresholds"] = enforce_thresholds(case, target_results[0], target_results[1]) + report["thresholds"] = enforce_thresholds(case, target_results[0], target_results[1]) + enforce_storage_thresholds(storage, storage_results[0] if storage_results else None, storage_results[1] if len(storage_results) > 1 else None) + elif storage: + report["thresholds"] = planned_storage_thresholds(storage) report["status"] = "planned" if args.dry_run else ("failed" if any(t["status"] == "failed" for t in report["thresholds"]) or any(t.get("status") == "failed" for t in report["targets"]) else "ok") finally: for cluster in reversed(clusters): @@ -1015,7 +1295,9 @@ def run_remote_write_scenario(args: argparse.Namespace, case: dict[str, Any], ca def main() -> int: args = parse_args() case_path = args.case.resolve() - case = load_case(case_path) + if args.fixture_generator is not None: + require_binary(args.fixture_generator, "fixture generator", dry_run=False) + case = load_normalized_case(case_path, args.fixture_generator) scenario_config = scenario(case) scenario_kind = scenario_config.get("kind") tables = case_tables(case) @@ -1025,9 +1307,6 @@ def main() -> int: require_binary(args.candidate_bin, "candidate", dry_run=args.dry_run or args.fixture_only) if scenario_kind == "direct_readable_sst" and not args.dry_run and args.fixture_generator is None: raise ValueError("--fixture-generator is required unless --dry-run is set") - if args.fixture_generator is not None: - require_binary(args.fixture_generator, "fixture generator", dry_run=args.dry_run) - ports = allocate_ports(16) targets = [make_target("base", args.base_bin.resolve(), work_root, ports[:8]), make_target("candidate", args.candidate_bin.resolve(), work_root, ports[8:])] require_fresh_work_dirs(targets, reuse_work_dir=args.reuse_work_dir, dry_run=args.dry_run, fixture_only=args.fixture_only) diff --git a/tests/perf/test_query_regression_runner_compaction_toctou.py b/tests/perf/test_query_regression_runner_compaction_toctou.py new file mode 100644 index 00000000000..675f451a10e --- /dev/null +++ b/tests/perf/test_query_regression_runner_compaction_toctou.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# Copyright 2023 Greptime Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Regression coverage for quiescing remote-write storage before read benches.""" + +import argparse +import importlib.util +import sys +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + + +RUNNER_PATH = Path(__file__).with_name("query_regression_runner.py") +SPEC = importlib.util.spec_from_file_location("query_regression_runner_under_test", RUNNER_PATH) +assert SPEC is not None and SPEC.loader is not None +runner = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = runner +SPEC.loader.exec_module(runner) + + +class RemoteWriteCompactionToctouTest(unittest.TestCase): + def test_quiesces_each_target_before_storage_inspection_and_read_bench(self) -> None: + lifecycle: dict[str, list[str]] = {"base": [], "candidate": []} + clusters = [] + + class RecordingCluster: + def __init__(self, target): + self.target = target + self.active = set() + self.stop_all_calls = 0 + clusters.append(self) + + def start_all(self, _config_path): + self.active.update(("metasrv", "datanode", "frontend")) + + def stop_component(self, name): + if name in self.active: + lifecycle[self.target.name].append(f'cluster.stop_component("{name}")') + self.active.remove(name) + + def stop_all(self): + self.stop_all_calls += 1 + for name in ("frontend", "datanode", "metasrv"): + self.stop_component(name) + + def component_report(self): + return {} + + def run_queries(target, _case, _tables, _http_timeout): + lifecycle[target.name].append("query measurement completes") + return {"validation": [], "validation_errors": [], "measurements": [], "status": "ok"} + + def inspect_storage(_helper, target, _storage, *, dry_run): + self.assertFalse(dry_run) + lifecycle[target.name].append("storage inspection") + return {"status": "ok", "summary": {"summary": {}}} + + def read_bench(_binary, target, _read_bench, _inspection, *, dry_run): + self.assertFalse(dry_run) + lifecycle[target.name].append("read bench") + return {"status": "ok"} + + remote_write = { + "database": "public", + "metric": "cpu_usage", + "physical_table": "cpu_usage_physical", + "series_count": 1, + "samples_per_series": 1, + "visibility_timeout_seconds": 1, + "storage": { + "inspect": True, + "min_files": None, + "min_files_with_column": None, + "max_total_file_size_bytes": None, + "max_column_compressed_size_bytes": None, + "max_column_uncompressed_size_bytes": None, + "require_encodings": [], + "forbid_encodings": [], + }, + "read_bench": {"enabled": True}, + "prom_store": { + "pending_rows_flush_interval": "1s", + "max_batch_rows": 1, + "max_concurrent_flushes": 1, + "worker_channel_capacity": 1, + "max_inflight_requests": 1, + }, + } + case = { + "scenario": { + "kind": "prom_remote_write_then_query", + "remote_write": remote_write, + "queries": [], + } + } + args = argparse.Namespace( + fixture_only=False, + fixture_generator=Path("/bin/true"), + remote_write_generator=None, + storage_inspector=None, + candidate_bin=Path("/bin/true"), + dry_run=False, + http_timeout=1.0, + ) + + with tempfile.TemporaryDirectory() as tmpdir: + root = Path(tmpdir) + targets = [ + runner.make_target("base", Path("/bin/true"), root, list(range(10000, 10008))), + runner.make_target("candidate", Path("/bin/true"), root, list(range(10008, 10016))), + ] + report = {"targets": [], "thresholds": []} + with ( + patch.object(runner, "DistributedCluster", RecordingCluster), + patch.object(runner, "run_remote_write_ingestion", return_value=({"status": "ok"}, [{"ok": True}])), + patch.object(runner, "http_post_sql", return_value={"ok": True}), + patch.object(runner, "poll_expected_count", return_value={"ok": True, "row_count_ok": True}), + patch.object(runner, "run_queries", side_effect=run_queries), + patch.object(runner, "run_storage_inspection", side_effect=inspect_storage), + patch.object(runner, "run_read_bench", side_effect=read_bench), + ): + runner.run_remote_write_scenario(args, case, root / "case.toml", targets, report) + + for cluster in clusters: + self.assertEqual(cluster.stop_all_calls, 1) + self.assertEqual(cluster.active, set()) + cluster.stop_all() + self.assertEqual(cluster.stop_all_calls, 2) + self.assertEqual(cluster.active, set()) + + expected = [ + "query measurement completes", + 'cluster.stop_component("datanode")', + "storage inspection", + "read bench", + ] + relevant = {name: [event for event in events if event in expected] for name, events in lifecycle.items()} + for target_name in ("base", "candidate"): + with self.subTest(target=target_name): + self.assertEqual(relevant[target_name], expected) + + +if __name__ == "__main__": + unittest.main()