Merge branch 'main' into series-expansion

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
This commit is contained in:
Ruihang Xia
2026-07-14 16:06:18 +08:00
47 changed files with 3177 additions and 993 deletions
@@ -3,6 +3,7 @@ logging:
format: "json"
filters:
- log_store=debug
- rskafka=debug
meta:
configData: |-
[runtime]
@@ -3,6 +3,7 @@ logging:
format: "json"
filters:
- log_store=debug
- rskafka=debug
meta:
configData: |-
[runtime]
@@ -16,8 +16,8 @@ runs:
helm upgrade \
--install kafka oci://registry-1.docker.io/bitnamicharts/kafka \
--set controller.replicaCount=${{ inputs.controller-replicas }} \
--set controller.resources.requests.cpu=50m \
--set controller.resources.requests.memory=128Mi \
--set controller.resources.requests.cpu=500m \
--set controller.resources.requests.memory=512Mi \
--set controller.resources.limits.cpu=2000m \
--set controller.resources.limits.memory=2Gi \
--set listeners.controller.protocol=PLAINTEXT \
+4 -10
View File
@@ -26,6 +26,10 @@ from pathlib import Path
DEFAULT_CASES = [
"tests/perf/query_cases/smoke_direct_sst/case.toml",
"tests/perf/query_cases/prom_remote_write_seeded_random/case.toml",
"tests/perf/query_cases/prom_remote_write_run_heavy/case.toml",
"tests/perf/query_cases/prom_remote_write_mixed_every/case.toml",
"tests/perf/query_cases/prom_remote_write_integer_counter/case.toml",
"tests/perf/query_cases/promql_pushdown_7913/case.toml",
"tests/perf/query_cases/sql_topk_order_by/case.toml",
"tests/perf/query_cases/sql_aggregate_order_by/case.toml",
@@ -106,8 +110,6 @@ def run_case(args: argparse.Namespace, case_path: Path, work_dir: Path) -> int:
str(candidate_bin),
"--fixture-generator",
str(fixture_generator),
"--remote-write-generator",
str(args.remote_write_generator),
"--work-dir",
str(work_dir),
"--http-timeout",
@@ -160,11 +162,6 @@ def main() -> int:
parser.add_argument("--work-dir", default=Path("query-regression-work"), type=Path)
parser.add_argument("--http-timeout", default=os.environ.get("HTTP_TIMEOUT", "300"))
parser.add_argument("--allow-large-fixture", default=os.environ.get("ALLOW_LARGE_FIXTURE", "false"))
parser.add_argument(
"--remote-write-generator",
type=Path,
default=configured_path(os.environ.get("REMOTE_WRITE_GENERATOR")),
)
parser.add_argument(
"--summary-script",
type=Path,
@@ -177,9 +174,6 @@ def main() -> int:
parser.add_argument("--candidate-ref", default=os.environ.get("CANDIDATE_REF", ""))
parser.add_argument("--github-output", default=os.environ.get("GITHUB_OUTPUT"))
args = parser.parse_args()
if args.remote_write_generator is None:
args.remote_write_generator = args.candidate_src / "target" / profile_dir(args.cargo_profile) / "prom_remote_write_fixture"
try:
cases = split_cases(args.cases or [os.environ.get("CASE_PATHS", "all")])
except ValueError as err:
+11 -5
View File
@@ -119,10 +119,15 @@ jobs:
git reset --hard FETCH_HEAD
git clean -ffdx
- name: Test query regression runner lifecycle
working-directory: src
run: uv run --no-project python tests/perf/test_query_regression_runner_compaction_toctou.py
- name: Build candidate greptime and fixture generators
working-directory: src
run: |
cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime --bin query_perf_fixture --bin prom_remote_write_fixture
cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime
cargo build --profile "${CARGO_PROFILE}" -p cmd --bin query_perf_fixture --features dev-tools
target_dir="${CARGO_PROFILE}"
if [[ "${CARGO_PROFILE}" == "dev" ]]; then
target_dir="debug"
@@ -132,8 +137,6 @@ jobs:
"${GITHUB_WORKSPACE}/query-regression-bins/candidate/greptime"
cp "${CARGO_TARGET_DIR}/${target_dir}/query_perf_fixture" \
"${GITHUB_WORKSPACE}/query-regression-bins/candidate/query_perf_fixture"
cp "${CARGO_TARGET_DIR}/${target_dir}/prom_remote_write_fixture" \
"${GITHUB_WORKSPACE}/query-regression-bins/candidate/prom_remote_write_fixture"
- name: Run query regression
id: run
@@ -146,7 +149,6 @@ jobs:
BASE_BIN: ${{ github.workspace }}/query-regression-bins/base/greptime
CANDIDATE_BIN: ${{ github.workspace }}/query-regression-bins/candidate/greptime
FIXTURE_GENERATOR: ${{ github.workspace }}/query-regression-bins/candidate/query_perf_fixture
REMOTE_WRITE_GENERATOR: ${{ github.workspace }}/query-regression-bins/candidate/prom_remote_write_fixture
run: >-
uv run --no-project python src/.github/scripts/query-regression-run.py
--base-src src
@@ -171,9 +173,13 @@ jobs:
with:
name: query-regression-report
path: |
query-regression-work/**
query-regression-work/**/query-regression-report.json
query-regression-work/**/base/report.json
query-regression-work/**/candidate/report.json
query-regression-work/**/logs/**
query-regression-summary.md
if-no-files-found: warn
retention-days: 7
- name: Upload trusted comment artifact
if: ${{ always() && github.event_name == 'pull_request' }}
Generated
+4 -4
View File
@@ -11038,9 +11038,9 @@ dependencies = [
[[package]]
name = "promql-parser"
version = "0.7.3"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f514ab521bb2b4ce26aca29bf133498b2a2651b785d03a262bbf5772329c279f"
checksum = "9e8c90c956be4237a93be6f2298302b56974ef862ab11e0f98869d1f27c69f88"
dependencies = [
"cfgrammar",
"chrono",
@@ -11183,7 +11183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
dependencies = [
"anyhow",
"itertools 0.11.0",
"itertools 0.14.0",
"proc-macro2",
"quote",
"syn 2.0.117",
@@ -11196,7 +11196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425"
dependencies = [
"anyhow",
"itertools 0.11.0",
"itertools 0.14.0",
"proc-macro2",
"quote",
"syn 2.0.117",
+1 -1
View File
@@ -198,7 +198,7 @@ paste = "1.0"
pin-project = "1.0"
pretty_assertions = "1.4.0"
prometheus = { version = "0.14", features = ["process"] }
promql-parser = { version = "0.7.3", features = ["ser"] }
promql-parser = { version = "0.10.0", features = ["ser"] }
prost = { version = "0.14", features = ["no-recursion-limit"] }
prost-types = "0.14"
raft-engine = { version = "0.4.1", default-features = false }
+7 -1
View File
@@ -209,7 +209,13 @@ Read the [v1.0 highlights](https://greptime.com/blogs/2025-11-05-greptimedb-v1-h
If GreptimeDB is useful to you, please star the repo.
[![Star History Chart](https://api.star-history.com/svg?repos=GreptimeTeam/GreptimeDB&type=Date)](https://www.star-history.com/#GreptimeTeam/GreptimeDB&Date)
<a href="https://www.star-history.com/?repos=GreptimeTeam%2FGreptimeDB&type=date&legend=top-left">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=GreptimeTeam/GreptimeDB&type=date&theme=dark&legend=top-left&sealed_token=1kD9cyZVPnWvxbpbPPTWo_r7W351gDAhGx9_55zfD1vSIfHsdTRfH6NAtdPVX_M2ac8sjNxiuw7-0Vk4WvbyXmOuufxlAoU3QF7z4onyltr0o0c9xVfUnPKltz7llUc3nEDW7l_r2fHBBMtJGf1-jsitNozdAUpb8eb7OJp9N1LqEnMQK78ESp2zgjyr" />
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=GreptimeTeam/GreptimeDB&type=date&legend=top-left&sealed_token=1kD9cyZVPnWvxbpbPPTWo_r7W351gDAhGx9_55zfD1vSIfHsdTRfH6NAtdPVX_M2ac8sjNxiuw7-0Vk4WvbyXmOuufxlAoU3QF7z4onyltr0o0c9xVfUnPKltz7llUc3nEDW7l_r2fHBBMtJGf1-jsitNozdAUpb8eb7OJp9N1LqEnMQK78ESp2zgjyr" />
<img alt="Star History Chart" src="https://api.star-history.com/chart?repos=GreptimeTeam/GreptimeDB&type=date&legend=top-left&sealed_token=1kD9cyZVPnWvxbpbPPTWo_r7W351gDAhGx9_55zfD1vSIfHsdTRfH6NAtdPVX_M2ac8sjNxiuw7-0Vk4WvbyXmOuufxlAoU3QF7z4onyltr0o0c9xVfUnPKltz7llUc3nEDW7l_r2fHBBMtJGf1-jsitNozdAUpb8eb7OJp9N1LqEnMQK78ESp2zgjyr" />
</picture>
</a>
<img alt="Known Users" src="https://greptime.com/logo/img/users.png"/>
+2 -5
View File
@@ -12,10 +12,7 @@ path = "src/bin/greptime.rs"
[[bin]]
name = "query_perf_fixture"
path = "src/bin/query_perf_fixture.rs"
[[bin]]
name = "prom_remote_write_fixture"
path = "src/bin/prom_remote_write_fixture.rs"
required-features = ["dev-tools"]
[features]
default = [
@@ -25,7 +22,7 @@ default = [
"meta-srv/mysql_kvbackend",
]
enterprise = ["common-meta/enterprise", "frontend/enterprise", "meta-srv/enterprise"]
# Developer-only tools (benchmarks, diagnostics) gating the `datanode parquetbench` subcommand.
# Developer-only helper binary gate for `query_perf_fixture`.
# Kept out of `default` so normal/release builds don't compile them.
dev-tools = []
tokio-console = ["common-telemetry/tokio-console"]
-1
View File
@@ -117,7 +117,6 @@ async fn start(cli: Command) -> Result<()> {
}
datanode::SubCommand::Objbench(ref bench) => bench.run().await,
datanode::SubCommand::Scanbench(ref bench) => bench.run().await,
#[cfg(feature = "dev-tools")]
datanode::SubCommand::Parquetbench(ref bench) => bench.run().await,
},
SubCommand::Flownode(cmd) => {
@@ -1,144 +0,0 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#![allow(clippy::print_stdout)]
use std::time::{Duration, Instant};
use api::prom_store::remote::{Label, Sample, TimeSeries, WriteRequest};
use clap::Parser;
use prost::Message;
use serde_json::json;
use servers::prom_store::snappy_compress;
#[derive(Debug, Parser)]
#[command(about = "Generate a deterministic Prometheus remote-write fixture")]
struct Args {
#[arg(long, default_value = "http://127.0.0.1:4000/v1/prometheus/write")]
endpoint: String,
#[arg(long, default_value = "public")]
database: String,
#[arg(long)]
metric: String,
#[arg(long, default_value = "greptime_physical_table")]
physical_table: String,
#[arg(long, default_value_t = 8)]
series_count: u64,
#[arg(long, default_value_t = 30)]
samples_per_series: u64,
#[arg(long, default_value_t = 1_704_067_200_000)]
start_unix_millis: i64,
#[arg(long, default_value_t = 15_000)]
step_millis: i64,
#[arg(long, alias = "batch-size", default_value_t = 8)]
chunk_series_count: u64,
#[arg(long, default_value_t = 60)]
timeout_seconds: u64,
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let args = Args::parse();
let started = Instant::now();
let client = reqwest::Client::builder()
.timeout(Duration::from_secs(args.timeout_seconds))
.build()?;
let mut batches = 0_u64;
let mut rows = 0_u64;
let mut http_statuses = Vec::new();
let chunk = args.chunk_series_count.max(1);
for first in (0..args.series_count).step_by(chunk as usize) {
let last = (first + chunk).min(args.series_count);
let timeseries = (first..last)
.map(|series_idx| make_series(&args, series_idx))
.collect::<Vec<_>>();
rows += timeseries
.iter()
.map(|ts| ts.samples.len() as u64)
.sum::<u64>();
let body = snappy_compress(
&WriteRequest {
timeseries,
..Default::default()
}
.encode_to_vec(),
)?;
let response = client
.post(&args.endpoint)
.header("Content-Encoding", "snappy")
.body(body)
.send()
.await?;
let status = response.status();
http_statuses.push(status.as_u16());
if !status.is_success() {
let body_text = response.text().await.unwrap_or_default();
return Err(format!("remote-write failed with status {status}: {body_text}").into());
}
batches += 1;
}
println!(
"{}",
json!({
"status": "ok",
"endpoint": args.endpoint,
"database": args.database,
"metric": args.metric,
"physical_table": args.physical_table,
"series_count": args.series_count,
"samples_per_series": args.samples_per_series,
"rows": rows,
"samples_written": rows,
"batches": batches,
"elapsed_seconds": started.elapsed().as_secs_f64(),
"http_statuses": http_statuses,
})
);
Ok(())
}
fn make_series(args: &Args, series_idx: u64) -> TimeSeries {
let mut labels = vec![
label("__name__", &args.metric),
label("x_greptime_database", &args.database),
label("x_greptime_physical_table", &args.physical_table),
label("host", &format!("host{:04}", series_idx % 1024)),
label("instance", &format!("instance{:06}", series_idx)),
];
labels.sort_unstable_by(|left, right| left.name.cmp(&right.name));
let samples = (0..args.samples_per_series)
.map(|sample_idx| Sample {
value: deterministic_value(series_idx, sample_idx),
timestamp: args.start_unix_millis + (sample_idx as i64 * args.step_millis),
})
.collect();
TimeSeries {
labels,
samples,
..Default::default()
}
}
fn label(name: &str, value: &str) -> Label {
Label {
name: name.to_string(),
value: value.to_string(),
}
}
fn deterministic_value(series_idx: u64, sample_idx: u64) -> f64 {
((series_idx % 97) as f64) + (sample_idx as f64 * 0.125)
}
+3 -649
View File
@@ -14,656 +14,10 @@
#![allow(clippy::print_stderr, clippy::print_stdout)]
//! Generic direct-readable-SST fixture generator for query performance labs.
//!
//! This is a bounded MVP: it reads one table/one region case TOML, builds
//! synthetic region metadata, writes real Mito readable SST parquet files, and
//! emits a manifest checkpoint plus validation metadata. The first case uses
//! `sst_format = "flat"`; primary-key output is also accepted for early lab use
//! because the writer can convert the same flat input batch to primary-key SSTs.
//!
//! TODO(query-perf): add multi-region generation and seed-manifest/catalog DB
//! integration so the generated fixture can be mounted directly by a full
//! GreptimeDB open/query path without manual metadata wiring.
use std::collections::HashMap;
use std::fs;
use std::io::{BufWriter, Write};
use std::num::{NonZeroU64, NonZeroUsize};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use api::v1::{OpType, SemanticType};
use async_trait::async_trait;
use clap::Parser;
use datatypes::arrow::array::{
ArrayRef, BinaryDictionaryBuilder, Float64Array, RecordBatch, StringDictionaryBuilder,
TimestampMillisecondArray, TimestampNanosecondArray, UInt8Array, UInt64Array,
};
use datatypes::arrow::datatypes::UInt32Type;
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{ColumnSchema, SkippingIndexOptions};
use mito_codec::row_converter::{DensePrimaryKeyCodec, PrimaryKeyCodecExt, SortField};
use mito2::access_layer::{FilePathProvider, Metrics, WriteType};
use mito2::config::IndexConfig;
use mito2::manifest::action::{RegionCheckpoint, RegionManifest, RemovedFilesRecord};
use mito2::read::FlatSource;
use mito2::sst::file::{FileMeta, RegionFileId};
use mito2::sst::index::{Indexer, IndexerBuilder};
use mito2::sst::parquet::writer::ParquetWriter;
use mito2::sst::parquet::{SstInfo, WriteOptions};
use mito2::sst::{
DEFAULT_WRITE_BUFFER_SIZE, FlatSchemaOptions, FormatType, to_flat_sst_arrow_schema,
};
use object_store::ObjectStore;
use object_store::services::Fs as FsBuilder;
use serde::Deserialize;
use store_api::codec::PrimaryKeyEncoding;
use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataRef};
use store_api::path_utils::region_name;
use store_api::region_request::PathType;
use store_api::storage::{FileId, RegionId};
#[derive(Parser, Debug)]
#[command(name = "query_perf_fixture")]
#[command(about = "Generate direct-readable SST fixtures for query performance cases")]
struct Args {
/// TOML case file, for example tests/perf/query_cases/.../case.toml.
#[arg(long, value_name = "PATH")]
case: PathBuf,
/// Output directory (creates object-store/, manifest/, files.jsonl, summary.json).
#[arg(long, value_name = "DIR")]
out_dir: PathBuf,
/// Region ID to synthesize. Defaults to table id 1024, region 0.
#[arg(long, default_value = "4398046511104")]
region_id: u64,
/// Table directory relative to object-store root. Defaults to data/{database}/{table}/.
#[arg(long, value_name = "DIR")]
table_dir: Option<String>,
/// Table name to generate when the case contains multiple [[tables]].
#[arg(long, value_name = "NAME")]
table: Option<String>,
/// Manifest/checkpoint version.
#[arg(long, default_value = "1000000")]
checkpoint_version: u64,
/// Safety flag when sst_count exceeds 1000.
#[arg(long)]
allow_large: bool,
/// Print plan only.
#[arg(long)]
dry_run: bool,
}
#[derive(Debug, Deserialize)]
struct CaseFile {
scenario: Scenario,
}
#[derive(Debug, Deserialize)]
#[serde(tag = "kind")]
enum Scenario {
#[serde(rename = "direct_readable_sst")]
DirectReadableSst(DirectReadableSstScenario),
}
#[derive(Debug, Deserialize)]
struct DirectReadableSstScenario {
#[serde(default)]
seed: Option<u64>,
tables: Vec<TableConfig>,
layout: LayoutConfig,
}
impl Scenario {
fn kind(&self) -> &'static str {
match self {
Scenario::DirectReadableSst(_) => "direct_readable_sst",
}
}
fn direct_readable_sst(&self) -> &DirectReadableSstScenario {
match self {
Scenario::DirectReadableSst(scenario) => scenario,
}
}
}
#[derive(Debug, Deserialize)]
struct TableConfig {
database: String,
name: String,
engine: String,
#[serde(default)]
append_mode: Option<bool>,
#[serde(default)]
sst_format: Option<String>,
primary_key: Vec<String>,
time_index: String,
columns: Vec<ColumnConfig>,
}
#[derive(Debug, Deserialize)]
struct ColumnConfig {
name: String,
#[serde(rename = "type")]
ty: String,
semantic: String,
distribution: Option<Distribution>,
}
#[derive(Debug, Deserialize)]
#[serde(tag = "kind")]
enum Distribution {
#[serde(rename = "cardinality")]
Cardinality {
values: NonZeroUsize,
prefix: String,
},
#[serde(rename = "deterministic_wave")]
DeterministicWave { min: f64, max: f64 },
}
#[derive(Debug, Deserialize)]
struct LayoutConfig {
regions: usize,
sst_count: usize,
rows_per_sst: usize,
row_group_size: usize,
series_count: NonZeroUsize,
start_unix_nanos: i64,
step_nanos: i64,
time_range_layout: String,
series_layout: String,
}
struct NoopIndexBuilder;
#[async_trait]
impl IndexerBuilder for NoopIndexBuilder {
async fn build(&self, _file_id: FileId, _index_version: u64) -> Indexer {
Indexer::default()
}
}
#[derive(Clone)]
struct FixedPathProvider {
table_dir: String,
}
impl FilePathProvider for FixedPathProvider {
fn build_index_file_path(&self, file_id: RegionFileId) -> String {
mito2::sst::location::index_file_path_legacy(&self.table_dir, file_id, PathType::Bare)
}
fn build_index_file_path_with_version(
&self,
index_id: mito2::sst::file::RegionIndexId,
) -> String {
mito2::sst::location::index_file_path(&self.table_dir, index_id, PathType::Bare)
}
fn build_sst_file_path(&self, file_id: RegionFileId) -> String {
mito2::sst::location::sst_file_path(&self.table_dir, file_id, PathType::Bare)
}
}
fn semantic_type(s: &str) -> SemanticType {
match s.to_ascii_lowercase().as_str() {
"tag" => SemanticType::Tag,
"field" => SemanticType::Field,
"timestamp" => SemanticType::Timestamp,
other => panic!("unsupported semantic {other}"),
}
}
fn concrete_type(ty: &str) -> ConcreteDataType {
match ty.to_ascii_uppercase().as_str() {
"STRING" => ConcreteDataType::string_datatype(),
"DOUBLE" => ConcreteDataType::float64_datatype(),
"UINT64" => ConcreteDataType::uint64_datatype(),
"TIMESTAMP(9)" => ConcreteDataType::timestamp_nanosecond_datatype(),
"TIMESTAMP(3)" => ConcreteDataType::timestamp_millisecond_datatype(),
other => panic!("unsupported column type {other}"),
}
}
fn build_region_metadata(table: &TableConfig, region_id: RegionId) -> RegionMetadata {
let mut builder = store_api::metadata::RegionMetadataBuilder::new(region_id);
for (idx, col) in table.columns.iter().enumerate() {
let mut schema = ColumnSchema::new(
&col.name,
concrete_type(&col.ty),
col.semantic != "timestamp",
);
if col.semantic == "tag" {
schema = schema.with_inverted_index(true);
}
if col.semantic == "tag" || col.semantic == "field" {
schema = schema
.with_skipping_options(SkippingIndexOptions {
granularity: 1,
..Default::default()
})
.expect("valid skipping index options for query perf fixture");
}
if col.name == table.time_index {
schema = schema.with_time_index(true);
}
builder.push_column_metadata(ColumnMetadata {
column_schema: schema,
semantic_type: semantic_type(&col.semantic),
column_id: idx as u32,
});
}
let pk_ids = table
.primary_key
.iter()
.map(|name| {
table
.columns
.iter()
.position(|c| c.name == *name)
.unwrap_or_else(|| panic!("primary key column {name} not found")) as u32
})
.collect();
builder.primary_key(pk_ids);
builder.primary_key_encoding(PrimaryKeyEncoding::Dense);
builder
.build()
.expect("region metadata should be valid for fixture table config")
}
fn encode_dense_primary_key(table: &TableConfig, values: &HashMap<String, String>) -> Vec<u8> {
let fields = table
.primary_key
.iter()
.enumerate()
.map(|(idx, _)| {
(
idx as u32,
SortField::new(ConcreteDataType::string_datatype()),
)
})
.collect();
let converter = DensePrimaryKeyCodec::with_fields(fields);
converter
.encode(
table
.primary_key
.iter()
.map(|name| datatypes::value::ValueRef::String(values[name].as_str())),
)
.expect("dense primary key encoding should match fixture primary key fields")
}
fn tag_value(col: &ColumnConfig, series: usize) -> String {
match col
.distribution
.as_ref()
.expect("tag distribution is required")
{
Distribution::Cardinality { values, prefix } => {
format!("{}{}", prefix, series % values.get())
}
_ => panic!("tag column {} requires cardinality distribution", col.name),
}
}
fn series_for_row(layout: &LayoutConfig, sst_idx: usize, base_row: usize, row: usize) -> usize {
match layout.series_layout.as_str() {
"round_robin" | "timestamp_major" => (base_row + row) % layout.series_count.get(),
"per_sst" => sst_idx % layout.series_count.get(),
other => panic!("unsupported series_layout {other}"),
}
}
fn timestamp_for_row(layout: &LayoutConfig, base_row: usize, row: usize) -> i64 {
let logical_row = base_row + row;
let timestamp_step = if layout.series_layout == "timestamp_major" {
logical_row / layout.series_count.get()
} else {
logical_row
};
layout.start_unix_nanos + (timestamp_step as i64 * layout.step_nanos)
}
fn wave_value(min: f64, max: f64, row: usize) -> f64 {
let span = max - min;
let phase = (row % 1024) as f64 / 1023.0;
min + span * (0.5 - 0.5 * (std::f64::consts::TAU * phase).cos())
}
fn generate_record_batch(
table: &TableConfig,
metadata: &RegionMetadataRef,
layout: &LayoutConfig,
sst_idx: usize,
sequence: u64,
) -> RecordBatch {
let flat_schema = to_flat_sst_arrow_schema(metadata, &FlatSchemaOptions::default());
let rows = layout.rows_per_sst;
let mut columns = Vec::new();
let base_row = sst_idx * rows;
for col in &table.columns {
match (col.semantic.as_str(), col.ty.to_ascii_uppercase().as_str()) {
("tag", "STRING") => {
let mut b = StringDictionaryBuilder::<UInt32Type>::new();
for row in 0..rows {
let series = series_for_row(layout, sst_idx, base_row, row);
let value = tag_value(col, series);
b.append_value(value);
}
columns.push(Arc::new(b.finish()) as ArrayRef);
}
("field", "DOUBLE") => {
let (min, max) = match col.distribution.as_ref() {
Some(Distribution::DeterministicWave { min, max }) => (*min, *max),
_ => (0.0, 1.0),
};
columns.push(Arc::new(Float64Array::from(
(0..rows)
.map(|r| wave_value(min, max, base_row + r))
.collect::<Vec<_>>(),
)) as ArrayRef);
}
("field", "UINT64") => columns.push(Arc::new(UInt64Array::from(
(0..rows).map(|r| (base_row + r) as u64).collect::<Vec<_>>(),
)) as ArrayRef),
("timestamp", "TIMESTAMP(9)") => columns.push(Arc::new(TimestampNanosecondArray::from(
(0..rows)
.map(|r| timestamp_for_row(layout, base_row, r))
.collect::<Vec<_>>(),
)) as ArrayRef),
("timestamp", "TIMESTAMP(3)") => {
columns.push(Arc::new(TimestampMillisecondArray::from(
(0..rows)
.map(|r| timestamp_for_row(layout, base_row, r) / 1_000_000)
.collect::<Vec<_>>(),
)) as ArrayRef)
}
other => panic!("unsupported column combination {other:?}"),
}
}
let mut pk_builder = BinaryDictionaryBuilder::<UInt32Type>::new();
for row in 0..rows {
let series = series_for_row(layout, sst_idx, base_row, row);
let values = table
.columns
.iter()
.filter(|c| c.semantic == "tag")
.map(|c| (c.name.clone(), tag_value(c, series)))
.collect();
pk_builder
.append(encode_dense_primary_key(table, &values))
.expect("append generated dense primary key to Arrow dictionary builder");
}
columns.push(Arc::new(pk_builder.finish()));
columns.push(Arc::new(UInt64Array::from_value(sequence, rows)));
columns.push(Arc::new(UInt8Array::from_value(OpType::Put as u8, rows)));
RecordBatch::try_new(flat_schema, columns)
.expect("generated fixture columns should match flat SST Arrow schema")
}
fn file_meta_from_sst_info(
info: &SstInfo,
region_id: RegionId,
file_id: FileId,
sequence: u64,
) -> FileMeta {
FileMeta {
region_id,
file_id,
time_range: info.time_range,
level: 0,
file_size: info.file_size,
max_row_group_uncompressed_size: info.max_row_group_uncompressed_size,
available_indexes: Default::default(),
indexes: Default::default(),
index_file_size: 0,
index_version: 0,
num_rows: info.num_rows as u64,
num_row_groups: info.num_row_groups,
sequence: NonZeroU64::new(sequence),
partition_expr: None,
num_series: info.num_series,
primary_key_min: None,
primary_key_max: None,
}
}
fn deterministic_file_id(seed: u64, index: usize) -> FileId {
let seed_hi = ((seed >> 16) & 0xffff) as u16;
let seed_lo = (seed & 0xffff) as u16;
FileId::parse_str(&format!(
"00000000-0000-{seed_hi:04x}-{seed_lo:04x}-{index:012x}"
))
.expect("deterministic file id must be a valid UUID")
}
fn remap_sst_file(
object_store_dir: &Path,
table_dir: &str,
region_id: RegionId,
old_file_id: FileId,
new_file_id: FileId,
) {
if old_file_id == new_file_id {
return;
}
let old_path = object_store_dir.join(mito2::sst::location::sst_file_path(
table_dir,
RegionFileId::new(region_id, old_file_id),
PathType::Bare,
));
let new_path = object_store_dir.join(mito2::sst::location::sst_file_path(
table_dir,
RegionFileId::new(region_id, new_file_id),
PathType::Bare,
));
if let Some(parent) = new_path.parent() {
fs::create_dir_all(parent).expect("failed to create remapped SST parent directory");
}
fs::rename(&old_path, &new_path).unwrap_or_else(|err| {
panic!(
"failed to remap SST file {} -> {}: {err}",
old_path.display(),
new_path.display()
)
});
}
fn case_name_from_path(path: &Path) -> String {
path.parent()
.and_then(Path::file_name)
.and_then(|name| name.to_str())
.unwrap_or("query_perf_case")
.to_string()
}
#[path = "query_perf_fixture/mod.rs"]
mod query_perf_fixture;
#[tokio::main]
async fn main() {
let args = Args::parse();
let case_text = fs::read_to_string(&args.case).expect("failed to read query perf case TOML");
let case: CaseFile = toml::from_str(&case_text).expect("failed to parse query perf case TOML");
let case_name = case_name_from_path(&args.case);
let scenario = case.scenario.direct_readable_sst();
if scenario.tables.is_empty() || scenario.layout.regions != 1 {
panic!("fixture generator supports one or more tables and exactly one region per table");
}
if scenario.layout.time_range_layout != "non_overlapping_per_sst" {
panic!("MVP supports time_range_layout=non_overlapping_per_sst");
}
if scenario.layout.series_layout == "timestamp_major"
&& !scenario
.layout
.rows_per_sst
.is_multiple_of(scenario.layout.series_count.get())
{
panic!(
"series_layout=timestamp_major requires rows_per_sst to be divisible by series_count"
);
}
if scenario.layout.sst_count > 1000 && !args.allow_large {
panic!("sst_count exceeds 1000; pass --allow-large");
}
let seed = scenario.seed.unwrap_or(0);
let table_index = match args.table.as_deref() {
Some(name) => scenario
.tables
.iter()
.position(|table| table.name == name)
.unwrap_or_else(|| panic!("--table {name} was not found in case {case_name}")),
None if scenario.tables.len() == 1 => 0,
None => panic!(
"case {} contains {} tables; pass --table <name> to choose one",
case_name,
scenario.tables.len()
),
};
let table = &scenario.tables[table_index];
if table.engine != "mito" {
panic!("MVP supports engine=mito");
}
let region_id = RegionId::from(args.region_id);
let table_dir = args
.table_dir
.unwrap_or_else(|| format!("data/{}/{}/", table.database, table.name));
if Path::new(&table_dir).is_absolute() {
panic!("--table-dir must be relative");
}
if table_dir.trim_end_matches('/').rsplit('/').next()
== Some(region_name(region_id.table_id(), region_id.region_sequence()).as_str())
{
panic!("--table-dir must be table-level directory");
}
let region_dir =
mito2::sst::location::region_dir_from_table_dir(&table_dir, region_id, PathType::Bare);
let format = match table.sst_format.as_deref().unwrap_or("flat") {
"flat" => FormatType::Flat,
"primary_key" | "primary-key" => FormatType::PrimaryKey,
other => panic!("unsupported sst_format {other}"),
};
println!(
"query_perf_fixture case={} scenario={} table={}.{} ssts={} rows_per_sst={} format={format:?}",
case_name,
case.scenario.kind(),
table.database,
table.name,
scenario.layout.sst_count,
scenario.layout.rows_per_sst
);
println!(
"out_dir={} table_dir={} region_dir={}",
args.out_dir.display(),
table_dir,
region_dir
);
if args.dry_run {
return;
}
let obj_store_dir = args.out_dir.join("object-store");
let manifest_dir = args.out_dir.join("manifest");
fs::create_dir_all(&obj_store_dir).expect("failed to create fixture object-store directory");
fs::create_dir_all(&manifest_dir).expect("failed to create fixture manifest directory");
let ostorage = ObjectStore::new(FsBuilder::default().root(&obj_store_dir.to_string_lossy()))
.expect("failed to create filesystem object store for fixture output")
.finish();
let metadata: RegionMetadataRef = Arc::new(build_region_metadata(table, region_id));
let mut files = HashMap::with_capacity(scenario.layout.sst_count);
let mut next_file_index = 1;
for i in 0..scenario.layout.sst_count {
let sequence = 1000 + i as u64;
let batch = generate_record_batch(table, &metadata, &scenario.layout, i, sequence);
let source =
FlatSource::new_iter(batch.schema(), Box::new(vec![batch].into_iter().map(Ok)));
let mut metrics = Metrics::new(WriteType::Flush);
let mut writer = ParquetWriter::new_with_object_store(
ostorage.clone(),
metadata.clone(),
IndexConfig::default(),
NoopIndexBuilder,
FixedPathProvider {
table_dir: table_dir.clone(),
},
&mut metrics,
)
.await;
let opts = WriteOptions {
write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
row_group_size: scenario.layout.row_group_size,
max_file_size: None,
};
let infos = match format {
FormatType::Flat => writer.write_all_flat(source, Some(sequence), &opts).await,
FormatType::PrimaryKey => {
writer
.write_all_flat_as_primary_key(source, Some(sequence), &opts)
.await
}
}
.expect("failed to write fixture SST through Mito parquet writer");
for info in infos {
let file_id = deterministic_file_id(
seed.wrapping_add((table_index as u64) << 16),
next_file_index,
);
next_file_index += 1;
remap_sst_file(&obj_store_dir, &table_dir, region_id, info.file_id, file_id);
files.insert(
file_id,
file_meta_from_sst_info(&info, region_id, file_id, sequence),
);
}
}
let overall_seq = files
.values()
.filter_map(|m| m.sequence)
.map(|s| s.get())
.max()
.unwrap_or(0);
let manifest = RegionManifest {
metadata,
files,
removed_files: RemovedFilesRecord::default(),
flushed_entry_id: overall_seq,
flushed_sequence: overall_seq,
committed_sequence: Some(overall_seq),
manifest_version: args.checkpoint_version,
truncated_entry_id: None,
compaction_time_window: None,
sst_format: format,
append_mode: table.append_mode,
};
let checkpoint = RegionCheckpoint {
last_version: args.checkpoint_version,
compacted_actions: manifest.files.len(),
checkpoint: Some(manifest.clone()),
};
let checkpoint_bytes = checkpoint
.encode()
.expect("failed to encode fixture region checkpoint");
let checkpoint_path = manifest_dir.join(format!("{:020}.checkpoint", args.checkpoint_version));
fs::write(&checkpoint_path, &checkpoint_bytes).expect("failed to write checkpoint file");
fs::write(manifest_dir.join("_last_checkpoint"), serde_json::to_vec_pretty(&serde_json::json!({ "size": checkpoint_bytes.len(), "version": args.checkpoint_version, "checksum": null, "extend_metadata": {} })).expect("failed to serialize _last_checkpoint metadata")).expect("failed to write _last_checkpoint file");
let files_jsonl_path = args.out_dir.join("files.jsonl");
let mut jsonl = BufWriter::new(
fs::File::create(&files_jsonl_path).expect("failed to create fixture files.jsonl"),
);
let mut entries: Vec<_> = manifest.files.iter().collect();
entries.sort_by_key(|(id, _)| id.to_string());
for (file_id, meta) in entries {
writeln!(jsonl, "{}", serde_json::to_string(&serde_json::json!({ "file_id": file_id.to_string(), "region_id": meta.region_id.as_u64(), "object_path": mito2::sst::location::sst_file_path(&table_dir, RegionFileId::new(meta.region_id, *file_id), PathType::Bare), "time_range_start": meta.time_range.0.value(), "time_range_end": meta.time_range.1.value(), "num_rows": meta.num_rows, "num_row_groups": meta.num_row_groups, "file_size": meta.file_size, "num_series": meta.num_series, "sequence": meta.sequence.map(|s| s.get()) })).expect("failed to serialize fixture SST metadata JSONL entry")).expect("failed to write fixture SST metadata JSONL entry");
}
jsonl.flush().expect("failed to flush fixture files.jsonl");
fs::write(args.out_dir.join("summary.json"), serde_json::to_vec_pretty(&serde_json::json!({ "case": case_name, "seed": seed, "table_index": table_index, "table": table.name, "database": table.database, "region_id": region_id.as_u64(), "table_dir": table_dir, "region_dir": region_dir, "sst_format": format!("{format:?}"), "sst_count": scenario.layout.sst_count, "rows_per_sst": scenario.layout.rows_per_sst, "row_group_size": scenario.layout.row_group_size, "total_rows": scenario.layout.sst_count * scenario.layout.rows_per_sst, "checkpoint_path": checkpoint_path, "files_jsonl_path": files_jsonl_path, "readback_validated": false, "metadata_source": "synthetic" })).expect("failed to serialize fixture summary")).expect("failed to write fixture summary.json");
println!("Done. wrote {} SST file entries", manifest.files.len());
query_perf_fixture::run().await;
}
+470
View File
@@ -0,0 +1,470 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use std::num::NonZeroUsize;
use clap::ValueEnum;
use serde::{Deserialize, Serialize};
#[derive(Debug, Deserialize, Serialize)]
pub(super) struct CaseFile {
pub(super) scenario: Scenario,
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(tag = "kind")]
pub(super) enum Scenario {
#[serde(rename = "direct_readable_sst")]
DirectReadableSst(DirectReadableSstScenario),
#[serde(rename = "prom_remote_write_then_query")]
PromRemoteWriteThenQuery(PromRemoteWriteThenQueryScenario),
}
#[derive(Debug, Deserialize, Serialize)]
pub(super) struct DirectReadableSstScenario {
#[serde(default)]
pub(super) seed: Option<u64>,
pub(super) tables: Vec<TableConfig>,
pub(super) layout: LayoutConfig,
}
#[derive(Debug, Deserialize, Serialize)]
pub(super) struct PromRemoteWriteThenQueryScenario {
#[serde(default)]
pub(super) queries: Vec<serde_json::Value>,
pub(super) remote_write: PromRemoteWritePlan,
}
#[derive(Debug, Deserialize, Serialize)]
pub(super) struct PromRemoteWritePlan {
#[serde(default = "default_database")]
pub(super) database: String,
#[serde(alias = "metric_name")]
pub(super) metric: String,
#[serde(default = "default_physical_table")]
pub(super) physical_table: String,
#[serde(default = "default_series_count")]
pub(super) series_count: u64,
#[serde(default = "default_samples_per_series")]
pub(super) samples_per_series: u64,
#[serde(default = "default_start_unix_millis")]
pub(super) start_unix_millis: i64,
#[serde(default = "default_step_millis")]
pub(super) step_millis: i64,
#[serde(default = "default_chunk_series_count", alias = "batch_size")]
pub(super) chunk_series_count: u64,
#[serde(default = "default_timeout_seconds")]
pub(super) timeout_seconds: u64,
#[serde(default)]
pub(super) sample_chunk_size: Option<u64>,
#[serde(default = "default_flush_every_sample_chunks")]
pub(super) flush_every_sample_chunks: u64,
#[serde(default = "default_visibility_timeout_seconds")]
pub(super) visibility_timeout_seconds: u64,
#[serde(default)]
pub(super) prom_store: PromStoreConfig,
#[serde(default)]
pub(super) value: ValueConfig,
#[serde(default)]
pub(super) storage: Option<StorageConfig>,
#[serde(default)]
pub(super) read_bench: Option<ReadBenchConfig>,
}
pub(super) fn default_database() -> String {
"public".to_string()
}
pub(super) fn default_physical_table() -> String {
"greptime_physical_table".to_string()
}
pub(super) fn default_series_count() -> u64 {
8
}
pub(super) fn default_samples_per_series() -> u64 {
30
}
pub(super) fn default_start_unix_millis() -> i64 {
1_704_067_200_000
}
pub(super) fn default_step_millis() -> i64 {
15_000
}
pub(super) fn default_chunk_series_count() -> u64 {
8
}
pub(super) fn default_timeout_seconds() -> u64 {
60
}
pub(super) fn default_flush_every_sample_chunks() -> u64 {
1
}
pub(super) fn default_visibility_timeout_seconds() -> u64 {
30
}
#[derive(Debug, Deserialize, Serialize)]
pub(super) struct PromStoreConfig {
#[serde(default = "default_pending_rows_flush_interval")]
pub(super) pending_rows_flush_interval: String,
#[serde(default = "default_max_batch_rows")]
pub(super) max_batch_rows: u64,
#[serde(default = "default_max_concurrent_flushes")]
pub(super) max_concurrent_flushes: u64,
#[serde(default = "default_worker_channel_capacity")]
pub(super) worker_channel_capacity: u64,
#[serde(default = "default_max_inflight_requests")]
pub(super) max_inflight_requests: u64,
}
impl Default for PromStoreConfig {
fn default() -> Self {
Self {
pending_rows_flush_interval: default_pending_rows_flush_interval(),
max_batch_rows: default_max_batch_rows(),
max_concurrent_flushes: default_max_concurrent_flushes(),
worker_channel_capacity: default_worker_channel_capacity(),
max_inflight_requests: default_max_inflight_requests(),
}
}
}
pub(super) fn default_pending_rows_flush_interval() -> String {
"1s".to_string()
}
pub(super) fn default_max_batch_rows() -> u64 {
100000
}
pub(super) fn default_max_concurrent_flushes() -> u64 {
256
}
pub(super) fn default_worker_channel_capacity() -> u64 {
65526
}
pub(super) fn default_max_inflight_requests() -> u64 {
3000
}
#[derive(Debug, Clone, Copy, Deserialize, Serialize, ValueEnum, Eq, PartialEq)]
#[serde(rename_all = "snake_case")]
#[value(rename_all = "snake_case")]
pub(super) enum ValuePattern {
Linear,
Constant,
Modulo,
Unique,
SeededRandom,
RunLength,
QuantizedSignal,
SignalWithSporadicStalls,
MixedSignalRepeated,
}
impl std::fmt::Display for ValuePattern {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"{}",
serde_json::to_value(self).unwrap().as_str().unwrap()
)
}
}
#[derive(Debug, Deserialize, Serialize)]
pub(super) struct ValueConfig {
#[serde(default = "default_value_pattern")]
pub(super) pattern: ValuePattern,
#[serde(default)]
pub(super) base: f64,
#[serde(default = "default_value_step")]
pub(super) step: f64,
#[serde(default = "default_value_cardinality")]
pub(super) cardinality: u64,
#[serde(default)]
pub(super) seed: u64,
#[serde(default = "default_value_run_length")]
pub(super) run_length: u64,
#[serde(default = "default_value_stall_every")]
pub(super) stall_every: u64,
#[serde(default = "default_value_stall_length")]
pub(super) stall_length: u64,
#[serde(default = "default_value_mixed_every")]
pub(super) mixed_every: u64,
}
impl Default for ValueConfig {
fn default() -> Self {
Self {
pattern: default_value_pattern(),
base: 0.0,
step: default_value_step(),
cardinality: default_value_cardinality(),
seed: 0,
run_length: default_value_run_length(),
stall_every: default_value_stall_every(),
stall_length: default_value_stall_length(),
mixed_every: default_value_mixed_every(),
}
}
}
pub(super) fn default_value_pattern() -> ValuePattern {
ValuePattern::Linear
}
pub(super) fn default_value_step() -> f64 {
0.125
}
pub(super) fn default_value_cardinality() -> u64 {
97
}
pub(super) fn default_value_run_length() -> u64 {
8
}
pub(super) fn default_value_stall_every() -> u64 {
100
}
pub(super) fn default_value_stall_length() -> u64 {
16
}
pub(super) fn default_value_mixed_every() -> u64 {
5
}
#[derive(Debug, Deserialize, Serialize)]
pub(super) struct StorageConfig {
#[serde(default = "default_true")]
pub(super) inspect: bool,
#[serde(default = "default_storage_column")]
pub(super) column: String,
#[serde(default)]
pub(super) root_suffix: Option<String>,
#[serde(default)]
pub(super) include_metadata_files: bool,
#[serde(default = "default_min_files")]
pub(super) min_files: u64,
#[serde(default = "default_min_files")]
pub(super) min_files_with_column: u64,
#[serde(default)]
pub(super) require_encodings: Vec<String>,
#[serde(default)]
pub(super) forbid_encodings: Vec<String>,
#[serde(default)]
pub(super) max_total_file_size_bytes: Option<u64>,
#[serde(default)]
pub(super) max_column_compressed_size_bytes: Option<u64>,
#[serde(default)]
pub(super) max_column_uncompressed_size_bytes: Option<u64>,
#[serde(default)]
pub(super) max_candidate_total_file_size_regression_pct: Option<f64>,
#[serde(default)]
pub(super) max_candidate_column_compressed_size_regression_pct: Option<f64>,
#[serde(default)]
pub(super) max_candidate_column_uncompressed_size_regression_pct: Option<f64>,
#[serde(skip_deserializing, default)]
pub(super) planned_thresholds: Vec<StorageThresholdPlan>,
}
#[derive(Debug, Serialize)]
pub(super) struct StorageThresholdPlan {
pub(super) threshold: String,
pub(super) status: String,
pub(super) value: serde_json::Value,
}
impl StorageConfig {
pub(super) fn populate_planned_thresholds(&mut self) {
let mut planned = Vec::new();
planned.push(StorageThresholdPlan::new("min_files", self.min_files));
planned.push(StorageThresholdPlan::new(
"min_files_with_column",
self.min_files_with_column,
));
if !self.require_encodings.is_empty() {
planned.push(StorageThresholdPlan::new(
"require_encodings",
&self.require_encodings,
));
}
if !self.forbid_encodings.is_empty() {
planned.push(StorageThresholdPlan::new(
"forbid_encodings",
&self.forbid_encodings,
));
}
macro_rules! push_optional {
($name:literal, $value:expr) => {
if let Some(value) = $value {
planned.push(StorageThresholdPlan::new($name, value));
}
};
}
push_optional!("max_total_file_size_bytes", self.max_total_file_size_bytes);
push_optional!(
"max_column_compressed_size_bytes",
self.max_column_compressed_size_bytes
);
push_optional!(
"max_column_uncompressed_size_bytes",
self.max_column_uncompressed_size_bytes
);
push_optional!(
"max_candidate_total_file_size_regression_pct",
self.max_candidate_total_file_size_regression_pct
);
push_optional!(
"max_candidate_column_compressed_size_regression_pct",
self.max_candidate_column_compressed_size_regression_pct
);
push_optional!(
"max_candidate_column_uncompressed_size_regression_pct",
self.max_candidate_column_uncompressed_size_regression_pct
);
self.planned_thresholds = planned;
}
}
impl StorageThresholdPlan {
fn new<T: Serialize>(threshold: &str, value: T) -> Self {
Self {
threshold: threshold.to_string(),
status: "planned".to_string(),
value: serde_json::to_value(value).expect("storage threshold value must serialize"),
}
}
}
pub(super) fn default_true() -> bool {
true
}
pub(super) fn default_storage_column() -> String {
"greptime_value".to_string()
}
pub(super) fn default_min_files() -> u64 {
1
}
#[derive(Debug, Deserialize, Serialize, Clone)]
pub(super) struct ReadBenchConfig {
#[serde(default = "default_true")]
pub(super) enabled: bool,
#[serde(default = "default_true")]
pub(super) parquetbench: bool,
#[serde(default = "default_true")]
pub(super) scanbench: bool,
#[serde(default = "default_iterations")]
pub(super) iterations: u64,
#[serde(default)]
pub(super) projection: Vec<String>,
#[serde(default = "default_parquet_reader")]
pub(super) parquet_reader: String,
#[serde(default = "default_scan_scanner")]
pub(super) scan_scanner: String,
#[serde(default = "default_parallelism")]
pub(super) parallelism: u64,
#[serde(default)]
pub(super) max_files: Option<usize>,
#[serde(flatten)]
pub(super) thresholds: HashMap<String, serde_json::Value>,
}
impl Default for ReadBenchConfig {
fn default() -> Self {
Self {
enabled: true,
parquetbench: true,
scanbench: true,
iterations: default_iterations(),
projection: vec![],
parquet_reader: default_parquet_reader(),
scan_scanner: default_scan_scanner(),
parallelism: default_parallelism(),
max_files: None,
thresholds: HashMap::new(),
}
}
}
pub(super) fn default_iterations() -> u64 {
7
}
pub(super) fn default_parquet_reader() -> String {
"direct".to_string()
}
pub(super) fn default_scan_scanner() -> String {
"seq".to_string()
}
pub(super) fn default_parallelism() -> u64 {
1
}
impl Scenario {
pub(super) fn kind(&self) -> &'static str {
match self {
Scenario::DirectReadableSst(_) => "direct_readable_sst",
Scenario::PromRemoteWriteThenQuery(_) => "prom_remote_write_then_query",
}
}
pub(super) fn direct_readable_sst(&self) -> &DirectReadableSstScenario {
match self {
Scenario::DirectReadableSst(scenario) => scenario,
_ => panic!("scenario is not direct_readable_sst"),
}
}
}
#[derive(Debug, Deserialize, Serialize)]
pub(super) struct TableConfig {
pub(super) database: String,
pub(super) name: String,
pub(super) engine: String,
#[serde(default)]
pub(super) append_mode: Option<bool>,
#[serde(default)]
pub(super) sst_format: Option<String>,
pub(super) primary_key: Vec<String>,
pub(super) time_index: String,
pub(super) columns: Vec<ColumnConfig>,
}
#[derive(Debug, Deserialize, Serialize)]
pub(super) struct ColumnConfig {
pub(super) name: String,
#[serde(rename = "type")]
pub(super) ty: String,
pub(super) semantic: String,
pub(super) distribution: Option<Distribution>,
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(tag = "kind")]
pub(super) enum Distribution {
#[serde(rename = "cardinality")]
Cardinality {
values: NonZeroUsize,
prefix: String,
},
#[serde(rename = "deterministic_wave")]
DeterministicWave { min: f64, max: f64 },
}
#[derive(Debug, Deserialize, Serialize)]
pub(super) struct LayoutConfig {
pub(super) regions: usize,
pub(super) sst_count: usize,
pub(super) rows_per_sst: usize,
pub(super) row_group_size: usize,
pub(super) series_count: NonZeroUsize,
pub(super) start_unix_nanos: i64,
pub(super) step_nanos: i64,
pub(super) time_range_layout: String,
pub(super) series_layout: String,
}
@@ -0,0 +1,529 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::HashMap;
use std::fs::{self};
use std::io::{BufWriter, Write};
use std::num::NonZeroU64;
use std::path::Path;
use std::sync::Arc;
use api::v1::{OpType, SemanticType};
use async_trait::async_trait;
use datatypes::arrow::array::{
ArrayRef, BinaryDictionaryBuilder, Float64Array, RecordBatch, StringDictionaryBuilder,
TimestampMillisecondArray, TimestampNanosecondArray, UInt8Array, UInt64Array,
};
use datatypes::arrow::datatypes::UInt32Type;
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{ColumnSchema, SkippingIndexOptions};
use mito_codec::row_converter::{DensePrimaryKeyCodec, PrimaryKeyCodecExt, SortField};
use mito2::access_layer::{FilePathProvider, Metrics, WriteType};
use mito2::config::IndexConfig;
use mito2::manifest::action::{RegionCheckpoint, RegionManifest, RemovedFilesRecord};
use mito2::read::FlatSource;
use mito2::sst::file::{FileMeta, RegionFileId};
use mito2::sst::index::{Indexer, IndexerBuilder};
use mito2::sst::parquet::writer::ParquetWriter;
use mito2::sst::parquet::{SstInfo, WriteOptions};
use mito2::sst::{
DEFAULT_WRITE_BUFFER_SIZE, FlatSchemaOptions, FormatType, to_flat_sst_arrow_schema,
};
use object_store::ObjectStore;
use object_store::services::Fs as FsBuilder;
use store_api::codec::PrimaryKeyEncoding;
use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataRef};
use store_api::path_utils::region_name;
use store_api::region_request::PathType;
use store_api::storage::{FileId, RegionId};
use crate::query_perf_fixture::DirectArgs;
use crate::query_perf_fixture::case::*;
use crate::query_perf_fixture::util::case_name_from_path;
struct NoopIndexBuilder;
#[async_trait]
impl IndexerBuilder for NoopIndexBuilder {
async fn build(&self, _file_id: FileId, _index_version: u64) -> Indexer {
Indexer::default()
}
}
#[derive(Clone)]
struct FixedPathProvider {
table_dir: String,
}
impl FilePathProvider for FixedPathProvider {
fn build_index_file_path(&self, file_id: RegionFileId) -> String {
mito2::sst::location::index_file_path_legacy(&self.table_dir, file_id, PathType::Bare)
}
fn build_index_file_path_with_version(
&self,
index_id: mito2::sst::file::RegionIndexId,
) -> String {
mito2::sst::location::index_file_path(&self.table_dir, index_id, PathType::Bare)
}
fn build_sst_file_path(&self, file_id: RegionFileId) -> String {
mito2::sst::location::sst_file_path(&self.table_dir, file_id, PathType::Bare)
}
}
fn semantic_type(s: &str) -> SemanticType {
match s.to_ascii_lowercase().as_str() {
"tag" => SemanticType::Tag,
"field" => SemanticType::Field,
"timestamp" => SemanticType::Timestamp,
other => panic!("unsupported semantic {other}"),
}
}
fn concrete_type(ty: &str) -> ConcreteDataType {
match ty.to_ascii_uppercase().as_str() {
"STRING" => ConcreteDataType::string_datatype(),
"DOUBLE" => ConcreteDataType::float64_datatype(),
"UINT64" => ConcreteDataType::uint64_datatype(),
"TIMESTAMP(9)" => ConcreteDataType::timestamp_nanosecond_datatype(),
"TIMESTAMP(3)" => ConcreteDataType::timestamp_millisecond_datatype(),
other => panic!("unsupported column type {other}"),
}
}
fn build_region_metadata(table: &TableConfig, region_id: RegionId) -> RegionMetadata {
let mut builder = store_api::metadata::RegionMetadataBuilder::new(region_id);
for (idx, col) in table.columns.iter().enumerate() {
let mut schema = ColumnSchema::new(
&col.name,
concrete_type(&col.ty),
col.semantic != "timestamp",
);
if col.semantic == "tag" {
schema = schema.with_inverted_index(true);
}
if col.semantic == "tag" || col.semantic == "field" {
schema = schema
.with_skipping_options(SkippingIndexOptions {
granularity: 1,
..Default::default()
})
.expect("valid skipping index options for query perf fixture");
}
if col.name == table.time_index {
schema = schema.with_time_index(true);
}
builder.push_column_metadata(ColumnMetadata {
column_schema: schema,
semantic_type: semantic_type(&col.semantic),
column_id: idx as u32,
});
}
let pk_ids = table
.primary_key
.iter()
.map(|name| {
table
.columns
.iter()
.position(|c| c.name == *name)
.unwrap_or_else(|| panic!("primary key column {name} not found")) as u32
})
.collect();
builder.primary_key(pk_ids);
builder.primary_key_encoding(PrimaryKeyEncoding::Dense);
builder
.build()
.expect("region metadata should be valid for fixture table config")
}
fn encode_dense_primary_key(table: &TableConfig, values: &HashMap<String, String>) -> Vec<u8> {
let fields = table
.primary_key
.iter()
.enumerate()
.map(|(idx, _)| {
(
idx as u32,
SortField::new(ConcreteDataType::string_datatype()),
)
})
.collect();
let converter = DensePrimaryKeyCodec::with_fields(fields);
converter
.encode(
table
.primary_key
.iter()
.map(|name| datatypes::value::ValueRef::String(values[name].as_str())),
)
.expect("dense primary key encoding should match fixture primary key fields")
}
fn tag_value(col: &ColumnConfig, series: usize) -> String {
match col
.distribution
.as_ref()
.expect("tag distribution is required")
{
Distribution::Cardinality { values, prefix } => {
format!("{}{}", prefix, series % values.get())
}
_ => panic!("tag column {} requires cardinality distribution", col.name),
}
}
fn series_for_row(layout: &LayoutConfig, sst_idx: usize, base_row: usize, row: usize) -> usize {
match layout.series_layout.as_str() {
"round_robin" | "timestamp_major" => (base_row + row) % layout.series_count.get(),
"per_sst" => sst_idx % layout.series_count.get(),
other => panic!("unsupported series_layout {other}"),
}
}
fn timestamp_for_row(layout: &LayoutConfig, base_row: usize, row: usize) -> i64 {
let logical_row = base_row + row;
let timestamp_step = if layout.series_layout == "timestamp_major" {
logical_row / layout.series_count.get()
} else {
logical_row
};
layout.start_unix_nanos + (timestamp_step as i64 * layout.step_nanos)
}
fn wave_value(min: f64, max: f64, row: usize) -> f64 {
let span = max - min;
let phase = (row % 1024) as f64 / 1023.0;
min + span * (0.5 - 0.5 * (std::f64::consts::TAU * phase).cos())
}
fn generate_record_batch(
table: &TableConfig,
metadata: &RegionMetadataRef,
layout: &LayoutConfig,
sst_idx: usize,
sequence: u64,
) -> RecordBatch {
let flat_schema = to_flat_sst_arrow_schema(metadata, &FlatSchemaOptions::default());
let rows = layout.rows_per_sst;
let mut columns = Vec::new();
let base_row = sst_idx * rows;
for col in &table.columns {
match (col.semantic.as_str(), col.ty.to_ascii_uppercase().as_str()) {
("tag", "STRING") => {
let mut b = StringDictionaryBuilder::<UInt32Type>::new();
for row in 0..rows {
let series = series_for_row(layout, sst_idx, base_row, row);
let value = tag_value(col, series);
b.append_value(value);
}
columns.push(Arc::new(b.finish()) as ArrayRef);
}
("field", "DOUBLE") => {
let (min, max) = match col.distribution.as_ref() {
Some(Distribution::DeterministicWave { min, max }) => (*min, *max),
_ => (0.0, 1.0),
};
columns.push(Arc::new(Float64Array::from(
(0..rows)
.map(|r| wave_value(min, max, base_row + r))
.collect::<Vec<_>>(),
)) as ArrayRef);
}
("field", "UINT64") => columns.push(Arc::new(UInt64Array::from(
(0..rows).map(|r| (base_row + r) as u64).collect::<Vec<_>>(),
)) as ArrayRef),
("timestamp", "TIMESTAMP(9)") => columns.push(Arc::new(TimestampNanosecondArray::from(
(0..rows)
.map(|r| timestamp_for_row(layout, base_row, r))
.collect::<Vec<_>>(),
)) as ArrayRef),
("timestamp", "TIMESTAMP(3)") => {
columns.push(Arc::new(TimestampMillisecondArray::from(
(0..rows)
.map(|r| timestamp_for_row(layout, base_row, r) / 1_000_000)
.collect::<Vec<_>>(),
)) as ArrayRef)
}
other => panic!("unsupported column combination {other:?}"),
}
}
let mut pk_builder = BinaryDictionaryBuilder::<UInt32Type>::new();
for row in 0..rows {
let series = series_for_row(layout, sst_idx, base_row, row);
let values = table
.columns
.iter()
.filter(|c| c.semantic == "tag")
.map(|c| (c.name.clone(), tag_value(c, series)))
.collect();
pk_builder
.append(encode_dense_primary_key(table, &values))
.expect("append generated dense primary key to Arrow dictionary builder");
}
columns.push(Arc::new(pk_builder.finish()));
columns.push(Arc::new(UInt64Array::from_value(sequence, rows)));
columns.push(Arc::new(UInt8Array::from_value(OpType::Put as u8, rows)));
RecordBatch::try_new(flat_schema, columns)
.expect("generated fixture columns should match flat SST Arrow schema")
}
fn file_meta_from_sst_info(
info: &SstInfo,
region_id: RegionId,
file_id: FileId,
sequence: u64,
) -> FileMeta {
FileMeta {
region_id,
file_id,
time_range: info.time_range,
level: 0,
file_size: info.file_size,
max_row_group_uncompressed_size: info.max_row_group_uncompressed_size,
available_indexes: Default::default(),
indexes: Default::default(),
index_file_size: 0,
index_version: 0,
num_rows: info.num_rows as u64,
num_row_groups: info.num_row_groups,
sequence: NonZeroU64::new(sequence),
partition_expr: None,
num_series: info.num_series,
primary_key_min: None,
primary_key_max: None,
}
}
fn deterministic_file_id(seed: u64, index: usize) -> FileId {
let seed_hi = ((seed >> 16) & 0xffff) as u16;
let seed_lo = (seed & 0xffff) as u16;
FileId::parse_str(&format!(
"00000000-0000-{seed_hi:04x}-{seed_lo:04x}-{index:012x}"
))
.expect("deterministic file id must be a valid UUID")
}
fn remap_sst_file(
object_store_dir: &Path,
table_dir: &str,
region_id: RegionId,
old_file_id: FileId,
new_file_id: FileId,
) {
if old_file_id == new_file_id {
return;
}
let old_path = object_store_dir.join(mito2::sst::location::sst_file_path(
table_dir,
RegionFileId::new(region_id, old_file_id),
PathType::Bare,
));
let new_path = object_store_dir.join(mito2::sst::location::sst_file_path(
table_dir,
RegionFileId::new(region_id, new_file_id),
PathType::Bare,
));
if let Some(parent) = new_path.parent() {
fs::create_dir_all(parent).expect("failed to create remapped SST parent directory");
}
fs::rename(&old_path, &new_path).unwrap_or_else(|err| {
panic!(
"failed to remap SST file {} -> {}: {err}",
old_path.display(),
new_path.display()
)
});
}
pub(super) async fn run_direct_sst(args: DirectArgs) {
let case_path = args.case.as_ref().expect("--case is required");
let out_dir = args.out_dir.as_ref().expect("--out-dir is required");
let case_text = fs::read_to_string(case_path).expect("failed to read query perf case TOML");
let case: CaseFile = toml::from_str(&case_text).expect("failed to parse query perf case TOML");
let case_name = case_name_from_path(case_path);
let scenario = case.scenario.direct_readable_sst();
if scenario.tables.is_empty() || scenario.layout.regions != 1 {
panic!("fixture generator supports one or more tables and exactly one region per table");
}
if scenario.layout.time_range_layout != "non_overlapping_per_sst" {
panic!("MVP supports time_range_layout=non_overlapping_per_sst");
}
if scenario.layout.series_layout == "timestamp_major"
&& !scenario
.layout
.rows_per_sst
.is_multiple_of(scenario.layout.series_count.get())
{
panic!(
"series_layout=timestamp_major requires rows_per_sst to be divisible by series_count"
);
}
if scenario.layout.sst_count > 1000 && !args.allow_large {
panic!("sst_count exceeds 1000; pass --allow-large");
}
let seed = scenario.seed.unwrap_or(0);
let table_index = match args.table.as_deref() {
Some(name) => scenario
.tables
.iter()
.position(|table| table.name == name)
.unwrap_or_else(|| panic!("--table {name} was not found in case {case_name}")),
None if scenario.tables.len() == 1 => 0,
None => panic!(
"case {} contains {} tables; pass --table <name> to choose one",
case_name,
scenario.tables.len()
),
};
let table = &scenario.tables[table_index];
if table.engine != "mito" {
panic!("MVP supports engine=mito");
}
let region_id = RegionId::from(args.region_id);
let table_dir = args
.table_dir
.unwrap_or_else(|| format!("data/{}/{}/", table.database, table.name));
if Path::new(&table_dir).is_absolute() {
panic!("--table-dir must be relative");
}
if table_dir.trim_end_matches('/').rsplit('/').next()
== Some(region_name(region_id.table_id(), region_id.region_sequence()).as_str())
{
panic!("--table-dir must be table-level directory");
}
let region_dir =
mito2::sst::location::region_dir_from_table_dir(&table_dir, region_id, PathType::Bare);
let format = match table.sst_format.as_deref().unwrap_or("flat") {
"flat" => FormatType::Flat,
"primary_key" | "primary-key" => FormatType::PrimaryKey,
other => panic!("unsupported sst_format {other}"),
};
println!(
"query_perf_fixture case={} scenario={} table={}.{} ssts={} rows_per_sst={} format={format:?}",
case_name,
case.scenario.kind(),
table.database,
table.name,
scenario.layout.sst_count,
scenario.layout.rows_per_sst
);
println!(
"out_dir={} table_dir={} region_dir={}",
out_dir.display(),
table_dir,
region_dir
);
if args.dry_run {
return;
}
let obj_store_dir = out_dir.join("object-store");
let manifest_dir = out_dir.join("manifest");
fs::create_dir_all(&obj_store_dir).expect("failed to create fixture object-store directory");
fs::create_dir_all(&manifest_dir).expect("failed to create fixture manifest directory");
let ostorage = ObjectStore::new(FsBuilder::default().root(&obj_store_dir.to_string_lossy()))
.expect("failed to create filesystem object store for fixture output")
.finish();
let metadata: RegionMetadataRef = Arc::new(build_region_metadata(table, region_id));
let mut files = HashMap::with_capacity(scenario.layout.sst_count);
let mut next_file_index = 1;
for i in 0..scenario.layout.sst_count {
let sequence = 1000 + i as u64;
let batch = generate_record_batch(table, &metadata, &scenario.layout, i, sequence);
let source =
FlatSource::new_iter(batch.schema(), Box::new(vec![batch].into_iter().map(Ok)));
let mut metrics = Metrics::new(WriteType::Flush);
let mut writer = ParquetWriter::new_with_object_store(
ostorage.clone(),
metadata.clone(),
IndexConfig::default(),
NoopIndexBuilder,
FixedPathProvider {
table_dir: table_dir.clone(),
},
&mut metrics,
)
.await;
let opts = WriteOptions {
write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
row_group_size: scenario.layout.row_group_size,
max_file_size: None,
};
let infos = match format {
FormatType::Flat => writer.write_all_flat(source, Some(sequence), &opts).await,
FormatType::PrimaryKey => {
writer
.write_all_flat_as_primary_key(source, Some(sequence), &opts)
.await
}
}
.expect("failed to write fixture SST through Mito parquet writer");
for info in infos {
let file_id = deterministic_file_id(
seed.wrapping_add((table_index as u64) << 16),
next_file_index,
);
next_file_index += 1;
remap_sst_file(&obj_store_dir, &table_dir, region_id, info.file_id, file_id);
files.insert(
file_id,
file_meta_from_sst_info(&info, region_id, file_id, sequence),
);
}
}
let overall_seq = files
.values()
.filter_map(|m| m.sequence)
.map(|s| s.get())
.max()
.unwrap_or(0);
let manifest = RegionManifest {
metadata,
files,
removed_files: RemovedFilesRecord::default(),
flushed_entry_id: overall_seq,
flushed_sequence: overall_seq,
committed_sequence: Some(overall_seq),
manifest_version: args.checkpoint_version,
truncated_entry_id: None,
compaction_time_window: None,
sst_format: format,
append_mode: table.append_mode,
};
let checkpoint = RegionCheckpoint {
last_version: args.checkpoint_version,
compacted_actions: manifest.files.len(),
checkpoint: Some(manifest.clone()),
};
let checkpoint_bytes = checkpoint
.encode()
.expect("failed to encode fixture region checkpoint");
let checkpoint_path = manifest_dir.join(format!("{:020}.checkpoint", args.checkpoint_version));
fs::write(&checkpoint_path, &checkpoint_bytes).expect("failed to write checkpoint file");
fs::write(manifest_dir.join("_last_checkpoint"), serde_json::to_vec_pretty(&serde_json::json!({ "size": checkpoint_bytes.len(), "version": args.checkpoint_version, "checksum": null, "extend_metadata": {} })).expect("failed to serialize _last_checkpoint metadata")).expect("failed to write _last_checkpoint file");
let files_jsonl_path = out_dir.join("files.jsonl");
let mut jsonl = BufWriter::new(
fs::File::create(&files_jsonl_path).expect("failed to create fixture files.jsonl"),
);
let mut entries: Vec<_> = manifest.files.iter().collect();
entries.sort_by_key(|(id, _)| id.to_string());
for (file_id, meta) in entries {
writeln!(jsonl, "{}", serde_json::to_string(&serde_json::json!({ "file_id": file_id.to_string(), "region_id": meta.region_id.as_u64(), "object_path": mito2::sst::location::sst_file_path(&table_dir, RegionFileId::new(meta.region_id, *file_id), PathType::Bare), "time_range_start": meta.time_range.0.value(), "time_range_end": meta.time_range.1.value(), "num_rows": meta.num_rows, "num_row_groups": meta.num_row_groups, "file_size": meta.file_size, "num_series": meta.num_series, "sequence": meta.sequence.map(|s| s.get()) })).expect("failed to serialize fixture SST metadata JSONL entry")).expect("failed to write fixture SST metadata JSONL entry");
}
jsonl.flush().expect("failed to flush fixture files.jsonl");
fs::write(out_dir.join("summary.json"), serde_json::to_vec_pretty(&serde_json::json!({ "case": case_name, "seed": seed, "table_index": table_index, "table": table.name, "database": table.database, "region_id": region_id.as_u64(), "table_dir": table_dir, "region_dir": region_dir, "sst_format": format!("{format:?}"), "sst_count": scenario.layout.sst_count, "rows_per_sst": scenario.layout.rows_per_sst, "row_group_size": scenario.layout.row_group_size, "total_rows": scenario.layout.sst_count * scenario.layout.rows_per_sst, "checkpoint_path": checkpoint_path, "files_jsonl_path": files_jsonl_path, "readback_validated": false, "metadata_source": "synthetic" })).expect("failed to serialize fixture summary")).expect("failed to write fixture summary.json");
println!("Done. wrote {} SST file entries", manifest.files.len());
}
@@ -0,0 +1,181 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::BTreeSet;
use std::fs::{self, File};
use std::path::{Path, PathBuf};
use clap::Args as ClapArgs;
use parquet::file::reader::{FileReader, SerializedFileReader};
use serde::Serialize;
#[derive(Debug, ClapArgs)]
pub(super) struct InspectFooterArgs {
#[arg(long)]
root: PathBuf,
#[arg(long, default_value = "greptime_value")]
column: String,
#[arg(long)]
include_metadata_files: bool,
}
#[derive(Debug, Serialize)]
struct FooterReport {
root: String,
summary: FooterSummary,
files: Vec<FooterFileReport>,
}
#[derive(Debug, Default, Serialize)]
struct FooterSummary {
column: String,
file_count: usize,
files_with_column: usize,
total_file_size: u64,
total_rows: i64,
column_compressed_size: i64,
column_uncompressed_size: i64,
unique_encodings: Vec<String>,
}
#[derive(Debug, Serialize)]
struct FooterFileReport {
path: String,
relative_path: String,
file_size: u64,
num_rows: i64,
num_row_groups: usize,
columns: Vec<FooterColumnChunkReport>,
}
#[derive(Debug, Serialize)]
struct FooterColumnChunkReport {
row_group_index: usize,
column_index: usize,
column_path: String,
encodings: Vec<String>,
compression: String,
compressed_size: i64,
uncompressed_size: i64,
num_values: i64,
}
pub(super) fn run_inspect_footer(
args: InspectFooterArgs,
) -> Result<(), Box<dyn std::error::Error>> {
let files = collect_parquet_files(&args.root, args.include_metadata_files)?;
let reports = files
.iter()
.map(|p| inspect_file(&args.root, p, &args.column))
.collect::<Result<Vec<_>, _>>()?;
println!(
"{}",
serde_json::to_string_pretty(&FooterReport {
root: args.root.display().to_string(),
summary: summarize_footer(&args.column, &reports),
files: reports
})?
);
Ok(())
}
fn collect_parquet_files(
root: &Path,
include_metadata_files: bool,
) -> Result<Vec<PathBuf>, Box<dyn std::error::Error>> {
let mut files = Vec::new();
collect_parquet_files_inner(root, root, include_metadata_files, &mut files)?;
files.sort();
Ok(files)
}
fn collect_parquet_files_inner(
root: &Path,
path: &Path,
include_metadata_files: bool,
files: &mut Vec<PathBuf>,
) -> Result<(), Box<dyn std::error::Error>> {
if path.is_dir() {
for entry in fs::read_dir(path)? {
collect_parquet_files_inner(root, &entry?.path(), include_metadata_files, files)?;
}
} else if path.extension().is_some_and(|ext| ext == "parquet")
&& (include_metadata_files
|| !path
.strip_prefix(root)
.unwrap_or(path)
.components()
.any(|c| c.as_os_str() == "metadata"))
{
files.push(path.to_path_buf());
}
Ok(())
}
fn inspect_file(
root: &Path,
path: &Path,
column: &str,
) -> Result<FooterFileReport, Box<dyn std::error::Error>> {
let file_size = fs::metadata(path)?.len();
let reader = SerializedFileReader::new(File::open(path)?)?;
let metadata = reader.metadata().file_metadata();
let row_groups = reader.metadata().row_groups();
let mut columns = Vec::new();
for (row_group_index, rg) in row_groups.iter().enumerate() {
for (column_index, chunk) in rg.columns().iter().enumerate() {
let column_path = chunk.column_path().string();
if column_path == column {
columns.push(FooterColumnChunkReport {
row_group_index,
column_index,
column_path,
encodings: chunk.encodings().map(|e| format!("{e:?}")).collect(),
compression: format!("{:?}", chunk.compression()),
compressed_size: chunk.compressed_size(),
uncompressed_size: chunk.uncompressed_size(),
num_values: chunk.num_values(),
});
}
}
}
Ok(FooterFileReport {
path: path.display().to_string(),
relative_path: path
.strip_prefix(root)
.unwrap_or(path)
.display()
.to_string(),
file_size,
num_rows: metadata.num_rows(),
num_row_groups: row_groups.len(),
columns,
})
}
fn summarize_footer(column: &str, files: &[FooterFileReport]) -> FooterSummary {
let mut encodings = BTreeSet::new();
let mut s = FooterSummary {
column: column.to_string(),
file_count: files.len(),
..Default::default()
};
for file in files {
s.total_file_size += file.file_size;
s.total_rows += file.num_rows;
if !file.columns.is_empty() {
s.files_with_column += 1;
}
for c in &file.columns {
s.column_compressed_size += c.compressed_size;
s.column_uncompressed_size += c.uncompressed_size;
encodings.extend(c.encodings.iter().cloned());
}
}
s.unique_encodings = encodings.into_iter().collect();
s
}
+133
View File
@@ -0,0 +1,133 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
mod case;
mod direct_sst;
mod inspect_footer;
mod prom_remote_write;
mod util;
use std::fs;
use std::path::PathBuf;
use case::*;
use clap::{Args as ClapArgs, Parser, Subcommand};
use direct_sst::run_direct_sst;
use inspect_footer::{InspectFooterArgs, run_inspect_footer};
use prom_remote_write::{PromRemoteWriteArgs, run_prom_remote_write};
use serde_json::json;
#[derive(Parser, Debug)]
#[command(name = "query_perf_fixture")]
#[command(about = "Query performance fixture helper")]
struct Args {
#[command(subcommand)]
command: Option<Command>,
#[command(flatten)]
legacy: DirectArgs,
}
#[derive(Subcommand, Debug)]
enum Command {
DirectSst(DirectArgs),
PromRemoteWrite(PromRemoteWriteArgs),
InspectFooter(InspectFooterArgs),
Plan(PlanArgs),
}
#[derive(ClapArgs, Debug, Clone)]
struct DirectArgs {
/// TOML case file, for example tests/perf/query_cases/.../case.toml.
#[arg(long, value_name = "PATH")]
case: Option<PathBuf>,
/// Output directory (creates object-store/, manifest/, files.jsonl, summary.json).
#[arg(long, value_name = "DIR")]
out_dir: Option<PathBuf>,
/// Region ID to synthesize. Defaults to table id 1024, region 0.
#[arg(long, default_value = "4398046511104")]
region_id: u64,
/// Table directory relative to object-store root. Defaults to data/{database}/{table}/.
#[arg(long, value_name = "DIR")]
table_dir: Option<String>,
/// Table name to generate when the case contains multiple [[tables]].
#[arg(long, value_name = "NAME")]
table: Option<String>,
/// Manifest/checkpoint version.
#[arg(long, default_value = "1000000")]
checkpoint_version: u64,
/// Safety flag when sst_count exceeds 1000.
#[arg(long)]
allow_large: bool,
/// Print plan only.
#[arg(long)]
dry_run: bool,
}
#[derive(ClapArgs, Debug)]
struct PlanArgs {
#[arg(long, value_name = "PATH")]
case: PathBuf,
}
pub async fn run() {
let args = Args::parse();
match args.command {
Some(Command::DirectSst(direct)) => run_direct_sst(direct).await,
Some(Command::PromRemoteWrite(rw)) => run_prom_remote_write(rw)
.await
.expect("prom remote write failed"),
Some(Command::InspectFooter(inspect)) => {
run_inspect_footer(inspect).expect("inspect footer failed")
}
Some(Command::Plan(plan)) => run_plan(plan).expect("plan failed"),
None => run_direct_sst(args.legacy).await,
}
}
fn run_plan(args: PlanArgs) -> Result<(), Box<dyn std::error::Error>> {
let case_text = fs::read_to_string(&args.case)?;
let mut case: CaseFile = toml::from_str(&case_text)?;
if let Scenario::PromRemoteWriteThenQuery(s) = &mut case.scenario
&& let Some(storage) = &mut s.remote_write.storage
{
storage.populate_planned_thresholds();
if storage.inspect {
let mut rb = s.remote_write.read_bench.clone().unwrap_or_default();
if rb.enabled && rb.projection.is_empty() {
rb.projection = vec![storage.column.clone()];
}
s.remote_write.read_bench = Some(rb);
} else if s
.remote_write
.read_bench
.as_ref()
.is_some_and(|rb| rb.enabled)
{
return Err("scenario.remote_write.read_bench requires scenario.remote_write.storage.inspect = true".into());
}
}
println!(
"{}",
serde_json::to_string_pretty(&json!({"schema_version": 1, "scenario": case.scenario}))?
);
Ok(())
}
@@ -0,0 +1,333 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use clap::Args as ClapArgs;
use prost::Message;
use serde_json::json;
use servers::prom_store::snappy_compress;
use crate::query_perf_fixture::case::ValuePattern;
#[derive(Debug, ClapArgs)]
pub(super) struct PromRemoteWriteArgs {
#[arg(long, default_value = "http://127.0.0.1:4000/v1/prometheus/write")]
endpoint: String,
#[arg(long, default_value = "public")]
database: String,
#[arg(long)]
metric: String,
#[arg(long, default_value = "greptime_physical_table")]
physical_table: String,
#[arg(long, default_value_t = 8)]
series_count: u64,
#[arg(long, default_value_t = 30)]
samples_per_series: u64,
#[arg(long, default_value_t = 1_704_067_200_000)]
start_unix_millis: i64,
#[arg(long, default_value_t = 15_000)]
step_millis: i64,
#[arg(long, alias = "batch-size", default_value_t = 8)]
chunk_series_count: u64,
#[arg(long, default_value_t = 60)]
timeout_seconds: u64,
#[arg(long, default_value_t = ValuePattern::Linear)]
value_pattern: ValuePattern,
#[arg(long, default_value_t = 0.0)]
value_base: f64,
#[arg(long, default_value_t = 0.125)]
value_step: f64,
#[arg(long, default_value_t = 97)]
value_cardinality: u64,
#[arg(long, default_value_t = 0)]
value_seed: u64,
#[arg(long, default_value_t = 8)]
value_run_length: u64,
#[arg(long, default_value_t = 100)]
value_stall_every: u64,
#[arg(long, default_value_t = 16)]
value_stall_length: u64,
#[arg(long, default_value_t = 5)]
value_mixed_every: u64,
#[arg(long, default_value_t = 0)]
value_sample_offset: u64,
#[arg(long)]
value_total_samples_per_series: Option<u64>,
}
pub(super) async fn run_prom_remote_write(
args: PromRemoteWriteArgs,
) -> Result<(), Box<dyn std::error::Error>> {
use api::prom_store::remote::WriteRequest;
let started = std::time::Instant::now();
let client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(args.timeout_seconds))
.build()?;
let mut batches = 0_u64;
let mut rows = 0_u64;
let mut http_statuses = Vec::new();
for first in (0..args.series_count).step_by(args.chunk_series_count.max(1) as usize) {
let last = (first + args.chunk_series_count.max(1)).min(args.series_count);
let timeseries = (first..last)
.map(|series_idx| make_prom_series(&args, series_idx))
.collect::<Vec<_>>();
rows += timeseries
.iter()
.map(|ts| ts.samples.len() as u64)
.sum::<u64>();
let body = snappy_compress(
&WriteRequest {
timeseries,
..Default::default()
}
.encode_to_vec(),
)?;
let response = client
.post(&args.endpoint)
.header("Content-Encoding", "snappy")
.body(body)
.send()
.await?;
let status = response.status();
http_statuses.push(status.as_u16());
if !status.is_success() {
return Err(format!(
"remote-write failed with status {status}: {}",
response.text().await.unwrap_or_default()
)
.into());
}
batches += 1;
}
println!(
"{}",
json!({"status":"ok","endpoint":args.endpoint,"database":args.database,"metric":args.metric,"physical_table":args.physical_table,"series_count":args.series_count,"samples_per_series":args.samples_per_series,"rows":rows,"samples_written":rows,"batches":batches,"elapsed_seconds":started.elapsed().as_secs_f64(),"http_statuses":http_statuses,"value":{"pattern":args.value_pattern.to_string(),"base":args.value_base,"step":args.value_step,"cardinality":args.value_cardinality,"seed":args.value_seed,"run_length":args.value_run_length,"stall_every":args.value_stall_every,"stall_length":args.value_stall_length,"mixed_every":args.value_mixed_every,"sample_offset":args.value_sample_offset,"total_samples_per_series":args.value_total_samples_per_series.unwrap_or(args.samples_per_series)}})
);
Ok(())
}
fn make_prom_series(
args: &PromRemoteWriteArgs,
series_idx: u64,
) -> api::prom_store::remote::TimeSeries {
let mut labels = vec![
prom_label("__name__", &args.metric),
prom_label("x_greptime_database", &args.database),
prom_label("x_greptime_physical_table", &args.physical_table),
prom_label("host", &format!("host{:04}", series_idx % 1024)),
prom_label("instance", &format!("instance{:06}", series_idx)),
];
labels.sort_unstable_by(|l, r| l.name.cmp(&r.name));
let samples = (0..args.samples_per_series)
.map(|sample_idx| api::prom_store::remote::Sample {
value: deterministic_prom_value(args, series_idx, sample_idx),
timestamp: args.start_unix_millis + (sample_idx as i64 * args.step_millis),
})
.collect();
api::prom_store::remote::TimeSeries {
labels,
samples,
..Default::default()
}
}
fn prom_label(name: &str, value: &str) -> api::prom_store::remote::Label {
api::prom_store::remote::Label {
name: name.to_string(),
value: value.to_string(),
}
}
fn deterministic_prom_value(args: &PromRemoteWriteArgs, series_idx: u64, sample_idx: u64) -> f64 {
let ordinal = series_idx
* args
.value_total_samples_per_series
.unwrap_or(args.samples_per_series)
+ args.value_sample_offset
+ sample_idx;
let local = args.value_sample_offset + sample_idx;
match args.value_pattern {
ValuePattern::Linear => {
args.value_base + (series_idx % 97) as f64 + local as f64 * args.value_step
}
ValuePattern::Constant => args.value_base,
ValuePattern::Modulo => {
args.value_base + (ordinal % args.value_cardinality.max(1)) as f64 * args.value_step
}
ValuePattern::Unique => args.value_base + ordinal as f64 * args.value_step,
ValuePattern::SeededRandom => {
args.value_base
+ (splitmix64(ordinal ^ args.value_seed) % args.value_cardinality.max(1)) as f64
* args.value_step
}
ValuePattern::RunLength => {
args.value_base
+ ((ordinal / args.value_run_length.max(1)) % args.value_cardinality.max(1)) as f64
* args.value_step
}
ValuePattern::QuantizedSignal => {
args.value_base
+ (((series_idx % args.value_cardinality.max(1))
+ (local / args.value_run_length.max(1)))
% args.value_cardinality.max(1)) as f64
* args.value_step
}
ValuePattern::SignalWithSporadicStalls => {
let every = args.value_stall_every.max(1);
let phase = local % every;
let effective = if phase < args.value_stall_length.min(every) {
local - phase
} else {
local
};
args.value_base + (series_idx % 97) as f64 + effective as f64 * args.value_step
}
ValuePattern::MixedSignalRepeated => {
if local.is_multiple_of(args.value_mixed_every.max(1)) {
args.value_base
} else {
args.value_base + (series_idx % 97) as f64 + local as f64 * args.value_step
}
}
}
}
fn splitmix64(mut value: u64) -> u64 {
value = value.wrapping_add(0x9e3779b97f4a7c15);
let mut mixed = value;
mixed = (mixed ^ (mixed >> 30)).wrapping_mul(0xbf58476d1ce4e5b9);
mixed = (mixed ^ (mixed >> 27)).wrapping_mul(0x94d049bb133111eb);
mixed ^ (mixed >> 31)
}
#[cfg(test)]
mod tests {
use std::collections::HashSet;
use super::*;
fn args(pattern: ValuePattern) -> PromRemoteWriteArgs {
PromRemoteWriteArgs {
endpoint: "http://127.0.0.1:4000/v1/prometheus/write".to_string(),
database: "public".to_string(),
metric: "test_metric".to_string(),
physical_table: "greptime_physical_table".to_string(),
series_count: 2,
samples_per_series: 64,
start_unix_millis: 1_704_067_200_000,
step_millis: 60_000,
chunk_series_count: 2,
timeout_seconds: 60,
value_pattern: pattern,
value_base: 0.0,
value_step: 1.0,
value_cardinality: 1_024,
value_seed: 8_444,
value_run_length: 16,
value_stall_every: 100,
value_stall_length: 16,
value_mixed_every: 5,
value_sample_offset: 0,
value_total_samples_per_series: None,
}
}
#[test]
fn seeded_random_is_deterministic_in_range_and_highly_distinct() {
let mut args = args(ValuePattern::SeededRandom);
args.samples_per_series = 4_096;
args.value_cardinality = 29_491_200;
let values = (0..args.samples_per_series)
.map(|sample_idx| deterministic_prom_value(&args, 0, sample_idx))
.collect::<Vec<_>>();
let repeated = (0..args.samples_per_series)
.map(|sample_idx| deterministic_prom_value(&args, 0, sample_idx))
.collect::<Vec<_>>();
assert_eq!(values, repeated);
assert!(values.iter().all(|value| {
(args.value_base..=args.value_base + (args.value_cardinality - 1) as f64)
.contains(value)
}));
assert!(
values
.iter()
.map(|value| value.to_bits())
.collect::<HashSet<_>>()
.len()
> 4_000
);
}
#[test]
fn run_length_values_have_exact_sixteen_sample_runs() {
let args = args(ValuePattern::RunLength);
for sample_idx in 0..64 {
assert_eq!(
deterministic_prom_value(&args, 0, sample_idx),
(sample_idx / 16) as f64
);
}
}
#[test]
fn mixed_every_five_repeats_only_the_expected_positions() {
let mut args = args(ValuePattern::MixedSignalRepeated);
args.value_step = 0.125;
for sample_idx in 0..20 {
let expected = if sample_idx % 5 == 0 {
0.0
} else {
1.0 + sample_idx as f64 * 0.125
};
assert_eq!(deterministic_prom_value(&args, 1, sample_idx), expected);
}
}
#[test]
fn chunked_values_match_monolithic_values_through_offsets() {
let mut monolithic = args(ValuePattern::SeededRandom);
monolithic.samples_per_series = 14_400;
monolithic.value_total_samples_per_series = Some(14_400);
for series_idx in 0..2 {
let expected = (0..monolithic.samples_per_series)
.map(|sample_idx| deterministic_prom_value(&monolithic, series_idx, sample_idx))
.collect::<Vec<_>>();
for chunk_idx in 0..10 {
let mut chunk = args(ValuePattern::SeededRandom);
chunk.samples_per_series = 1_440;
chunk.value_sample_offset = chunk_idx * 1_440;
chunk.value_total_samples_per_series = Some(14_400);
for sample_idx in 0..chunk.samples_per_series {
assert_eq!(
deterministic_prom_value(&chunk, series_idx, sample_idx),
expected[(chunk.value_sample_offset + sample_idx) as usize]
);
}
}
}
}
#[test]
fn values_continue_across_series_boundaries() {
let mut args = args(ValuePattern::Unique);
args.samples_per_series = 16;
args.value_total_samples_per_series = Some(16);
args.value_base = 1.0;
args.value_step = 0.5;
assert_eq!(deterministic_prom_value(&args, 0, 15), 8.5);
assert_eq!(deterministic_prom_value(&args, 1, 0), 9.0);
}
}
@@ -0,0 +1,23 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::path::Path;
pub(super) fn case_name_from_path(path: &Path) -> String {
path.parent()
.and_then(Path::file_name)
.and_then(|name| name.to_str())
.unwrap_or("query_perf_case")
.to_string()
}
+2 -7
View File
@@ -15,11 +15,10 @@
pub mod builder;
#[allow(clippy::print_stdout)]
pub(crate) mod objbench;
#[cfg(feature = "dev-tools")]
#[allow(clippy::print_stdout)]
mod parquetbench;
pub mod parquetbench;
#[allow(clippy::print_stdout)]
mod scanbench;
pub mod scanbench;
use std::path::Path;
use std::time::Duration;
@@ -40,7 +39,6 @@ use tracing_appender::non_blocking::WorkerGuard;
use crate::App;
use crate::datanode::builder::InstanceBuilder;
use crate::datanode::objbench::ObjbenchCommand;
#[cfg(feature = "dev-tools")]
use crate::datanode::parquetbench::ParquetbenchCommand;
use crate::datanode::scanbench::ScanbenchCommand;
use crate::error::{
@@ -115,7 +113,6 @@ impl Command {
SubCommand::Start(cmd) => cmd.load_options(global_options),
// Bench commands are standalone utilities and don't need to load DatanodeOptions.
SubCommand::Objbench(_) | SubCommand::Scanbench(_) => Self::default_bench_options(),
#[cfg(feature = "dev-tools")]
SubCommand::Parquetbench(_) => Self::default_bench_options(),
}
}
@@ -141,7 +138,6 @@ pub enum SubCommand {
/// Scan benchmark tool - benchmarks scanning a region directly from storage
Scanbench(ScanbenchCommand),
/// Benchmark scanning a single parquet SST.
#[cfg(feature = "dev-tools")]
Parquetbench(ParquetbenchCommand),
}
@@ -160,7 +156,6 @@ impl SubCommand {
cmd.run().await?;
std::process::exit(0);
}
#[cfg(feature = "dev-tools")]
SubCommand::Parquetbench(cmd) => {
cmd.run().await?;
std::process::exit(0);
+43
View File
@@ -12,11 +12,17 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use std::sync::atomic::Ordering;
use common_base::Plugins;
use store_api::region_engine::RegionEngine;
use store_api::region_request::{RegionCloseRequest, RegionRequest};
use store_api::storage::RegionId;
use crate::config::MitoConfig;
use crate::engine::flush_test::MockRegionHook;
use crate::engine::region_hook::RegionHookRef;
use crate::test_util::{CreateRequestBuilder, TestEnv};
#[tokio::test]
@@ -60,3 +66,40 @@ async fn test_engine_close_region_with_format(flat_format: bool) {
.await
.unwrap();
}
#[tokio::test]
async fn test_region_hook_on_close() {
common_telemetry::init_default_ut_logging();
let mut env = TestEnv::new().await;
let hook = Arc::new(MockRegionHook::new());
let plugins = Plugins::new();
plugins.insert(hook.clone() as RegionHookRef);
let engine = env
.create_engine_with_plugins(MitoConfig::default(), plugins)
.await;
let region_id = RegionId::new(1, 1);
let request = CreateRequestBuilder::new().build();
engine
.handle_request(region_id, RegionRequest::Create(request))
.await
.unwrap();
// Sanity: no lifecycle events before closing besides the create open.
assert_eq!(hook.opened_count.load(Ordering::Relaxed), 1);
assert_eq!(hook.closed_count.load(Ordering::Relaxed), 0);
assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 0);
engine
.handle_request(region_id, RegionRequest::Close(RegionCloseRequest {}))
.await
.unwrap();
// Closing fires on_region_closed exactly once.
assert_eq!(hook.closed_count.load(Ordering::Relaxed), 1);
// Close must not be confused with drop.
assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 0);
assert_eq!(hook.files_removed_count.load(Ordering::Relaxed), 0);
}
+76
View File
@@ -13,9 +13,11 @@
// limitations under the License.
use std::sync::Arc;
use std::sync::atomic::Ordering;
use std::time::Duration;
use api::v1::Rows;
use common_base::Plugins;
use common_meta::key::SchemaMetadataManager;
use common_meta::kv_backend::KvBackendRef;
use object_store::util::join_path;
@@ -25,7 +27,9 @@ use store_api::storage::RegionId;
use crate::config::MitoConfig;
use crate::engine::MitoEngine;
use crate::engine::flush_test::MockRegionHook;
use crate::engine::listener::DropListener;
use crate::engine::region_hook::RegionHookRef;
use crate::test_util::{
CreateRequestBuilder, TestEnv, build_rows_for_key, flush_region, put_rows, rows_schema,
};
@@ -302,3 +306,75 @@ async fn test_engine_drop_region_for_custom_store_with_format(flat_format: bool)
.unwrap()
);
}
#[tokio::test]
async fn test_region_hook_on_drop() {
common_telemetry::init_default_ut_logging();
let mut env = TestEnv::with_prefix("drop_hook").await;
let hook = Arc::new(MockRegionHook::new());
let plugins = Plugins::new();
plugins.insert(hook.clone() as RegionHookRef);
let engine = env
.create_engine_with_plugins(MitoConfig::default(), plugins)
.await;
let region_id = RegionId::new(1, 1);
env.get_schema_metadata_manager()
.register_region_table_info(
region_id.table_id(),
"test_table",
"test_catalog",
"test_schema",
None,
env.get_kv_backend(),
)
.await;
let request = CreateRequestBuilder::new().build();
let column_schemas = rows_schema(&request);
engine
.handle_request(region_id, RegionRequest::Create(request))
.await
.unwrap();
// Put some rows so the region has active data (kept in memtable; we
// intentionally do not flush so the drop GC worker finds no parquet files
// and removes the directory immediately, avoiding the multi-minute GC wait).
let rows = Rows {
schema: column_schemas,
rows: build_rows_for_key("a", 0, 2, 0),
};
put_rows(&engine, region_id, rows).await;
// Sanity: nothing fired yet besides the open from create.
assert_eq!(hook.opened_count.load(Ordering::Relaxed), 1);
assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 0);
assert_eq!(hook.closed_count.load(Ordering::Relaxed), 0);
// full drop (partial_drop = false) so the GC worker physically deletes the
// directory and fires on_region_files_removed.
engine
.handle_request(
region_id,
RegionRequest::Drop(RegionDropRequest {
fast_path: false,
force: false,
partial_drop: false,
}),
)
.await
.unwrap();
// on_region_dropped fires inline while handling the drop request.
hook.wait_for_dropped().await;
assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 1);
// Drop must not be confused with close.
assert_eq!(hook.closed_count.load(Ordering::Relaxed), 0);
// on_region_files_removed fires from the background GC worker once the
// directory is physically deleted.
hook.wait_for_files_removed().await;
assert_eq!(hook.files_removed_count.load(Ordering::Relaxed), 1);
}
+55 -4
View File
@@ -895,24 +895,46 @@ async fn test_update_topic_latest_entry_id(factory: Option<LogStoreFactory>) {
}
#[derive(Debug)]
struct MockRegionHook {
sst_written_count: AtomicUsize,
manifest_updated_count: AtomicUsize,
pub(super) struct MockRegionHook {
pub(super) sst_written_count: AtomicUsize,
pub(super) manifest_updated_count: AtomicUsize,
pub(super) opened_count: AtomicUsize,
pub(super) closed_count: AtomicUsize,
pub(super) dropped_count: AtomicUsize,
pub(super) files_removed_count: AtomicUsize,
notify: Notify,
dropped_notify: Notify,
files_removed_notify: Notify,
}
impl MockRegionHook {
fn new() -> Self {
pub(super) fn new() -> Self {
Self {
sst_written_count: AtomicUsize::new(0),
manifest_updated_count: AtomicUsize::new(0),
opened_count: AtomicUsize::new(0),
closed_count: AtomicUsize::new(0),
dropped_count: AtomicUsize::new(0),
files_removed_count: AtomicUsize::new(0),
notify: Notify::new(),
dropped_notify: Notify::new(),
files_removed_notify: Notify::new(),
}
}
async fn wait_for_manifest_update(&self) {
self.notify.notified().await;
}
/// Waits until `on_region_dropped` has fired at least once.
pub(super) async fn wait_for_dropped(&self) {
self.dropped_notify.notified().await;
}
/// Waits until `on_region_files_removed` has fired at least once.
pub(super) async fn wait_for_files_removed(&self) {
self.files_removed_notify.notified().await;
}
}
#[async_trait]
@@ -966,6 +988,35 @@ impl RegionHook for MockRegionHook {
);
self.notify.notify_one();
}
async fn on_region_opened(&self, region_id: RegionId, _region_metadata: &RegionMetadataRef) {
self.opened_count.fetch_add(1, Ordering::Relaxed);
common_telemetry::info!("MockRegionHook::on_region_opened: region={}", region_id);
}
async fn on_region_closed(&self, region_id: RegionId, _region_metadata: &RegionMetadataRef) {
self.closed_count.fetch_add(1, Ordering::Relaxed);
common_telemetry::info!("MockRegionHook::on_region_closed: region={}", region_id);
}
async fn on_region_dropped(&self, region_id: RegionId, _region_metadata: &RegionMetadataRef) {
self.dropped_count.fetch_add(1, Ordering::Relaxed);
common_telemetry::info!("MockRegionHook::on_region_dropped: region={}", region_id);
self.dropped_notify.notify_one();
}
async fn on_region_files_removed(
&self,
region_id: RegionId,
_region_metadata: &RegionMetadataRef,
) {
self.files_removed_count.fetch_add(1, Ordering::Relaxed);
common_telemetry::info!(
"MockRegionHook::on_region_files_removed: region={}",
region_id
);
self.files_removed_notify.notify_one();
}
}
#[tokio::test]
+35
View File
@@ -32,6 +32,8 @@ use tokio::sync::oneshot;
use crate::compaction::compactor::{OpenCompactionRegionRequest, open_compaction_region};
use crate::config::MitoConfig;
use crate::engine::flush_test::MockRegionHook;
use crate::engine::region_hook::RegionHookRef;
use crate::error;
use crate::region::opener::{PartitionExprFetcher, PartitionExprFetcherRef};
use crate::region::options::RegionOptions;
@@ -805,3 +807,36 @@ async fn test_open_keeps_none_without_fetcher() {
let meta = engine.get_region(region_id).unwrap().metadata();
assert!(meta.partition_expr.is_none());
}
#[tokio::test]
async fn test_region_hook_on_open() {
common_telemetry::init_default_ut_logging();
let mut env = TestEnv::with_prefix("open_hook").await;
let hook = Arc::new(MockRegionHook::new());
let plugins = Plugins::new();
plugins.insert(hook.clone() as RegionHookRef);
let engine = env
.create_engine_with_plugins(MitoConfig::default(), plugins)
.await;
let region_id = RegionId::new(1, 1);
let request = CreateRequestBuilder::new().build();
let table_dir = request.table_dir.clone();
engine
.handle_request(region_id, RegionRequest::Create(request))
.await
.unwrap();
// Creating a region fires on_region_opened exactly once.
assert_eq!(hook.opened_count.load(Ordering::Relaxed), 1);
assert_eq!(hook.closed_count.load(Ordering::Relaxed), 0);
// `reopen_region` closes then opens; the open must fire the hook again.
reopen_region(&engine, region_id, table_dir, false, Default::default()).await;
assert_eq!(hook.opened_count.load(Ordering::Relaxed), 2);
assert_eq!(hook.closed_count.load(Ordering::Relaxed), 1);
assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 0);
}
+171 -17
View File
@@ -16,15 +16,24 @@
//!
//! ## Design
//!
//! The [`RegionHook`] trait provides two methods with clear separation of concerns:
//! The [`RegionHook`] trait observes region activity through two categories of
//! callbacks — manifest/file observation and region lifecycle:
//!
//! - [`on_sst_files_written`]: Fires when mito2 physically writes SST **data files**.
//! Provides per-file [`SstInfo`] + [`FileMeta`]; metadata richness varies by path
//! (see [`SstFileInfo`] and the coverage footnote).
//!
//! - [`on_manifest_updated`]: Fires after **any** manifest write is successfully committed.
//! Receives the full [`RegionMetaActionList`] so consumers can inspect what changed
//! (file additions/removals, schema changes, truncation, partition expression changes, etc.).
//! - [`on_manifest_updated`]: Fires after a manifest write is committed to the **live**
//! (normal) manifest directory. Writes to the staging directory (enter staging,
//! operations during staging, the intermediate apply-staging edit) are suppressed —
//! their effects are accumulated and delivered in a single notification when the
//! staged actions are promoted to the live manifest. Receives the full
//! [`RegionMetaActionList`] so consumers can inspect what changed (file additions/
//! removals, schema changes, truncation, partition expression changes, etc.).
//!
//! - [`on_region_opened`] / [`on_region_closed`] / [`on_region_dropped`] / [`on_region_files_removed`]:
//! Region **lifecycle** callbacks for open, close, logical drop, and physical file removal.
//! See [Region lifecycle](#region-lifecycle) below.
//!
//! Hook implementations are registered via the [`Plugins`](common_base::Plugins) system:
//! ```ignore
@@ -33,6 +42,12 @@
//!
//! ## Coverage
//!
//! Only manifest writes to the **normal** (live) manifest directory trigger
//! `on_manifest_updated`. Writes to the staging manifest directory (operations
//! that happen while the region is in staging mode) are intentionally suppressed:
//! their effects are accumulated and delivered in a single notification when
//! the staged actions are promoted to the live manifest via `exit_staging_on_success`.
//!
//! | Scenario | `on_sst_files_written` | `on_manifest_updated` |
//! |------------------------------|:----------------------:|:---------------------:|
//! | Flush (memtable → SST) | ✅ Yes | ✅ Yes |
@@ -40,23 +55,57 @@
//! | Remote compaction | ✅ (compactor node) ¹ | ✅ (compactor node) ¹ |
//! | RegionEdit / bulk ingestion | ❌ (files pre-written) | ✅ Yes |
//! | Copy region | ❌ (object-store copy) | ✅ Yes |
//! | Apply staging | ❌ (delegates to edit) | ✅ Yes ² |
//! | Apply staging (promote) | ❌ (delegates to edit) | ✅ Yes ² |
//! | Alter (schema change) | ❌ (no SST files) | ✅ Yes |
//! | Truncate | ❌ (removes files) | ✅ Yes |
//! | Enter staging | ❌ (no SST files) | ✅ Yes |
//! | Enter staging | ❌ (no SST files) | ❌ (staging dir) |
//! | Operations during staging | N/A | ❌ (staging dir) |
//! | Async index build | ❌ (index files only) | ✅ Yes |
//!
//! ¹ Remote compaction runs on a dedicated compactor node via `open_compaction_region()`;
//! pass plugins via `OpenCompactionRegionRequest` to enable hooks there. `sst_infos` is
//! `#[serde(skip)]` over the wire, so each [`SstInfo`] is rebuilt from [`FileMeta`] with
//! empty footer/index — see [`SstFileInfo`] for field-level detail.
//! ² Apply staging fires `on_manifest_updated` twice: once when the staging SST files are
//! committed via `RegionEdit`, and once when `exit_staging_on_success` merges all staged
//! manifest actions into the live manifest.
//! ² Apply staging fires `on_manifest_updated` once when `exit_staging_on_success` promotes
//! all staged manifest actions (including the SST file additions) into the live manifest.
//! The intermediate staging `RegionEdit` is written to the staging directory and does not
//! fire the hook — its file list is included in the promote notification.
//!
//! The following paths do **not** trigger any hook:
//! - Follower region sync / catchup (manifest read-only; followers don't author changes)
//! - GC / checkpoint / drop / remap (internal bookkeeping, not logical state changes)
//! - GC / checkpoint / remap (internal bookkeeping, not logical state changes)
//!
//! An explicit region **drop** does fire lifecycle hooks — see
//! [Region lifecycle](#region-lifecycle).
//!
//! ## Region lifecycle
//!
//! Beyond manifest/SST observation, the hook observes the high-level lifecycle of an
//! active region:
//!
//! | Event | Method | When |
//! |-------|--------|------|
//! | Open | [`on_region_opened`] | A create or open request registers the region as active (the counterpart to close/drop). Does not fire for the compactor's transient regions or the catch-up reopen. |
//! | Close | [`on_region_closed`] | A close request (or a close-after-flush) removes the region from the active set. Data files, manifest and WAL state are **preserved**; the region may be reopened. |
//! | Logical drop | [`on_region_dropped`] | A drop request has been handled: the region leaves the active set and its WAL entries are marked obsolete. Data files are **not yet deleted**. |
//! | Physical file removal | [`on_region_files_removed`] | The drop GC worker has deleted the region directory. Terminal file-lifecycle event. |
//!
//! Notes:
//! - `on_region_closed` / `on_region_dropped` run **inline in the region worker loop**,
//! so implementations must be fast (same contract as `on_manifest_updated`).
//! - `on_region_opened` runs inline in the worker loop on the **create** path, but on the
//! **open** path it fires inside the spawned open task (`common_runtime::spawn_global`),
//! i.e. concurrently with the worker loop — after WAL replay, before the region is
//! registered and its open request is acknowledged. Implementations must still be fast
//! and must not assume worker-loop-thread affinity or strict ordering against concurrent
//! requests to other regions.
//! - `on_region_files_removed` runs on the background drop GC task, outside the worker loop.
//! - When global GC is enabled and a normal table region is dropped with `partial_drop`, its
//! directory is left for global reclamation and `on_region_files_removed` is **not** fired
//! by the drop worker (observe it via the global GC path instead).
//! - Logical file removal (compaction, region edit, truncate) is already observable via
//! [`on_manifest_updated`] (`Edit.files_to_remove` / `Truncate` action); only the drop
//! worker's physical directory deletion needs a dedicated file hook.
//!
//! ## Invocation points
//!
@@ -77,11 +126,18 @@
//!
//! ## Future work
//!
//! A future `on_files_removed` hook may be added to observe file lifecycle end
//! (GC, drop, truncate, compaction removal). This is not yet implemented.
//! `on_region_files_removed` currently covers only the **drop** GC worker's physical
//! directory removal. A broader per-file `on_files_removed` hook covering compaction
//! removal, truncate, and the global GC reclamation path is not yet implemented
//! (though logical file removal is already observable via `on_manifest_updated`).
//! Role/leadership transitions (`on_region_role_changed`) are also not hooked.
//!
//! [`on_sst_files_written`]: RegionHook::on_sst_files_written
//! [`on_manifest_updated`]: RegionHook::on_manifest_updated
//! [`on_region_opened`]: RegionHook::on_region_opened
//! [`on_region_closed`]: RegionHook::on_region_closed
//! [`on_region_dropped`]: RegionHook::on_region_dropped
//! [`on_region_files_removed`]: RegionHook::on_region_files_removed
//! [`RegionManifestManager::update`]: crate::manifest::manager::RegionManifestManager::update
//! [`ManifestContext::update_locked`]: crate::region::ManifestContext::update_locked
//! [`ManifestContext::update_manifest`]: crate::region::ManifestContext::update_manifest
@@ -103,6 +159,13 @@ use crate::sst::parquet::SstInfo;
///
/// Must be [`fire`](Self::fire)d **after** the manifest write lock is released
/// (the hook may read the manifest). `#[must_use]` so a forgotten receipt warns.
///
/// ## Staging suppression
///
/// When `is_staging` is `true`, [`fire`](Self::fire) is a no-op — the write went to
/// the staging manifest directory. The hook only observes writes to the live (normal)
/// manifest directory. Staging actions are accumulated and delivered in a single
/// notification when `exit_staging_on_success` promotes them (`is_staging = false`).
#[must_use = "the region hook must be fired after releasing the manifest write lock"]
pub(crate) struct PendingManifestHook {
region_id: RegionId,
@@ -110,6 +173,9 @@ pub(crate) struct PendingManifestHook {
action_list: Option<RegionMetaActionList>,
version: ManifestVersion,
hook: Option<RegionHookRef>,
/// Whether the manifest write went to the staging directory.
/// When `true`, `fire()` is suppressed — the hook only observes live manifest writes.
is_staging: bool,
}
impl PendingManifestHook {
@@ -118,12 +184,14 @@ impl PendingManifestHook {
action_list: Option<RegionMetaActionList>,
version: ManifestVersion,
hook: Option<RegionHookRef>,
is_staging: bool,
) -> Self {
Self {
region_id,
action_list,
version,
hook,
is_staging,
}
}
@@ -132,9 +200,13 @@ impl PendingManifestHook {
self.version
}
/// Fires the hook if one is registered. Safe to call unconditionally: it is
/// a no-op when no hook is registered.
/// Fires the hook if one is registered, **unless** the write went to the staging
/// manifest directory (`is_staging = true`). Safe to call unconditionally:
/// it is a no-op when no hook is registered or when the write is staging-only.
pub(crate) async fn fire(self) {
if self.is_staging {
return;
}
if let (Some(hook), Some(action_list)) = (self.hook, self.action_list) {
hook.on_manifest_updated(self.region_id, &action_list, self.version)
.await;
@@ -146,6 +218,9 @@ impl PendingManifestHook {
/// keeps `self`'s actions followed by `other`'s, and the *later* manifest
/// version wins. Used when a sequence of writes (e.g. staging-exit followed
/// by metadata backfill) should notify the hook exactly once.
///
/// `is_staging` is `true` only if **both** sides are staging; if either side
/// is a live write, the merged result is also live.
pub(crate) fn merge(self, other: PendingManifestHook) -> PendingManifestHook {
debug_assert_eq!(
self.region_id, other.region_id,
@@ -164,6 +239,7 @@ impl PendingManifestHook {
},
version: self.version.max(other.version),
hook: self.hook.or(other.hook),
is_staging: self.is_staging && other.is_staging,
}
}
}
@@ -211,10 +287,14 @@ pub trait RegionHook: Send + Sync + Debug {
let _ = (region_id, region_metadata, files);
}
/// Called after the region manifest is successfully committed.
/// Called after the region manifest is successfully committed to the **live**
/// (normal) manifest directory.
///
/// Fires for **all** manifest write paths: flush, compaction, region edit,
/// copy region, alter, truncate, enter staging, index build, etc.
/// Fires for: flush, compaction, region edit, copy region, alter, truncate,
/// async index build, and apply-staging promote. Does **not** fire for writes
/// to the staging manifest directory (enter staging, operations during staging,
/// the intermediate apply-staging edit) — those are suppressed because their
/// effects are accumulated and delivered in a single promote notification.
///
/// Does **not** fire for:
/// - Manifest reads / follower sync (no write)
@@ -228,6 +308,80 @@ pub trait RegionHook: Send + Sync + Debug {
) {
let _ = (region_id, action_list, manifest_version);
}
/// Called once a region **open** or **create** succeeds, but **before** the
/// region is registered in the engine's active set (`insert_region` runs
/// immediately afterwards).
///
/// Fires once when a region becomes active via a create or open request —
/// the natural counterpart to [`on_region_closed`] / [`on_region_dropped`].
/// It does **not** fire for the compactor's transient compaction regions
/// (`open_compaction_region`), nor for the internal reopen performed during
/// follower catch-up / leadership promotion.
///
/// On the **create** path it runs inline in the region worker loop; on the
/// **open** path it runs inside the spawned open task
/// (`common_runtime::spawn_global`), concurrently with the worker loop
/// (after WAL replay, before the region is registered/acknowledged).
/// Implementations must be fast and must **not** assume worker-loop-thread
/// affinity or strict ordering against concurrent requests to other regions.
///
/// [`on_region_closed`]: RegionHook::on_region_closed
/// [`on_region_dropped`]: RegionHook::on_region_dropped
async fn on_region_opened(&self, region_id: RegionId, region_metadata: &RegionMetadataRef) {
let _ = (region_id, region_metadata);
}
/// Called after a region is **closed** via a close request.
///
/// The region is removed from the engine's active set, but its data files,
/// manifest, and WAL state are **preserved**; the region may be reopened
/// later. Fires once per successful close, after the region's background
/// tasks (flush/compaction) have been stopped.
///
/// Fires for a region of **any** role (leader or follower) that is closed.
/// Does **not** fire when a region is dropped (see [`on_region_dropped`]).
///
/// Runs inline in the region worker loop; implementations should be fast.
///
/// [`on_region_dropped`]: RegionHook::on_region_dropped
async fn on_region_closed(&self, region_id: RegionId, region_metadata: &RegionMetadataRef) {
let _ = (region_id, region_metadata);
}
/// Called after a region is **logically dropped** (a drop request has been
/// handled).
///
/// The region is removed from the active set and its WAL entries are marked
/// obsolete. Its data files are **not yet deleted** — they are scheduled for
/// asynchronous removal by the GC worker. Observe physical deletion via
/// [`on_region_files_removed`].
///
/// Runs inline in the region worker loop; implementations should be fast.
///
/// [`on_region_files_removed`]: RegionHook::on_region_files_removed
async fn on_region_dropped(&self, region_id: RegionId, region_metadata: &RegionMetadataRef) {
let _ = (region_id, region_metadata);
}
/// Called after a dropped region's data files are **physically removed** by
/// the drop GC worker (the region directory has been deleted).
///
/// This is the terminal event in a region's file lifecycle; no further
/// callbacks fire for this region id afterwards. Fires only when the drop
/// worker itself deletes the directory. When global GC is enabled and the
/// region is a normal table region dropped with `partial_drop`, the
/// directory is left for global reclamation and this hook is **not** fired
/// by the drop worker.
///
/// Runs on a background task, outside the region worker loop.
async fn on_region_files_removed(
&self,
region_id: RegionId,
region_metadata: &RegionMetadataRef,
) {
let _ = (region_id, region_metadata);
}
}
pub type RegionHookRef = Arc<dyn RegionHook>;
+1
View File
@@ -1236,6 +1236,7 @@ impl ManifestContext {
action_list_for_hook,
version,
self.hook.clone(),
is_staging,
))
}
+8
View File
@@ -89,5 +89,13 @@ impl<S: LogStore> RegionWorkerLoop<S> {
// clean index build status.
self.index_build_scheduler.on_region_closed(region_id).await;
self.region_count.dec();
// Notify the region hook that the region has been closed. The region is
// fully stopped and unregistered, but its files/manifest are preserved.
// Runs inline; the hook contract requires it to be fast.
if let Some(hook) = region.manifest_ctx.hook() {
let metadata = region.metadata();
hook.on_region_closed(region_id, &metadata).await;
}
}
}
+6
View File
@@ -86,6 +86,12 @@ impl<S: LogStore> RegionWorkerLoop<S> {
self.region_count.inc();
// Notify the region hook that the region has been opened (created).
// Fires before registration; allocates nothing when no hook is registered.
if let Some(hook) = region.manifest_ctx.hook() {
hook.on_region_opened(region_id, &region.metadata()).await;
}
// Insert the MitoRegion into the RegionMap.
self.regions.insert_region(region);
+47
View File
@@ -23,10 +23,12 @@ use object_store::util::join_path;
use object_store::{EntryMode, ObjectStore};
use snafu::ResultExt;
use store_api::logstore::LogStore;
use store_api::metadata::RegionMetadataRef;
use store_api::region_request::{AffectedRows, PathType};
use store_api::storage::RegionId;
use tokio::time::sleep;
use crate::engine::region_hook::RegionHookRef;
use crate::error::{OpenDalSnafu, Result};
use crate::region::{RegionLeaderState, RegionMapRef};
use crate::worker::{DROPPING_MARKER_FILE, RegionWorkerLoop};
@@ -106,6 +108,22 @@ where
self.region_count.dec();
// Notify registered hooks that the region has been logically dropped, and
// prepare a payload for the background GC task to fire the terminal
// `on_region_files_removed`. When no hook is registered this allocates
// nothing — no metadata snapshot is taken and no payload is built.
let hook_payload = match region.manifest_ctx.hook() {
Some(hook) => {
let region_metadata = region.metadata();
hook.on_region_dropped(region_id, &region_metadata).await;
Some(DropHookPayload {
hook,
metadata: region_metadata,
})
}
None => None,
};
let object_store = region.access_layer.object_store().clone();
let dropping_regions = self.dropping_regions.clone();
let listener = self.listener.clone();
@@ -122,6 +140,7 @@ where
object_store,
dropping_regions,
partial_drop,
hook_payload.as_ref(),
)
.await
} else {
@@ -135,6 +154,7 @@ where
object_store,
dropping_regions,
gc_duration,
hook_payload.as_ref(),
)
.await
};
@@ -156,6 +176,19 @@ where
}
}
/// Carries the region hook and region metadata into the background GC task so
/// it can fire [`RegionHook::on_region_files_removed`] once the dropped region's
/// directory is physically deleted.
///
/// Only constructed when a hook is registered; the task receives it as
/// `Option<&DropHookPayload>` so the no-hook path allocates nothing.
///
/// [`RegionHook::on_region_files_removed`]: crate::engine::region_hook::RegionHook::on_region_files_removed
struct DropHookPayload {
hook: RegionHookRef,
metadata: RegionMetadataRef,
}
/// Background GC task to remove the entire region path once one of the following
/// conditions is true:
/// - It finds there is no parquet file left.
@@ -172,6 +205,7 @@ async fn later_drop_task_without_global_gc(
object_store: ObjectStore,
dropping_regions: RegionMapRef,
gc_duration: Duration,
hook_payload: Option<&DropHookPayload>,
) -> bool {
remove_region_with_retry(
region_id,
@@ -180,6 +214,7 @@ async fn later_drop_task_without_global_gc(
dropping_regions,
Some(gc_duration),
false,
hook_payload,
)
.await
}
@@ -191,6 +226,7 @@ async fn remove_region_with_retry(
dropping_regions: std::sync::Arc<crate::region::RegionMap>,
gc_duration: Option<Duration>,
mut force: bool,
hook_payload: Option<&DropHookPayload>,
) -> bool {
for _ in 0..MAX_RETRY_TIMES {
let result = remove_region_dir_once(&region_path, &object_store, force).await;
@@ -204,6 +240,15 @@ async fn remove_region_with_retry(
Ok(true) => {
dropping_regions.remove_region(region_id);
info!("Region {} is dropped, force: {}", region_path, force);
// The region directory has been physically deleted; fire the
// terminal file-lifecycle event. The partial-drop/global-GC path
// never reaches here (it does not delete the directory itself).
if let Some(hook_payload) = hook_payload {
hook_payload
.hook
.on_region_files_removed(region_id, &hook_payload.metadata)
.await;
}
return true;
}
Ok(false) => (),
@@ -230,6 +275,7 @@ async fn later_drop_task_with_global_gc(
object_store: ObjectStore,
dropping_regions: RegionMapRef,
partial_drop: bool,
hook_payload: Option<&DropHookPayload>,
) -> bool {
// For metadata regions or regions marked for full deletion (such as when dropping a table)
// the region directory is forcefully removed immediately.
@@ -243,6 +289,7 @@ async fn later_drop_task_with_global_gc(
dropping_regions,
None,
true,
hook_payload,
)
.await
} else {
+7
View File
@@ -149,6 +149,13 @@ impl<S: LogStore> RegionWorkerLoop<S> {
);
region_count.inc();
// Notify the region hook that the region has been opened.
// Fires before registration; allocates nothing when no hook
// is registered.
if let Some(hook) = region.manifest_ctx.hook() {
hook.on_region_opened(region_id, &region.metadata()).await;
}
// Insert the Region into the RegionMap.
regions.insert_region(region);
+30 -19
View File
@@ -52,45 +52,37 @@ pub enum QueryStatement {
}
impl QueryStatement {
pub fn post_process(&self, params: HashMap<String, String>) -> Result<QueryStatement> {
pub fn post_process(self, params: HashMap<String, String>) -> Result<QueryStatement> {
match self {
QueryStatement::Sql(_) => UnimplementedSnafu {
operation: "sql post process",
}
.fail(),
QueryStatement::Promql(eval_stmt, alias) => {
QueryStatement::Promql(mut eval_stmt, alias) => {
let node_name = match params.get("name") {
Some(name) => name.as_str(),
None => "",
};
let extension_node = Self::create_extension_node(node_name, &eval_stmt.expr);
Ok(QueryStatement::Promql(
EvalStmt {
expr: Extension(extension_node.unwrap()),
start: eval_stmt.start,
end: eval_stmt.end,
interval: eval_stmt.interval,
lookback_delta: eval_stmt.lookback_delta,
},
alias.clone(),
))
let extension_node = Self::create_extension_node(node_name, eval_stmt.expr);
eval_stmt.expr = Extension(extension_node.unwrap());
Ok(QueryStatement::Promql(eval_stmt, alias))
}
}
}
fn create_extension_node(node_name: &str, expr: &Expr) -> Option<NodeExtension> {
fn create_extension_node(node_name: &str, expr: Expr) -> Option<NodeExtension> {
match node_name {
ANALYZE_NODE_NAME => Some(NodeExtension {
expr: Arc::new(AnalyzeExpr { expr: expr.clone() }),
expr: Arc::new(AnalyzeExpr { expr }),
}),
ANALYZE_VERBOSE_NODE_NAME => Some(NodeExtension {
expr: Arc::new(AnalyzeVerboseExpr { expr: expr.clone() }),
expr: Arc::new(AnalyzeVerboseExpr { expr }),
}),
EXPLAIN_NODE_NAME => Some(NodeExtension {
expr: Arc::new(ExplainExpr { expr: expr.clone() }),
expr: Arc::new(ExplainExpr { expr }),
}),
EXPLAIN_VERBOSE_NODE_NAME => Some(NodeExtension {
expr: Arc::new(ExplainVerboseExpr { expr: expr.clone() }),
expr: Arc::new(ExplainVerboseExpr { expr }),
}),
_ => None,
}
@@ -204,9 +196,10 @@ impl QueryLanguageParser {
}
pub(crate) fn apply_alias_extension(mut eval_stmt: EvalStmt, alias: &str) -> EvalStmt {
let expr = eval_stmt.expr;
eval_stmt.expr = Extension(NodeExtension {
expr: Arc::new(AliasExpr {
expr: eval_stmt.expr.clone(),
expr,
alias: alias.to_string(),
}),
});
@@ -263,6 +256,14 @@ macro_rules! define_node_ast_extension {
fn children(&self) -> &[Expr] {
std::slice::from_ref(&self.expr)
}
fn with_new_children(&self, children: Vec<Expr>) -> Arc<dyn ExtensionExpr> {
let mut iter = children.into_iter();
match (iter.next(), iter.next()) {
(Some(expr), None) => Arc::new($name_expr { expr }),
_ => Arc::new(self.clone()),
}
}
}
#[allow(rustdoc::broken_intra_doc_links)]
@@ -313,6 +314,16 @@ impl ExtensionExpr for AliasExpr {
fn children(&self) -> &[Expr] {
std::slice::from_ref(&self.expr)
}
fn with_new_children(&self, children: Vec<Expr>) -> Arc<dyn ExtensionExpr> {
let mut iter = children.into_iter();
match (iter.next(), iter.next()) {
(Some(expr), None) => Arc::new(Self {
expr,
alias: self.alias.clone(),
}),
_ => Arc::new(self.clone()),
}
}
}
#[derive(Debug, Clone)]
pub struct Alias {
+41
View File
@@ -229,6 +229,8 @@ impl IslandExpr {
!modifier.return_bool
&& modifier.matching.is_none()
&& matches!(modifier.card, VectorMatchCardinality::OneToOne)
&& modifier.fill_values.lhs.is_none()
&& modifier.fill_values.rhs.is_none()
}) =>
{
let lhs = Self::try_new(lhs, env)?;
@@ -1093,6 +1095,18 @@ impl PromPlanner {
query_engine_state: &QueryEngineState,
binary_expr: &PromBinaryExpr,
) -> Result<LogicalPlan> {
// promql-parser accepts fill modifiers, but Greptime does not implement the
// required outer joins and missing-value substitution. Reject them before the
// binary-island fast path so they cannot silently behave like normal inner joins.
if let Some(modifier) = &binary_expr.modifier {
ensure!(
modifier.fill_values.lhs.is_none() && modifier.fill_values.rhs.is_none(),
UnsupportedExprSnafu {
name: "PromQL fill modifiers"
}
);
}
if let Some(plan) = self.try_plan_binary_island(binary_expr).await? {
return Ok(plan);
}
@@ -5542,6 +5556,33 @@ mod test {
);
}
#[tokio::test]
async fn reject_binary_fill_modifiers() {
let state = build_query_engine_state();
for query in [
"some_metric + fill(0) some_alt_metric",
"some_metric + fill_left(0) some_alt_metric",
"some_metric + fill_right(0) some_alt_metric",
"(some_metric + fill(0) some_alt_metric) + some_metric",
] {
let eval_stmt = build_eval_stmt(query);
let table_provider = build_test_table_provider(&[], 0, 0).await;
let err = PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &state)
.await
.unwrap_err();
assert!(
matches!(
&err,
crate::promql::error::Error::UnsupportedExpr { name, .. }
if name == "PromQL fill modifiers"
),
"{err}"
);
}
}
#[tokio::test]
async fn timestamp_binary_join_falls_back_when_tsid_is_projected_out() {
for query in [
+8 -4
View File
@@ -1098,6 +1098,10 @@ async fn test_execute_copy_from_csv_strict_headers(instance: Arc<dyn MockInstanc
let unknown_path = tmp_dir.path().join("unknown.csv");
let missing_path = tmp_dir.path().join("missing.csv");
let duplicate_path = tmp_dir.path().join("duplicate.csv");
let matching_location = prepare_path(matching_path.to_str().unwrap());
let unknown_location = prepare_path(unknown_path.to_str().unwrap());
let missing_location = prepare_path(missing_path.to_str().unwrap());
let duplicate_location = prepare_path(duplicate_path.to_str().unwrap());
std::fs::write(
&matching_path,
"ts,host_id,reading_value\n2024-01-01T00:00:00,1,10.5\n",
@@ -1127,7 +1131,7 @@ async fn test_execute_copy_from_csv_strict_headers(instance: Arc<dyn MockInstanc
&instance,
&format!(
"COPY csv_strict_headers FROM '{}' WITH (FORMAT='csv', STRICT_HEADERS='true');",
matching_path.display()
matching_location
),
)
.await
@@ -1149,7 +1153,7 @@ async fn test_execute_copy_from_csv_strict_headers(instance: Arc<dyn MockInstanc
&instance,
&format!(
"COPY csv_strict_headers FROM '{}' WITH (FORMAT='csv', STRICT_HEADERS='true');",
unknown_path.display()
unknown_location
),
)
.await
@@ -1173,7 +1177,7 @@ async fn test_execute_copy_from_csv_strict_headers(instance: Arc<dyn MockInstanc
&instance,
&format!(
"COPY csv_strict_headers FROM '{}' WITH (FORMAT='csv', STRICT_HEADERS='true');",
missing_path.display()
missing_location
),
)
.await
@@ -1197,7 +1201,7 @@ async fn test_execute_copy_from_csv_strict_headers(instance: Arc<dyn MockInstanc
&instance,
&format!(
"COPY csv_strict_headers FROM '{}' WITH (FORMAT='csv', STRICT_HEADERS='true');",
duplicate_path.display()
duplicate_location
),
)
.await
@@ -62,16 +62,6 @@ tql eval (10, 10, '1s') double_exponential_smoothing(prom_series[10s], 0.4 + 0.1
| 1970-01-01T00:00:10 | 47.0806953125 | p |
+---------------------+------------------------------------------------------------------------------------------+------+
-- holt_winters (backward compatibility)
-- SQLNESS SORT_RESULT 3 1
tql eval (10, 10, '1s') holt_winters(prom_series[10s], 0.4 + 0.1, 0.1);
+---------------------+------------------------------------------------------------------------------------------+------+
| ts | prom_double_exponential_smoothing(ts_range,val,Float64(0.4) + Float64(0.1),Float64(0.1)) | host |
+---------------------+------------------------------------------------------------------------------------------+------+
| 1970-01-01T00:00:10 | 47.0806953125 | p |
+---------------------+------------------------------------------------------------------------------------------+------+
DROP TABLE prom_series;
Affected Rows: 0
@@ -34,10 +34,6 @@ tql eval (3, 3, '1s') predict_linear(prom_series[3s], 40 + 2);
-- SQLNESS SORT_RESULT 3 1
tql eval (10, 10, '1s') double_exponential_smoothing(prom_series[10s], 0.4 + 0.1, 0.1);
-- holt_winters (backward compatibility)
-- SQLNESS SORT_RESULT 3 1
tql eval (10, 10, '1s') holt_winters(prom_series[10s], 0.4 + 0.1, 0.1);
DROP TABLE prom_series;
CREATE TABLE
@@ -0,0 +1,8 @@
name = "legacy_json2_variant_payload"
reason = "Verify current binaries can read JSON2 variant payloads written by old binaries after the JSON2 encoding change."
introduced_by = "PR #8435"
topologies = ["distributed"]
from_range = ["=v1.1.0", "=v1.1.1", "=v1.1.2"]
to_range = [">v1.1.1"]
features = ["json", "json2", "table", "read"]
owner = "query"
@@ -0,0 +1,29 @@
CREATE TABLE t_legacy_json2_variant_payload (
ts TIMESTAMP TIME INDEX,
j JSON2
) WITH (
'append_mode' = 'true'
);
INSERT INTO t_legacy_json2_variant_payload (ts, j) VALUES
('2026-07-08 00:00:00+0000', '{"a": {"b": 1}, "c": "s1", "d": [{"e": {"f": 0.1}}]}'),
('2026-07-08 00:01:00+0000', '{"a": {"b": -2}, "c": [1], "d": [{"e": {"f": 0.2}}]}'),
('2026-07-08 00:02:00+0000', '{"a": {"b": "s3"}, "c": "s3", "d": [{"e": {"g": -0.3}}]}'),
('2026-07-08 00:03:00+0000', '{"a": {"b": null}, "c": null}'),
('2026-07-08 00:04:00+0000', '{"c": "s5", "d": {"e": true}}'),
('2026-07-08 00:05:00+0000', '{"a": {"b": 6}}');
ADMIN FLUSH_TABLE('t_legacy_json2_variant_payload');
CREATE TABLE t_legacy_json2_variant_payload_object_d (
ts TIMESTAMP TIME INDEX,
j JSON2
) WITH (
'append_mode' = 'true'
);
INSERT INTO t_legacy_json2_variant_payload_object_d (ts, j) VALUES
('2026-07-08 01:00:00+0000', '{"d": {"e": true}}'),
('2026-07-08 01:01:00+0000', '{"d": {"e": false}}');
ADMIN FLUSH_TABLE('t_legacy_json2_variant_payload_object_d');
@@ -0,0 +1,60 @@
SELECT ts, j.a.b AS a_b, j.c AS c
FROM t_legacy_json2_variant_payload
ORDER BY ts;
+---------------------+-----+-----+
| ts | a_b | c |
+---------------------+-----+-----+
| 2026-07-08T00:00:00 | 1 | s1 |
| 2026-07-08T00:01:00 | -2 | [1] |
| 2026-07-08T00:02:00 | s3 | s3 |
| 2026-07-08T00:03:00 | | |
| 2026-07-08T00:04:00 | | s5 |
| 2026-07-08T00:05:00 | 6 | |
+---------------------+-----+-----+
SELECT
ts,
j.d.e AS d_e
FROM t_legacy_json2_variant_payload_object_d
ORDER BY ts;
+---------------------+-------+
| ts | d_e |
+---------------------+-------+
| 2026-07-08T01:00:00 | true |
| 2026-07-08T01:01:00 | false |
+---------------------+-------+
-- JSON2 projection rendering is a user-visible compatibility contract.
SELECT ts, j.d AS d
FROM t_legacy_json2_variant_payload
ORDER BY ts;
+---------------------+--------------------+
| ts | d |
+---------------------+--------------------+
| 2026-07-08T00:00:00 | [{"e":{"f":0.1}}] |
| 2026-07-08T00:01:00 | [{"e":{"f":0.2}}] |
| 2026-07-08T00:02:00 | [{"e":{"g":-0.3}}] |
| 2026-07-08T00:03:00 | |
| 2026-07-08T00:04:00 | {"e":true} |
| 2026-07-08T00:05:00 | |
+---------------------+--------------------+
SHOW CREATE TABLE t_legacy_json2_variant_payload;
+--------------------------------+---------------------------------------------------------------+
| Table | Create Table |
+--------------------------------+---------------------------------------------------------------+
| t_legacy_json2_variant_payload | CREATE TABLE IF NOT EXISTS "t_legacy_json2_variant_payload" ( |
| | "ts" TIMESTAMP(3) NOT NULL, |
| | "j" JSON2 NULL, |
| | TIME INDEX ("ts") |
| | ) |
| | |
| | ENGINE=mito |
| | WITH( |
| | append_mode = 'true' |
| | ) |
+--------------------------------+---------------------------------------------------------------+
@@ -0,0 +1,16 @@
SELECT ts, j.a.b AS a_b, j.c AS c
FROM t_legacy_json2_variant_payload
ORDER BY ts;
SELECT
ts,
j.d.e AS d_e
FROM t_legacy_json2_variant_payload_object_d
ORDER BY ts;
-- JSON2 projection rendering is a user-visible compatibility contract.
SELECT ts, j.d AS d
FROM t_legacy_json2_variant_payload
ORDER BY ts;
SHOW CREATE TABLE t_legacy_json2_variant_payload;
+4 -4
View File
@@ -11,14 +11,14 @@
docs together.
- `[case]` is report metadata only. `[scenario]` is the executable regression
configuration and must include `kind`, data layout, tables, queries, and
thresholds. Keep scenario parsing explicit in both
`tests/perf/query_regression_runner.py` and
`src/cmd/src/bin/query_perf_fixture.rs`.
thresholds. Rust owns case schema, defaults, validation, and normalized plan
output through `query_perf_fixture plan`; `tests/perf/query_regression_runner.py`
should only orchestrate from that normalized JSON.
- Keep the direct-SST generator generic. Issue-specific behavior belongs in case
files and thresholds, not in Rust generator logic.
- Before pushing perf harness changes, run at least:
- `uv run --no-project python -m py_compile .github/scripts/query-regression-run.py .github/scripts/query-regression-summary.py .github/scripts/query-regression-pr-metadata.py tests/perf/query_regression_runner.py`
- `cargo fmt --all -- --check`
- `cargo build -p cmd --bin query_perf_fixture`
- `cargo build -p cmd --bin query_perf_fixture --features dev-tools`
- dry-run the Python runner and Rust fixture generator against all built-in
cases when the DSL or workflow case selection changes.
+144 -22
View File
@@ -4,6 +4,11 @@ This directory is for query performance cases that compare a base build with a
candidate build. It is not a replacement for sqlness: the goal is to measure the
effect of optimizer/query-engine changes on realistic scan work.
The Prometheus remote-write cases are end-to-end regression coverage for the
automatic write, flush, SST, and query path. They do not replace controlled
encoding experiments that explicitly compare `plain`/dictionary,
`no_dictionary`, BYTE_STREAM_SPLIT, or Auto policies.
## Phase 1: direct readable SST fixtures
Phase 1 should generate data by writing readable Mito SST files and matching
@@ -40,8 +45,10 @@ The runner also supports `scenario.kind = "prom_remote_write_then_query"` for a
bounded write-path smoke/regression flow. This path is explicit and separate from
the direct-SST fixture path: it starts the base and candidate distributed clusters,
writes deterministic Prometheus remote-write v1 samples through
`/v1/prometheus/write`, flushes the configured physical metric table, then runs
the configured SQL/TQL queries against the logical metric table.
`/v1/prometheus/write`, flushes the configured physical metric table, checks
visibility, runs the configured SQL/TQL queries, stops and awaits the datanode,
then optionally inspects the landed SST/Parquet footer, encoding, and size and
runs the read-bench against the quiescent data directory.
Remote-write cases configure one database, one logical metric, and one physical
table under `[scenario.remote_write]`:
@@ -52,34 +59,121 @@ kind = "prom_remote_write_then_query"
[scenario.remote_write]
database = "public"
metric = "prom_remote_write_smoke"
metric = "prom_remote_write_seeded_random"
physical_table = "greptime_physical_table"
series_count = 8
samples_per_series = 30
series_count = 512
samples_per_series = 1440
sample_chunk_size = 480
flush_every_sample_chunks = 1
start_unix_millis = 1_704_067_200_000
step_millis = 15_000
chunk_series_count = 4
visibility_timeout_seconds = 30
step_millis = 60_000
chunk_series_count = 128
timeout_seconds = 180
visibility_timeout_seconds = 120
# Optional: split by time so each helper invocation writes only this many samples
# per series, then periodically flush the physical metric table to produce
# multiple time-interval SSTs.
# sample_chunk_size = 300
# flush_every_sample_chunks = 1
[scenario.remote_write.prom_store]
pending_rows_flush_interval = "1s"
max_batch_rows = 100000
```
Cases can optionally add `[scenario.remote_write.value]` to control the generated
sample values without changing label cardinality or the runner lifecycle:
```toml
[scenario.remote_write.value]
pattern = "quantized_signal" # linear, constant, modulo, unique, seeded_random,
# run_length, quantized_signal,
# signal_with_sporadic_stalls, mixed_signal_repeated
base = 0.0
step = 0.125
cardinality = 4096 # buckets for modulo/seeded_random/quantized_signal/run_length
seed = 12345 # deterministic seeded_random input
run_length = 8 # adjacent samples per bucket for run_length/quantized_signal
stall_every = 100 # interval for signal_with_sporadic_stalls
stall_length = 16 # held samples inside each stall interval
mixed_every = 5 # every Nth sample becomes the repeated base value
```
The default `linear` pattern preserves the helper's historical formula. Use
`constant` or low-cardinality `modulo`/`seeded_random` values for repeated-value
data shapes, `run_length` for run-heavy low-cardinality series, `quantized_signal`
for signal-like values collapsed into a finite bucket set, `signal_with_sporadic_stalls`
for mostly continuous signals with periodic flat spots, and `mixed_signal_repeated`
for signal-plus-periodic-default mixtures. `unique` or high-cardinality buckets
still work for broad sample-value distributions. This is a generic sample-value
control for query/ingestion cases; it does not inspect or assert storage
encoding, Parquet footers, or storage policy choices. For chunked remote-write
ingestion, the runner passes the sample offset and total sample count to the
helper so non-linear value patterns use a stable global/per-series ordinal across
chunks.
Case schema, value distribution defaults, storage defaults, and read-bench
defaults are owned by Rust. The Python runner calls
`query_perf_fixture plan --case <case.toml>` and orchestrates the normalized JSON.
The same helper exposes `direct-sst`, `prom-remote-write`, and `inspect-footer`
subcommands; the old direct invocation (`query_perf_fixture --case ... --out-dir
...`) remains compatible.
Remote-write cases that need to validate storage output can add
`[scenario.remote_write.storage]`. When present, the scenario body becomes:
remote-write → flush → visibility check → query measurement → stop and await
datanode → Parquet footer/size/encoding inspection → optional read-bench. Footer
inspection is handled by `query_perf_fixture inspect-footer`, which reads local
Parquet footers directly; it does not require Python `pyarrow`.
```toml
[scenario.remote_write.storage]
inspect = true
column = "greptime_value"
include_metadata_files = false
# Optional: inspect below datanode data home instead of the whole fresh data dir.
# root_suffix = "greptime/public/<table_id>"
min_files = 1
min_files_with_column = 1
require_encodings = ["BYTE_STREAM_SPLIT"]
forbid_encodings = ["PLAIN_DICTIONARY"]
max_total_file_size_bytes = 104857600
max_column_compressed_size_bytes = 52428800
max_column_uncompressed_size_bytes = 209715200
max_candidate_total_file_size_regression_pct = 10.0
max_candidate_column_compressed_size_regression_pct = 10.0
max_candidate_column_uncompressed_size_regression_pct = 10.0
```
The `*_pct` storage thresholds are percent regression limits comparing candidate
against base; `pct` means percent, not percentile. Per-target storage checks
(`min_files`, `min_files_with_column`, required/forbidden encodings, and absolute
`max_*_bytes`) run for both base and candidate. Comparative
`max_candidate_*_regression_pct` checks remain base-vs-candidate. These checks
are generic footer and byte-size assertions and do not encode product-specific
heuristics. When storage inspection is enabled, `min_files` and
`min_files_with_column` default to `1` even if omitted, so dry-runs show these
planned checks and an empty flush or missing target column fails the scenario.
By default the inspector root is the target datanode data home, which is suitable
for a fresh single-case data directory. For reused or more complex data homes, set
`root_suffix` to a path relative to the datanode data home to narrow inspection.
When storage inspection is enabled, `[scenario.remote_write.read_bench]` defaults
to enabled with both datanode `parquetbench` and `scanbench`. Disable it with
`enabled = false`. `parquetbench` measures per-SST reader cost, `scanbench`
measures region scan cost, and query measurements still exercise the SQL/TQL
frontend path. Treat all performance conclusions as release-only; debug builds
are suitable only for command wiring and correctness checks.
The runner creates the configured database if needed, writes a per-target
frontend config enabling `[prom_store]` with metric engine storage and a non-zero
`pending_rows_flush_interval`, and validates that the logical metric table reaches
`series_count * samples_per_series` rows before trusting the query measurements.
Use `--remote-write-generator /path/to/prom_remote_write_fixture` to provide the
Rust payload helper. `--fixture-only` is rejected for remote-write cases; use
`--dry-run` for planning.
Use `--fixture-generator /path/to/query_perf_fixture` to provide the Rust helper.
Deprecated `--remote-write-generator` and `--storage-inspector` options are kept
only for CLI compatibility. `--fixture-only` is rejected for remote-write cases;
use `--dry-run` for planning.
Large manual remote-write cases can set `sample_chunk_size` to split ingestion by
time. For each chunk, the runner invokes `prom_remote_write_fixture` with the
time. For each chunk, the runner invokes `query_perf_fixture prom-remote-write` with the
same series cardinality but a shorter `--samples-per-series` and an advanced
`--start-unix-millis`. `flush_every_sample_chunks` controls periodic
`ADMIN FLUSH_TABLE('<physical_table>')` calls; with `flush_every_sample_chunks = 1`,
@@ -89,6 +183,25 @@ need multi-window file distribution should span multiple compaction windows. If
`sample_chunk_size` is omitted, the runner keeps the older single-helper-invocation
behavior and flushes once at the end.
The default remote-write coverage set contains four cases with the same 2048
series × 14,400 samples (29,491,200 rows per target) shape. Each writes ten
one-day chunks, flushes every chunk, requires at least two visible SSTs with the
value column, runs seven read-bench iterations with `parquetbench` capped at
four SSTs while `scanbench` covers the landed region, and measures one ten-day
TQL selector with two warmups and 15 iterations:
- `prom_remote_write_seeded_random`: high-distinct seeded-random values with
cardinality 29,491,200 and seed 8444.
- `prom_remote_write_run_heavy`: low-cardinality values in exact 16-sample runs.
- `prom_remote_write_mixed_every`: continuous signal values with every fifth
sample repeated at the base value.
- `prom_remote_write_integer_counter`: strictly increasing integer counter
values, with disjoint ranges for each series.
These cases record normal storage/footer and read-bench results but do not gate
specific value encodings or storage-size outcomes. Use the controlled encoding
experiment matrix for those policy comparisons.
`tests/perf/query_cases/prom_remote_write_7913/case.toml` is a larger manual
case for issue #7913. It writes 8192 series × 20160 samples through remote-write
in 1440-sample daily time chunks, flushing after each chunk before running 1d/7d/14d
@@ -246,7 +359,8 @@ as a fallback. Manifest files are reflinked or copied, not hardlinked.
Fixture generator smoke test:
```bash
cargo run -p cmd --bin query_perf_fixture -- \
cargo run -p cmd --bin query_perf_fixture --features dev-tools -- \
direct-sst \
--case tests/perf/query_cases/smoke_direct_sst/case.toml \
--out-dir /tmp/query-perf-smoke
```
@@ -271,10 +385,10 @@ Remote-write runner dry-run:
```bash
uv run --no-project python tests/perf/query_regression_runner.py \
--case tests/perf/query_cases/prom_remote_write_smoke/case.toml \
--case tests/perf/query_cases/prom_remote_write_seeded_random/case.toml \
--base-bin /path/to/base/greptime \
--candidate-bin /path/to/candidate/greptime \
--remote-write-generator /path/to/prom_remote_write_fixture \
--fixture-generator /path/to/query_perf_fixture \
--work-dir /tmp/query-perf-remote-write \
--dry-run
```
@@ -289,14 +403,22 @@ query regression runs. It builds its own binaries for now:
candidate checkout
- runner and summary formatter from the candidate checkout
The workflow runs automatically only for non-draft PRs labeled
`query-regression` (on label/ready-for-review/reopen events, not every push).
The workflow builds base and candidate `greptime` as normal release-equivalent
binaries. Candidate `query_perf_fixture` is the extra head-side helper binary;
the runner uses candidate `greptime datanode parquetbench/scanbench` as the
read-bench tool against each target's data directory.
The workflow runs automatically only when `query-regression` is added to a
non-draft PR; it does not rerun on pushes, ready-for-review, or reopen events.
PR runs build base/candidate once and then run the default case set with
`--allow-large-fixture`. Manual `workflow_dispatch` runs can pass `all`, one case
path, or a comma/whitespace-separated list of case paths, and can override refs.
It always uploads `query-regression-work/**` and `query-regression-summary.md`,
writes the Markdown summary to the workflow step summary, and updates a sticky PR
comment through the trusted follow-up workflow.
The main report artifact uploads only aggregate/per-target JSON reports,
component logs, and `query-regression-summary.md` with seven-day retention;
fixture data, SSTs, and cluster state are excluded. PR runs also upload a
separate trusted-comment artifact containing PR metadata and aggregate reports.
The workflow writes the Markdown summary to the workflow step summary and
updates a sticky PR comment through the trusted follow-up workflow.
## Built-in cases
@@ -0,0 +1,48 @@
[case]
name = "prom_remote_write_integer_counter"
description = "End-to-end Prometheus remote-write regression with strictly increasing integer counter values"
[scenario]
kind = "prom_remote_write_then_query"
[scenario.remote_write]
database = "public"
metric = "prom_remote_write_integer_counter"
physical_table = "greptime_physical_table"
series_count = 2048
samples_per_series = 14400
sample_chunk_size = 1440
flush_every_sample_chunks = 1
start_unix_millis = 1704067200000
step_millis = 60000
chunk_series_count = 256
timeout_seconds = 600
visibility_timeout_seconds = 300
[scenario.remote_write.value]
pattern = "unique"
base = 0
step = 1
[scenario.remote_write.prom_store]
pending_rows_flush_interval = "1s"
max_batch_rows = 100000
[scenario.remote_write.storage]
column = "greptime_value"
min_files = 2
min_files_with_column = 2
[scenario.remote_write.read_bench]
iterations = 7
max_files = 4
[[scenario.queries]]
name = "selector_10d_integer_counter"
kind = "tql"
query = "TQL ANALYZE VERBOSE (1704067200, 1704931200, '1h') prom_remote_write_integer_counter{host=~'host.*'}"
warmup = 2
iterations = 15
[scenario.queries.thresholds]
max_candidate_latency_regression_pct = 75
@@ -0,0 +1,49 @@
[case]
name = "prom_remote_write_mixed_every"
description = "End-to-end Prometheus remote-write regression with periodic repeated signal values"
[scenario]
kind = "prom_remote_write_then_query"
[scenario.remote_write]
database = "public"
metric = "prom_remote_write_mixed_every"
physical_table = "greptime_physical_table"
series_count = 2048
samples_per_series = 14400
sample_chunk_size = 1440
flush_every_sample_chunks = 1
start_unix_millis = 1704067200000
step_millis = 60000
chunk_series_count = 256
timeout_seconds = 600
visibility_timeout_seconds = 300
[scenario.remote_write.value]
pattern = "mixed_signal_repeated"
base = 0
step = 0.125
mixed_every = 5
[scenario.remote_write.prom_store]
pending_rows_flush_interval = "1s"
max_batch_rows = 100000
[scenario.remote_write.storage]
column = "greptime_value"
min_files = 2
min_files_with_column = 2
[scenario.remote_write.read_bench]
iterations = 7
max_files = 4
[[scenario.queries]]
name = "selector_10d_mixed_every"
kind = "tql"
query = "TQL ANALYZE VERBOSE (1704067200, 1704931200, '1h') prom_remote_write_mixed_every{host=~'host.*'}"
warmup = 2
iterations = 15
[scenario.queries.thresholds]
max_candidate_latency_regression_pct = 75
@@ -0,0 +1,50 @@
[case]
name = "prom_remote_write_run_heavy"
description = "End-to-end Prometheus remote-write regression with repeated 16-sample value runs"
[scenario]
kind = "prom_remote_write_then_query"
[scenario.remote_write]
database = "public"
metric = "prom_remote_write_run_heavy"
physical_table = "greptime_physical_table"
series_count = 2048
samples_per_series = 14400
sample_chunk_size = 1440
flush_every_sample_chunks = 1
start_unix_millis = 1704067200000
step_millis = 60000
chunk_series_count = 256
timeout_seconds = 600
visibility_timeout_seconds = 300
[scenario.remote_write.value]
pattern = "run_length"
base = 0
step = 1
cardinality = 1024
run_length = 16
[scenario.remote_write.prom_store]
pending_rows_flush_interval = "1s"
max_batch_rows = 100000
[scenario.remote_write.storage]
column = "greptime_value"
min_files = 2
min_files_with_column = 2
[scenario.remote_write.read_bench]
iterations = 7
max_files = 4
[[scenario.queries]]
name = "selector_10d_run_heavy"
kind = "tql"
query = "TQL ANALYZE VERBOSE (1704067200, 1704931200, '1h') prom_remote_write_run_heavy{host=~'host.*'}"
warmup = 2
iterations = 15
[scenario.queries.thresholds]
max_candidate_latency_regression_pct = 75
@@ -0,0 +1,48 @@
[case]
name = "prom_remote_write_seeded_random"
description = "End-to-end Prometheus remote-write regression with high-distinct seeded-random values"
[scenario]
kind = "prom_remote_write_then_query"
[scenario.remote_write]
database = "public"
metric = "prom_remote_write_seeded_random"
physical_table = "greptime_physical_table"
series_count = 2048
samples_per_series = 14400
sample_chunk_size = 1440
flush_every_sample_chunks = 1
start_unix_millis = 1704067200000
step_millis = 60000
chunk_series_count = 256
timeout_seconds = 600
visibility_timeout_seconds = 300
[scenario.remote_write.value]
pattern = "seeded_random"
cardinality = 29_491_200
seed = 8444
[scenario.remote_write.prom_store]
pending_rows_flush_interval = "1s"
max_batch_rows = 100000
[scenario.remote_write.storage]
column = "greptime_value"
min_files = 2
min_files_with_column = 2
[scenario.remote_write.read_bench]
iterations = 7
max_files = 4
[[scenario.queries]]
name = "selector_10d_seeded_random"
kind = "tql"
query = "TQL ANALYZE VERBOSE (1704067200, 1704931200, '1h') prom_remote_write_seeded_random{host=~'host.*'}"
warmup = 2
iterations = 15
[scenario.queries.thresholds]
max_candidate_latency_regression_pct = 75
@@ -1,31 +0,0 @@
[case]
name = "prom_remote_write_smoke"
description = "Tiny Prometheus remote-write ingestion then TQL query smoke case"
[scenario]
kind = "prom_remote_write_then_query"
[scenario.remote_write]
database = "public"
metric = "prom_remote_write_smoke"
physical_table = "greptime_physical_table"
series_count = 8
samples_per_series = 30
start_unix_millis = 1_704_067_200_000
step_millis = 15_000
chunk_series_count = 4
timeout_seconds = 30
[scenario.remote_write.prom_store]
pending_rows_flush_interval = "1s"
max_batch_rows = 10000
[[scenario.queries]]
name = "selector_5m_smoke"
kind = "tql"
query = "TQL ANALYZE VERBOSE (1704067200, 1704067650, '15s') prom_remote_write_smoke{host=~'host.*'}"
warmup = 0
iterations = 2
[scenario.queries.thresholds]
max_candidate_latency_regression_pct = 100
+328 -49
View File
@@ -66,7 +66,8 @@ def parse_args() -> argparse.Namespace:
p.add_argument("--base-bin", required=True, type=Path)
p.add_argument("--candidate-bin", required=True, type=Path)
p.add_argument("--fixture-generator", type=Path)
p.add_argument("--remote-write-generator", type=Path, help="prom_remote_write_fixture helper binary")
p.add_argument("--remote-write-generator", type=Path, help="deprecated: use --fixture-generator query_perf_fixture")
p.add_argument("--storage-inspector", type=Path, help="deprecated: use --fixture-generator query_perf_fixture")
p.add_argument("--work-dir", required=True, type=Path)
p.add_argument("--fixture-cache-dir", type=Path, help="persistent directory for generated fixtures, keyed by case content")
p.add_argument("--reuse-fixture", action="store_true")
@@ -83,6 +84,16 @@ def load_case(path: Path) -> dict[str, Any]:
return tomllib.load(f)
def load_normalized_case(case_path: Path, fixture_generator: Path | None) -> dict[str, Any]:
if fixture_generator is None:
raise ValueError("--fixture-generator query_perf_fixture is required for Rust-owned case planning")
result = run_command([str(fixture_generator), "plan", "--case", str(case_path)])
if result["returncode"] != 0:
raise RuntimeError(f"query_perf_fixture plan failed: {result['stderr'][:2000]}")
plan = json.loads(result["stdout"])
return {"case": load_case(case_path).get("case", {}), "scenario": plan["scenario"], "schema_version": plan.get("schema_version")}
def require_binary(path: Path, name: str, *, dry_run: bool) -> None:
if dry_run:
return
@@ -157,11 +168,9 @@ def scenario(case: dict[str, Any]) -> dict[str, Any]:
def case_tables(case: dict[str, Any]) -> list[dict[str, Any]]:
value = scenario(case)
if value.get("kind") == "prom_remote_write_then_query":
remote = value.get("remote_write") or {}
metric = remote.get("metric") or remote.get("metric_name")
database = remote.get("database", "public")
if not metric:
raise ValueError("remote-write scenario requires scenario.remote_write.metric")
remote = value["remote_write"]
metric = remote["metric"]
database = remote["database"]
return [{"database": database, "name": metric, "engine": "metric", "validate_show_create_engine": False}]
tables = value.get("tables") or []
if not tables or value.get("layout", {}).get("regions") != 1:
@@ -548,7 +557,7 @@ def generate_fixture(generator: Path | None, case_path: Path, fixture_dir: Path,
if fixture_dir.exists() and not reuse_fixture:
shutil.rmtree(fixture_dir)
fixture_dir.mkdir(parents=True, exist_ok=True)
cmd = [str(generator), "--case", str(case_path), "--out-dir", str(fixture_dir)]
cmd = [str(generator), "direct-sst", "--case", str(case_path), "--out-dir", str(fixture_dir)]
if table_name is not None:
cmd += ["--table", table_name]
if region_id is not None:
@@ -777,6 +786,243 @@ def enforce_thresholds(case: dict[str, Any], base: dict[str, Any], candidate: di
return results
def storage_config(remote: dict[str, Any]) -> dict[str, Any] | None:
value = remote["storage"]
if not isinstance(value, dict):
return None
if value["inspect"] is False:
return None
return value
def run_storage_inspection(helper: Path | None, target: RunTarget, storage: dict[str, Any], *, dry_run: bool) -> dict[str, Any]:
if helper is None:
if not dry_run:
raise ValueError("--fixture-generator query_perf_fixture is required when storage inspection is enabled")
helper = Path("query_perf_fixture")
root = target.datanode_data_dir
if storage.get("root_suffix"):
root = root / str(storage["root_suffix"])
cmd = [
str(helper),
"inspect-footer",
"--root", str(root),
"--column", str(storage["column"]),
]
if storage["include_metadata_files"]:
cmd.append("--include-metadata-files")
if dry_run:
return {"status": "dry-run", "cmd": cmd, "root": str(root)}
result = run_command(cmd)
result["status"] = "ok" if result["returncode"] == 0 else "failed"
if result["returncode"] != 0:
raise RuntimeError(f"storage inspector failed for {target.name}: {result['stderr'][:2000]}")
try:
result["summary"] = json.loads(result["stdout"])
except json.JSONDecodeError:
result["summary_parse_error"] = result["stdout"]
result["status"] = "failed"
return result
def storage_summary(inspection: dict[str, Any]) -> dict[str, Any]:
summary = inspection.get("summary")
if isinstance(summary, dict):
nested = summary.get("summary")
if isinstance(nested, dict):
return nested
return {}
def enforce_storage_thresholds(storage: dict[str, Any] | None, base: dict[str, Any] | None, candidate: dict[str, Any] | None) -> list[dict[str, Any]]:
if not storage:
return []
results: list[dict[str, Any]] = []
target_summaries = [("base", storage_summary(base or {})), ("candidate", storage_summary(candidate or {}))]
cand = target_summaries[1][1]
base_summary = storage_summary(base or {})
def add_abs(name: str, field: str) -> None:
limit = storage.get(name)
if limit is None:
return
for target_name, summary in target_summaries:
actual = summary.get(field)
ok = actual is not None and float(actual) <= float(limit)
results.append({"target": target_name, "threshold": name, "status": "passed" if ok else "failed", "actual": actual, "limit": limit})
def add_min(name: str, field: str) -> None:
limit = storage[name]
if limit is None:
return
for target_name, summary in target_summaries:
actual = summary.get(field)
ok = actual is not None and int(actual) >= int(limit)
results.append({"target": target_name, "threshold": name, "status": "passed" if ok else "failed", "actual": actual, "limit": limit})
def add_cmp(name: str, field: str) -> None:
limit = storage.get(name)
if limit is None:
return
base_value = base_summary.get(field)
candidate_value = cand.get(field)
if base_value in (None, 0) or candidate_value is None:
results.append({"threshold": name, "status": "failed", "reason": "missing or zero base/candidate value", "base": base_value, "candidate": candidate_value})
return
actual = (float(candidate_value) - float(base_value)) / float(base_value) * 100.0
results.append({"threshold": name, "status": "passed" if actual <= float(limit) else "failed", "actual_pct": actual, "limit_pct": limit, "base": base_value, "candidate": candidate_value})
add_min("min_files", "file_count")
add_min("min_files_with_column", "files_with_column")
add_abs("max_total_file_size_bytes", "total_file_size")
add_abs("max_column_compressed_size_bytes", "column_compressed_size")
add_abs("max_column_uncompressed_size_bytes", "column_uncompressed_size")
for target_name, summary in target_summaries:
encodings = set(summary.get("unique_encodings") or [])
for required in storage["require_encodings"]:
results.append({"target": target_name, "threshold": "require_encodings", "encoding": required, "status": "passed" if required in encodings else "failed"})
for forbidden in storage["forbid_encodings"]:
results.append({"target": target_name, "threshold": "forbid_encodings", "encoding": forbidden, "status": "passed" if forbidden not in encodings else "failed"})
add_cmp("max_candidate_total_file_size_regression_pct", "total_file_size")
add_cmp("max_candidate_column_compressed_size_regression_pct", "column_compressed_size")
add_cmp("max_candidate_column_uncompressed_size_regression_pct", "column_uncompressed_size")
return results
def planned_storage_thresholds(storage: dict[str, Any] | None) -> list[dict[str, Any]]:
if not storage:
return []
return list(storage["planned_thresholds"])
REGION_DIR_RE = re.compile(r"^(\d+)_(\d{10})$")
def inspection_report(inspection: dict[str, Any] | None) -> dict[str, Any]:
if not inspection:
return {}
report = inspection.get("summary")
return report if isinstance(report, dict) else {}
def inspected_relative_path(target: RunTarget, inspection: dict[str, Any], relative_path: str) -> Path:
report = inspection_report(inspection)
root_text = report.get("root") or inspection.get("root")
if not root_text:
return Path(relative_path)
root = Path(root_text).resolve(strict=False)
data_home = target.datanode_data_dir.resolve(strict=False)
try:
root_suffix = root.relative_to(data_home)
except ValueError as e:
raise RuntimeError(f"storage inspection root {root} is not under datanode data home {data_home}") from e
return root_suffix / relative_path
def parse_sst_bench_target(target: RunTarget, inspection: dict[str, Any], file: dict[str, Any], *, dry_run: bool) -> dict[str, str] | None:
relative_path = str(file.get("relative_path") or "")
if dry_run and (not relative_path or "<" in relative_path):
return {
"relative_path": relative_path or "<table-dir>/<table-id>_<region-seq>/data/<file-id>.parquet",
"table_dir": "<table-dir>/",
"region_id": "<table-id>:<region-seq>",
"path_type": "data",
"file_id": "<file-id>",
}
if not relative_path:
return None
full_relative = inspected_relative_path(target, inspection, relative_path)
parts = full_relative.parts
if not parts or not parts[-1].endswith(".parquet"):
return None
region_index = next((idx for idx, part in enumerate(parts) if REGION_DIR_RE.match(part)), None)
if region_index is None or region_index == 0:
if dry_run:
return {
"relative_path": relative_path,
"table_dir": "<table-dir>/",
"region_id": "<table-id>:<region-seq>",
"path_type": "data",
"file_id": Path(parts[-1]).stem,
}
return None
region_match = REGION_DIR_RE.match(parts[region_index])
assert region_match is not None
file_index = len(parts) - 1
path_type = "bare"
if region_index + 1 < file_index and parts[region_index + 1] in ("data", "metadata"):
path_type = parts[region_index + 1]
if path_type == "metadata":
return None
table_dir = "/".join(parts[:region_index]).rstrip("/") + "/"
return {
"relative_path": str(full_relative),
"table_dir": table_dir,
"region_id": f"{int(region_match.group(1))}:{int(region_match.group(2))}",
"path_type": path_type,
"file_id": Path(parts[-1]).stem,
}
def run_read_bench(bench_binary: Path, target: RunTarget, read_bench: dict[str, Any] | None, storage_inspection: dict[str, Any] | None, *, dry_run: bool) -> dict[str, Any]:
if not read_bench or read_bench["enabled"] is False:
return {"status": "skipped", "reason": "read_bench disabled"}
run_parquetbench = read_bench["parquetbench"]
run_scanbench = read_bench["scanbench"]
if not run_parquetbench and not run_scanbench:
return {"status": "skipped", "reason": "parquetbench and scanbench disabled"}
if storage_inspection is None:
raise ValueError("read_bench requires storage inspection")
bench_dir = target.work_dir / "read_bench"
config_toml = bench_dir / "bench.toml"
scan_json = bench_dir / "scan.json"
files = inspection_report(storage_inspection).get("files") or []
ssts = [f for f in files if f.get("relative_path", "").endswith(".parquet") and f.get("columns")]
if dry_run and not ssts:
ssts = [{"relative_path": "<landed-sst>.parquet"}]
if read_bench["max_files"] is not None:
ssts = ssts[: int(read_bench["max_files"])]
bench_targets = [parsed for f in ssts if (parsed := parse_sst_bench_target(target, storage_inspection, f, dry_run=dry_run)) is not None]
if not bench_targets:
return {"status": "failed", "reason": "no inspected data SST files available for read_bench"}
config_text = f'[storage]\ndata_home = "{target.datanode_data_dir}"\ntype = "File"\n\n[[region_engine]]\n[region_engine.mito]\n'
commands = []
parquet_runs = []
scan_runs = []
iterations = str(int(read_bench["iterations"]))
if run_parquetbench:
for bench_target in bench_targets:
cmd = [str(bench_binary), "datanode", "parquetbench", "--config", str(config_toml), "--region-id", bench_target["region_id"], "--table-dir", bench_target["table_dir"], "--file-id", bench_target["file_id"], "--scan-config", str(scan_json), "--path-type", bench_target["path_type"], "--iterations", iterations, "--reader", str(read_bench["parquet_reader"])]
commands.append(cmd)
parquet_runs.append({**bench_target, "cmd": cmd, "status": "dry-run" if dry_run else "planned"})
regions: dict[tuple[str, str, str], list[str]] = {}
if run_scanbench:
for bench_target in bench_targets:
regions.setdefault((bench_target["table_dir"], bench_target["region_id"], bench_target["path_type"]), []).append(bench_target["relative_path"])
for (table_dir, region_id, path_type), region_files in regions.items():
cmd = [str(bench_binary), "datanode", "scanbench", "--config", str(config_toml), "--region-id", region_id, "--table-dir", table_dir, "--scan-config", str(scan_json), "--path-type", path_type, "--scanner", str(read_bench["scan_scanner"]), "--parallelism", str(int(read_bench["parallelism"])), "--iterations", iterations]
commands.append(cmd)
scan_runs.append({"table_dir": table_dir, "region_id": region_id, "path_type": path_type, "files": region_files, "cmd": cmd, "status": "dry-run" if dry_run else "planned"})
if dry_run:
return {"status": "dry-run", "config_path": str(config_toml), "scan_config_path": str(scan_json), "commands": commands, "parquetbench": parquet_runs, "scanbench": scan_runs}
bench_dir.mkdir(parents=True, exist_ok=True)
config_toml.write_text(config_text)
scan_json.write_text(json.dumps({"projection_names": read_bench["projection"]}, indent=2) + "\n")
avg_re = re.compile(r"Average duration[^0-9]*([0-9.]+)\s*ms", re.I)
for runs in (parquet_runs, scan_runs):
for run in runs:
result = run_command(run["cmd"])
run.update(result)
run["status"] = "ok" if result["returncode"] == 0 else "failed"
m = avg_re.search(result.get("stdout", ""))
if m:
run["average_ms"] = float(m.group(1))
def median(runs: list[dict[str, Any]]) -> float | None:
vals = [r["average_ms"] for r in runs if "average_ms" in r]
return statistics.median(vals) if vals else None
return {"status": "ok" if all(r.get("status") == "ok" for r in parquet_runs + scan_runs) else "failed", "config_path": str(config_toml), "scan_config_path": str(scan_json), "parquetbench": parquet_runs, "scanbench": scan_runs, "aggregate": {"parquetbench_median_average_ms": median(parquet_runs), "scanbench_median_average_ms": median(scan_runs)}}
def write_json(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
@@ -791,18 +1037,18 @@ def require_fresh_work_dirs(targets: list[RunTarget], *, reuse_work_dir: bool, d
def write_frontend_prom_config(target: RunTarget, remote: dict[str, Any]) -> Path:
prom = remote.get("prom_store") or {}
prom = remote["prom_store"]
path = target.work_dir / "frontend-prom-store.toml"
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
"[prom_store]\n"
"enable = true\n"
"with_metric_engine = true\n"
f"pending_rows_flush_interval = {json.dumps(str(prom.get('pending_rows_flush_interval', '1s')))}\n"
f"max_batch_rows = {int(prom.get('max_batch_rows', 100000))}\n"
f"max_concurrent_flushes = {int(prom.get('max_concurrent_flushes', 256))}\n"
f"worker_channel_capacity = {int(prom.get('worker_channel_capacity', 65526))}\n"
f"max_inflight_requests = {int(prom.get('max_inflight_requests', 3000))}\n"
f"pending_rows_flush_interval = {json.dumps(str(prom['pending_rows_flush_interval']))}\n"
f"max_batch_rows = {int(prom['max_batch_rows'])}\n"
f"max_concurrent_flushes = {int(prom['max_concurrent_flushes'])}\n"
f"worker_channel_capacity = {int(prom['worker_channel_capacity'])}\n"
f"max_inflight_requests = {int(prom['max_inflight_requests'])}\n"
)
return path
@@ -810,23 +1056,38 @@ def write_frontend_prom_config(target: RunTarget, remote: dict[str, Any]) -> Pat
def run_remote_write(generator: Path | None, target: RunTarget, remote: dict[str, Any], *, dry_run: bool) -> dict[str, Any]:
if generator is None:
if not dry_run:
raise ValueError("--remote-write-generator is required for prom_remote_write_then_query unless --dry-run is set")
generator = Path("prom_remote_write_fixture")
metric = remote.get("metric") or remote.get("metric_name")
physical_table = remote.get("physical_table", "greptime_physical_table")
raise ValueError("--fixture-generator query_perf_fixture is required for prom_remote_write_then_query")
generator = Path("query_perf_fixture")
metric = remote["metric"]
physical_table = remote["physical_table"]
cmd = [
str(generator),
"prom-remote-write",
"--endpoint", f"http://127.0.0.1:{target.http_port}/v1/prometheus/write",
"--database", remote.get("database", "public"),
"--database", remote["database"],
"--metric", metric,
"--physical-table", physical_table,
"--series-count", str(int(remote.get("series_count", 8))),
"--samples-per-series", str(int(remote.get("samples_per_series", 30))),
"--start-unix-millis", str(int(remote.get("start_unix_millis", 1_704_067_200_000))),
"--step-millis", str(int(remote.get("step_millis", 15_000))),
"--chunk-series-count", str(int(remote.get("chunk_series_count", remote.get("batch_size", 8)))),
"--timeout-seconds", str(int(remote.get("timeout_seconds", 60))),
"--series-count", str(int(remote["series_count"])),
"--samples-per-series", str(int(remote["samples_per_series"])),
"--start-unix-millis", str(int(remote["start_unix_millis"])),
"--step-millis", str(int(remote["step_millis"])),
"--chunk-series-count", str(int(remote["chunk_series_count"])),
"--timeout-seconds", str(int(remote["timeout_seconds"])),
]
value = remote["value"]
cmd.extend(["--value-pattern", str(value["pattern"])])
cmd.extend(["--value-base", str(value["base"])])
cmd.extend(["--value-step", str(value["step"])])
cmd.extend(["--value-cardinality", str(int(value["cardinality"]))])
cmd.extend(["--value-seed", str(int(value["seed"]))])
cmd.extend(["--value-run-length", str(int(value["run_length"]))])
cmd.extend(["--value-stall-every", str(int(value["stall_every"]))])
cmd.extend(["--value-stall-length", str(int(value["stall_length"]))])
cmd.extend(["--value-mixed-every", str(int(value["mixed_every"]))])
if "sample_offset" in remote:
cmd.extend(["--value-sample-offset", str(int(remote["sample_offset"]))])
if "total_samples_per_series" in remote:
cmd.extend(["--value-total-samples-per-series", str(int(remote["total_samples_per_series"]))])
if dry_run:
return {"status": "dry-run", "cmd": cmd}
result = run_command(cmd)
@@ -862,23 +1123,23 @@ def flush_remote_physical_table(target: RunTarget, db: str, physical_table: str,
def run_remote_write_ingestion(generator: Path | None, target: RunTarget, remote: dict[str, Any], args: argparse.Namespace, *, dry_run: bool) -> tuple[dict[str, Any], list[dict[str, Any]]]:
sample_chunk_size = remote.get("sample_chunk_size")
db = remote.get("database", "public")
physical_table = remote.get("physical_table", "greptime_physical_table")
sample_chunk_size = remote["sample_chunk_size"]
db = remote["database"]
physical_table = remote["physical_table"]
if sample_chunk_size is None:
rw = run_remote_write(generator, target, remote, dry_run=dry_run)
flush = flush_remote_physical_table(target, db, physical_table, args, dry_run=dry_run, reason="final")
return rw, [flush]
total_samples = int(remote.get("samples_per_series", 30))
total_samples = int(remote["samples_per_series"])
chunk_samples = int(sample_chunk_size)
if chunk_samples <= 0:
raise ValueError("scenario.remote_write.sample_chunk_size must be positive")
flush_every = int(remote.get("flush_every_sample_chunks", 1))
flush_every = int(remote["flush_every_sample_chunks"])
if flush_every <= 0:
raise ValueError("scenario.remote_write.flush_every_sample_chunks must be positive")
start = int(remote.get("start_unix_millis", 1_704_067_200_000))
step = int(remote.get("step_millis", 15_000))
start = int(remote["start_unix_millis"])
step = int(remote["step_millis"])
chunks: list[dict[str, Any]] = []
flushes: list[dict[str, Any]] = []
chunk_index = 0
@@ -889,6 +1150,8 @@ def run_remote_write_ingestion(generator: Path | None, target: RunTarget, remote
chunk_remote = dict(remote)
chunk_remote["samples_per_series"] = current
chunk_remote["start_unix_millis"] = start + offset * step
chunk_remote["sample_offset"] = offset
chunk_remote["total_samples_per_series"] = total_samples
result = run_remote_write(generator, target, chunk_remote, dry_run=dry_run)
result["sample_offset"] = offset
result["samples_per_series"] = current
@@ -901,8 +1164,8 @@ def run_remote_write_ingestion(generator: Path | None, target: RunTarget, remote
flushes.append(flush_remote_physical_table(target, db, physical_table, args, dry_run=dry_run, reason="final", chunk_index=chunk_index))
aggregate = summarize_remote_write_chunks(chunks)
if dry_run:
series_count = int(remote.get("series_count", 8))
chunk_series_count = int(remote.get("chunk_series_count", remote.get("batch_size", 8)))
series_count = int(remote["series_count"])
chunk_series_count = int(remote["chunk_series_count"])
aggregate["rows"] = series_count * total_samples
aggregate["samples_written"] = aggregate["rows"]
aggregate["batches"] = chunk_index * ((series_count + chunk_series_count - 1) // chunk_series_count)
@@ -917,7 +1180,7 @@ def run_remote_write_ingestion(generator: Path | None, target: RunTarget, remote
def expected_remote_write_rows(remote: dict[str, Any]) -> int:
return int(remote.get("series_count", 8)) * int(remote.get("samples_per_series", 30))
return int(remote["series_count"]) * int(remote["samples_per_series"])
def extract_count_value(result: dict[str, Any]) -> int | None:
@@ -966,16 +1229,21 @@ def poll_expected_count(target: RunTarget, table_name: str, db: str, expected_ro
def run_remote_write_scenario(args: argparse.Namespace, case: dict[str, Any], case_path: Path, targets: list[RunTarget], report: dict[str, Any]) -> None:
remote = scenario(case).get("remote_write") or {}
remote = scenario(case)["remote_write"]
tables = case_tables(case)
if args.fixture_only:
raise ValueError("--fixture-only is not supported for prom_remote_write_then_query; use --dry-run for planning")
if args.remote_write_generator is not None:
require_binary(args.remote_write_generator, "remote-write generator", dry_run=args.dry_run)
helper = args.fixture_generator or args.remote_write_generator
if helper is not None:
require_binary(helper, "query_perf_fixture", dry_run=args.dry_run)
elif not args.dry_run:
raise ValueError("--remote-write-generator is required for prom_remote_write_then_query")
raise ValueError("--fixture-generator is required for prom_remote_write_then_query")
storage = storage_config(remote)
if storage and helper is None and not args.dry_run:
raise ValueError("--fixture-generator is required when scenario.remote_write.storage.inspect is enabled")
clusters: list[DistributedCluster] = []
target_results = []
storage_results = []
try:
for target in targets:
target.work_dir.mkdir(parents=True, exist_ok=True)
@@ -984,28 +1252,40 @@ def run_remote_write_scenario(args: argparse.Namespace, case: dict[str, Any], ca
clusters.append(cluster)
if not args.dry_run:
cluster.start_all(config_path)
db = remote.get("database", "public")
db = remote["database"]
create_database = {"status": "dry-run", "database": db}
if not args.dry_run:
create_database = http_post_sql(target.http_port, f"CREATE DATABASE IF NOT EXISTS {sql_ident(db)}", "public", args.http_timeout)
rw, flushes = run_remote_write_ingestion(args.remote_write_generator, target, remote, args, dry_run=args.dry_run)
physical = remote.get("physical_table", "greptime_physical_table")
rw, flushes = run_remote_write_ingestion(helper, target, remote, args, dry_run=args.dry_run)
visibility = {"status": "dry-run"}
query_result = {"validation": [], "validation_errors": [], "measurements": [], "status": "planned"}
if not args.dry_run:
visibility = poll_expected_count(target, tables[0]["name"], db, expected_remote_write_rows(remote), float(remote.get("visibility_timeout_seconds", 30)), args.http_timeout)
visibility = poll_expected_count(target, tables[0]["name"], db, expected_remote_write_rows(remote), float(remote["visibility_timeout_seconds"]), args.http_timeout)
query_result = run_queries(target, case, tables, args.http_timeout)
cluster.stop_component("datanode")
storage_inspection = None
if storage:
storage_inspection = run_storage_inspection(helper or args.storage_inspector, target, storage, dry_run=args.dry_run)
storage_results.append(storage_inspection)
read_bench_result = run_read_bench(args.candidate_bin, target, remote["read_bench"], storage_inspection, dry_run=args.dry_run)
tr = {"name": target.name, "binary": str(target.binary), "work_dir": str(target.work_dir), "components": cluster.component_report(), "frontend_config": str(config_path), "create_database": create_database, "remote_write": rw, "flushes": flushes, "flush": flushes[-1] if flushes else None, "visibility": visibility, **query_result}
if storage_inspection is not None:
tr["storage_inspection"] = storage_inspection
tr["read_bench"] = read_bench_result
flushes_ok = all(flush.get("ok") for flush in flushes)
remote_checks_ok = args.dry_run or (create_database.get("ok") and flushes_ok and visibility.get("ok") and visibility.get("row_count_ok"))
storage_ok = storage_inspection is None or args.dry_run or storage_inspection.get("status") == "ok"
read_bench_ok = args.dry_run or read_bench_result.get("status") in ("ok", "skipped")
remote_checks_ok = args.dry_run or (create_database.get("ok") and flushes_ok and visibility.get("ok") and visibility.get("row_count_ok") and storage_ok and read_bench_ok)
if not remote_checks_ok:
tr.setdefault("validation_errors", []).append({"phase": "remote_write_visibility", "create_database_ok": create_database.get("ok"), "flushes_ok": flushes_ok, "visibility_ok": visibility.get("ok"), "row_count_ok": visibility.get("row_count_ok"), "create_database": create_database, "flushes": flushes, "visibility": visibility})
tr.setdefault("validation_errors", []).append({"phase": "remote_write_visibility_storage", "create_database_ok": create_database.get("ok"), "flushes_ok": flushes_ok, "visibility_ok": visibility.get("ok"), "row_count_ok": visibility.get("row_count_ok"), "storage_ok": storage_ok, "read_bench_ok": read_bench_ok, "create_database": create_database, "flushes": flushes, "visibility": visibility, "read_bench": read_bench_result})
tr["status"] = "planned" if args.dry_run else ("measured" if remote_checks_ok and query_result["status"] == "ok" else "failed")
write_json(target.report_path, tr)
report["targets"].append(tr)
target_results.append(query_result)
if not args.dry_run:
report["thresholds"] = enforce_thresholds(case, target_results[0], target_results[1])
report["thresholds"] = enforce_thresholds(case, target_results[0], target_results[1]) + enforce_storage_thresholds(storage, storage_results[0] if storage_results else None, storage_results[1] if len(storage_results) > 1 else None)
elif storage:
report["thresholds"] = planned_storage_thresholds(storage)
report["status"] = "planned" if args.dry_run else ("failed" if any(t["status"] == "failed" for t in report["thresholds"]) or any(t.get("status") == "failed" for t in report["targets"]) else "ok")
finally:
for cluster in reversed(clusters):
@@ -1015,7 +1295,9 @@ def run_remote_write_scenario(args: argparse.Namespace, case: dict[str, Any], ca
def main() -> int:
args = parse_args()
case_path = args.case.resolve()
case = load_case(case_path)
if args.fixture_generator is not None:
require_binary(args.fixture_generator, "fixture generator", dry_run=False)
case = load_normalized_case(case_path, args.fixture_generator)
scenario_config = scenario(case)
scenario_kind = scenario_config.get("kind")
tables = case_tables(case)
@@ -1025,9 +1307,6 @@ def main() -> int:
require_binary(args.candidate_bin, "candidate", dry_run=args.dry_run or args.fixture_only)
if scenario_kind == "direct_readable_sst" and not args.dry_run and args.fixture_generator is None:
raise ValueError("--fixture-generator is required unless --dry-run is set")
if args.fixture_generator is not None:
require_binary(args.fixture_generator, "fixture generator", dry_run=args.dry_run)
ports = allocate_ports(16)
targets = [make_target("base", args.base_bin.resolve(), work_root, ports[:8]), make_target("candidate", args.candidate_bin.resolve(), work_root, ports[8:])]
require_fresh_work_dirs(targets, reuse_work_dir=args.reuse_work_dir, dry_run=args.dry_run, fixture_only=args.fixture_only)
@@ -0,0 +1,158 @@
#!/usr/bin/env python3
# Copyright 2023 Greptime Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Regression coverage for quiescing remote-write storage before read benches."""
import argparse
import importlib.util
import sys
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
RUNNER_PATH = Path(__file__).with_name("query_regression_runner.py")
SPEC = importlib.util.spec_from_file_location("query_regression_runner_under_test", RUNNER_PATH)
assert SPEC is not None and SPEC.loader is not None
runner = importlib.util.module_from_spec(SPEC)
sys.modules[SPEC.name] = runner
SPEC.loader.exec_module(runner)
class RemoteWriteCompactionToctouTest(unittest.TestCase):
def test_quiesces_each_target_before_storage_inspection_and_read_bench(self) -> None:
lifecycle: dict[str, list[str]] = {"base": [], "candidate": []}
clusters = []
class RecordingCluster:
def __init__(self, target):
self.target = target
self.active = set()
self.stop_all_calls = 0
clusters.append(self)
def start_all(self, _config_path):
self.active.update(("metasrv", "datanode", "frontend"))
def stop_component(self, name):
if name in self.active:
lifecycle[self.target.name].append(f'cluster.stop_component("{name}")')
self.active.remove(name)
def stop_all(self):
self.stop_all_calls += 1
for name in ("frontend", "datanode", "metasrv"):
self.stop_component(name)
def component_report(self):
return {}
def run_queries(target, _case, _tables, _http_timeout):
lifecycle[target.name].append("query measurement completes")
return {"validation": [], "validation_errors": [], "measurements": [], "status": "ok"}
def inspect_storage(_helper, target, _storage, *, dry_run):
self.assertFalse(dry_run)
lifecycle[target.name].append("storage inspection")
return {"status": "ok", "summary": {"summary": {}}}
def read_bench(_binary, target, _read_bench, _inspection, *, dry_run):
self.assertFalse(dry_run)
lifecycle[target.name].append("read bench")
return {"status": "ok"}
remote_write = {
"database": "public",
"metric": "cpu_usage",
"physical_table": "cpu_usage_physical",
"series_count": 1,
"samples_per_series": 1,
"visibility_timeout_seconds": 1,
"storage": {
"inspect": True,
"min_files": None,
"min_files_with_column": None,
"max_total_file_size_bytes": None,
"max_column_compressed_size_bytes": None,
"max_column_uncompressed_size_bytes": None,
"require_encodings": [],
"forbid_encodings": [],
},
"read_bench": {"enabled": True},
"prom_store": {
"pending_rows_flush_interval": "1s",
"max_batch_rows": 1,
"max_concurrent_flushes": 1,
"worker_channel_capacity": 1,
"max_inflight_requests": 1,
},
}
case = {
"scenario": {
"kind": "prom_remote_write_then_query",
"remote_write": remote_write,
"queries": [],
}
}
args = argparse.Namespace(
fixture_only=False,
fixture_generator=Path("/bin/true"),
remote_write_generator=None,
storage_inspector=None,
candidate_bin=Path("/bin/true"),
dry_run=False,
http_timeout=1.0,
)
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir)
targets = [
runner.make_target("base", Path("/bin/true"), root, list(range(10000, 10008))),
runner.make_target("candidate", Path("/bin/true"), root, list(range(10008, 10016))),
]
report = {"targets": [], "thresholds": []}
with (
patch.object(runner, "DistributedCluster", RecordingCluster),
patch.object(runner, "run_remote_write_ingestion", return_value=({"status": "ok"}, [{"ok": True}])),
patch.object(runner, "http_post_sql", return_value={"ok": True}),
patch.object(runner, "poll_expected_count", return_value={"ok": True, "row_count_ok": True}),
patch.object(runner, "run_queries", side_effect=run_queries),
patch.object(runner, "run_storage_inspection", side_effect=inspect_storage),
patch.object(runner, "run_read_bench", side_effect=read_bench),
):
runner.run_remote_write_scenario(args, case, root / "case.toml", targets, report)
for cluster in clusters:
self.assertEqual(cluster.stop_all_calls, 1)
self.assertEqual(cluster.active, set())
cluster.stop_all()
self.assertEqual(cluster.stop_all_calls, 2)
self.assertEqual(cluster.active, set())
expected = [
"query measurement completes",
'cluster.stop_component("datanode")',
"storage inspection",
"read bench",
]
relevant = {name: [event for event in events if event in expected] for name, events in lifecycle.items()}
for target_name in ("base", "candidate"):
with self.subTest(target=target_name):
self.assertEqual(relevant[target_name], expected)
if __name__ == "__main__":
unittest.main()