mirror of
https://github.com/GreptimeTeam/greptimedb.git
synced 2026-11-17 08:14:56 +00:00
Merge branch 'main' into series-expansion
Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
This commit is contained in:
@@ -3,6 +3,7 @@ logging:
|
||||
format: "json"
|
||||
filters:
|
||||
- log_store=debug
|
||||
- rskafka=debug
|
||||
meta:
|
||||
configData: |-
|
||||
[runtime]
|
||||
|
||||
@@ -3,6 +3,7 @@ logging:
|
||||
format: "json"
|
||||
filters:
|
||||
- log_store=debug
|
||||
- rskafka=debug
|
||||
meta:
|
||||
configData: |-
|
||||
[runtime]
|
||||
|
||||
@@ -16,8 +16,8 @@ runs:
|
||||
helm upgrade \
|
||||
--install kafka oci://registry-1.docker.io/bitnamicharts/kafka \
|
||||
--set controller.replicaCount=${{ inputs.controller-replicas }} \
|
||||
--set controller.resources.requests.cpu=50m \
|
||||
--set controller.resources.requests.memory=128Mi \
|
||||
--set controller.resources.requests.cpu=500m \
|
||||
--set controller.resources.requests.memory=512Mi \
|
||||
--set controller.resources.limits.cpu=2000m \
|
||||
--set controller.resources.limits.memory=2Gi \
|
||||
--set listeners.controller.protocol=PLAINTEXT \
|
||||
|
||||
@@ -26,6 +26,10 @@ from pathlib import Path
|
||||
|
||||
DEFAULT_CASES = [
|
||||
"tests/perf/query_cases/smoke_direct_sst/case.toml",
|
||||
"tests/perf/query_cases/prom_remote_write_seeded_random/case.toml",
|
||||
"tests/perf/query_cases/prom_remote_write_run_heavy/case.toml",
|
||||
"tests/perf/query_cases/prom_remote_write_mixed_every/case.toml",
|
||||
"tests/perf/query_cases/prom_remote_write_integer_counter/case.toml",
|
||||
"tests/perf/query_cases/promql_pushdown_7913/case.toml",
|
||||
"tests/perf/query_cases/sql_topk_order_by/case.toml",
|
||||
"tests/perf/query_cases/sql_aggregate_order_by/case.toml",
|
||||
@@ -106,8 +110,6 @@ def run_case(args: argparse.Namespace, case_path: Path, work_dir: Path) -> int:
|
||||
str(candidate_bin),
|
||||
"--fixture-generator",
|
||||
str(fixture_generator),
|
||||
"--remote-write-generator",
|
||||
str(args.remote_write_generator),
|
||||
"--work-dir",
|
||||
str(work_dir),
|
||||
"--http-timeout",
|
||||
@@ -160,11 +162,6 @@ def main() -> int:
|
||||
parser.add_argument("--work-dir", default=Path("query-regression-work"), type=Path)
|
||||
parser.add_argument("--http-timeout", default=os.environ.get("HTTP_TIMEOUT", "300"))
|
||||
parser.add_argument("--allow-large-fixture", default=os.environ.get("ALLOW_LARGE_FIXTURE", "false"))
|
||||
parser.add_argument(
|
||||
"--remote-write-generator",
|
||||
type=Path,
|
||||
default=configured_path(os.environ.get("REMOTE_WRITE_GENERATOR")),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--summary-script",
|
||||
type=Path,
|
||||
@@ -177,9 +174,6 @@ def main() -> int:
|
||||
parser.add_argument("--candidate-ref", default=os.environ.get("CANDIDATE_REF", ""))
|
||||
parser.add_argument("--github-output", default=os.environ.get("GITHUB_OUTPUT"))
|
||||
args = parser.parse_args()
|
||||
if args.remote_write_generator is None:
|
||||
args.remote_write_generator = args.candidate_src / "target" / profile_dir(args.cargo_profile) / "prom_remote_write_fixture"
|
||||
|
||||
try:
|
||||
cases = split_cases(args.cases or [os.environ.get("CASE_PATHS", "all")])
|
||||
except ValueError as err:
|
||||
|
||||
@@ -119,10 +119,15 @@ jobs:
|
||||
git reset --hard FETCH_HEAD
|
||||
git clean -ffdx
|
||||
|
||||
- name: Test query regression runner lifecycle
|
||||
working-directory: src
|
||||
run: uv run --no-project python tests/perf/test_query_regression_runner_compaction_toctou.py
|
||||
|
||||
- name: Build candidate greptime and fixture generators
|
||||
working-directory: src
|
||||
run: |
|
||||
cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime --bin query_perf_fixture --bin prom_remote_write_fixture
|
||||
cargo build --profile "${CARGO_PROFILE}" -p cmd --bin greptime
|
||||
cargo build --profile "${CARGO_PROFILE}" -p cmd --bin query_perf_fixture --features dev-tools
|
||||
target_dir="${CARGO_PROFILE}"
|
||||
if [[ "${CARGO_PROFILE}" == "dev" ]]; then
|
||||
target_dir="debug"
|
||||
@@ -132,8 +137,6 @@ jobs:
|
||||
"${GITHUB_WORKSPACE}/query-regression-bins/candidate/greptime"
|
||||
cp "${CARGO_TARGET_DIR}/${target_dir}/query_perf_fixture" \
|
||||
"${GITHUB_WORKSPACE}/query-regression-bins/candidate/query_perf_fixture"
|
||||
cp "${CARGO_TARGET_DIR}/${target_dir}/prom_remote_write_fixture" \
|
||||
"${GITHUB_WORKSPACE}/query-regression-bins/candidate/prom_remote_write_fixture"
|
||||
|
||||
- name: Run query regression
|
||||
id: run
|
||||
@@ -146,7 +149,6 @@ jobs:
|
||||
BASE_BIN: ${{ github.workspace }}/query-regression-bins/base/greptime
|
||||
CANDIDATE_BIN: ${{ github.workspace }}/query-regression-bins/candidate/greptime
|
||||
FIXTURE_GENERATOR: ${{ github.workspace }}/query-regression-bins/candidate/query_perf_fixture
|
||||
REMOTE_WRITE_GENERATOR: ${{ github.workspace }}/query-regression-bins/candidate/prom_remote_write_fixture
|
||||
run: >-
|
||||
uv run --no-project python src/.github/scripts/query-regression-run.py
|
||||
--base-src src
|
||||
@@ -171,9 +173,13 @@ jobs:
|
||||
with:
|
||||
name: query-regression-report
|
||||
path: |
|
||||
query-regression-work/**
|
||||
query-regression-work/**/query-regression-report.json
|
||||
query-regression-work/**/base/report.json
|
||||
query-regression-work/**/candidate/report.json
|
||||
query-regression-work/**/logs/**
|
||||
query-regression-summary.md
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload trusted comment artifact
|
||||
if: ${{ always() && github.event_name == 'pull_request' }}
|
||||
|
||||
Generated
+4
-4
@@ -11038,9 +11038,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "promql-parser"
|
||||
version = "0.7.3"
|
||||
version = "0.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f514ab521bb2b4ce26aca29bf133498b2a2651b785d03a262bbf5772329c279f"
|
||||
checksum = "9e8c90c956be4237a93be6f2298302b56974ef862ab11e0f98869d1f27c69f88"
|
||||
dependencies = [
|
||||
"cfgrammar",
|
||||
"chrono",
|
||||
@@ -11183,7 +11183,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.11.0",
|
||||
"itertools 0.14.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
@@ -11196,7 +11196,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"itertools 0.11.0",
|
||||
"itertools 0.14.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
|
||||
+1
-1
@@ -198,7 +198,7 @@ paste = "1.0"
|
||||
pin-project = "1.0"
|
||||
pretty_assertions = "1.4.0"
|
||||
prometheus = { version = "0.14", features = ["process"] }
|
||||
promql-parser = { version = "0.7.3", features = ["ser"] }
|
||||
promql-parser = { version = "0.10.0", features = ["ser"] }
|
||||
prost = { version = "0.14", features = ["no-recursion-limit"] }
|
||||
prost-types = "0.14"
|
||||
raft-engine = { version = "0.4.1", default-features = false }
|
||||
|
||||
@@ -209,7 +209,13 @@ Read the [v1.0 highlights](https://greptime.com/blogs/2025-11-05-greptimedb-v1-h
|
||||
|
||||
If GreptimeDB is useful to you, please star the repo.
|
||||
|
||||
[](https://www.star-history.com/#GreptimeTeam/GreptimeDB&Date)
|
||||
<a href="https://www.star-history.com/?repos=GreptimeTeam%2FGreptimeDB&type=date&legend=top-left">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://api.star-history.com/chart?repos=GreptimeTeam/GreptimeDB&type=date&theme=dark&legend=top-left&sealed_token=1kD9cyZVPnWvxbpbPPTWo_r7W351gDAhGx9_55zfD1vSIfHsdTRfH6NAtdPVX_M2ac8sjNxiuw7-0Vk4WvbyXmOuufxlAoU3QF7z4onyltr0o0c9xVfUnPKltz7llUc3nEDW7l_r2fHBBMtJGf1-jsitNozdAUpb8eb7OJp9N1LqEnMQK78ESp2zgjyr" />
|
||||
<source media="(prefers-color-scheme: light)" srcset="https://api.star-history.com/chart?repos=GreptimeTeam/GreptimeDB&type=date&legend=top-left&sealed_token=1kD9cyZVPnWvxbpbPPTWo_r7W351gDAhGx9_55zfD1vSIfHsdTRfH6NAtdPVX_M2ac8sjNxiuw7-0Vk4WvbyXmOuufxlAoU3QF7z4onyltr0o0c9xVfUnPKltz7llUc3nEDW7l_r2fHBBMtJGf1-jsitNozdAUpb8eb7OJp9N1LqEnMQK78ESp2zgjyr" />
|
||||
<img alt="Star History Chart" src="https://api.star-history.com/chart?repos=GreptimeTeam/GreptimeDB&type=date&legend=top-left&sealed_token=1kD9cyZVPnWvxbpbPPTWo_r7W351gDAhGx9_55zfD1vSIfHsdTRfH6NAtdPVX_M2ac8sjNxiuw7-0Vk4WvbyXmOuufxlAoU3QF7z4onyltr0o0c9xVfUnPKltz7llUc3nEDW7l_r2fHBBMtJGf1-jsitNozdAUpb8eb7OJp9N1LqEnMQK78ESp2zgjyr" />
|
||||
</picture>
|
||||
</a>
|
||||
|
||||
<img alt="Known Users" src="https://greptime.com/logo/img/users.png"/>
|
||||
|
||||
|
||||
+2
-5
@@ -12,10 +12,7 @@ path = "src/bin/greptime.rs"
|
||||
[[bin]]
|
||||
name = "query_perf_fixture"
|
||||
path = "src/bin/query_perf_fixture.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "prom_remote_write_fixture"
|
||||
path = "src/bin/prom_remote_write_fixture.rs"
|
||||
required-features = ["dev-tools"]
|
||||
|
||||
[features]
|
||||
default = [
|
||||
@@ -25,7 +22,7 @@ default = [
|
||||
"meta-srv/mysql_kvbackend",
|
||||
]
|
||||
enterprise = ["common-meta/enterprise", "frontend/enterprise", "meta-srv/enterprise"]
|
||||
# Developer-only tools (benchmarks, diagnostics) gating the `datanode parquetbench` subcommand.
|
||||
# Developer-only helper binary gate for `query_perf_fixture`.
|
||||
# Kept out of `default` so normal/release builds don't compile them.
|
||||
dev-tools = []
|
||||
tokio-console = ["common-telemetry/tokio-console"]
|
||||
|
||||
@@ -117,7 +117,6 @@ async fn start(cli: Command) -> Result<()> {
|
||||
}
|
||||
datanode::SubCommand::Objbench(ref bench) => bench.run().await,
|
||||
datanode::SubCommand::Scanbench(ref bench) => bench.run().await,
|
||||
#[cfg(feature = "dev-tools")]
|
||||
datanode::SubCommand::Parquetbench(ref bench) => bench.run().await,
|
||||
},
|
||||
SubCommand::Flownode(cmd) => {
|
||||
|
||||
@@ -1,144 +0,0 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#![allow(clippy::print_stdout)]
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use api::prom_store::remote::{Label, Sample, TimeSeries, WriteRequest};
|
||||
use clap::Parser;
|
||||
use prost::Message;
|
||||
use serde_json::json;
|
||||
use servers::prom_store::snappy_compress;
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(about = "Generate a deterministic Prometheus remote-write fixture")]
|
||||
struct Args {
|
||||
#[arg(long, default_value = "http://127.0.0.1:4000/v1/prometheus/write")]
|
||||
endpoint: String,
|
||||
#[arg(long, default_value = "public")]
|
||||
database: String,
|
||||
#[arg(long)]
|
||||
metric: String,
|
||||
#[arg(long, default_value = "greptime_physical_table")]
|
||||
physical_table: String,
|
||||
#[arg(long, default_value_t = 8)]
|
||||
series_count: u64,
|
||||
#[arg(long, default_value_t = 30)]
|
||||
samples_per_series: u64,
|
||||
#[arg(long, default_value_t = 1_704_067_200_000)]
|
||||
start_unix_millis: i64,
|
||||
#[arg(long, default_value_t = 15_000)]
|
||||
step_millis: i64,
|
||||
#[arg(long, alias = "batch-size", default_value_t = 8)]
|
||||
chunk_series_count: u64,
|
||||
#[arg(long, default_value_t = 60)]
|
||||
timeout_seconds: u64,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let args = Args::parse();
|
||||
let started = Instant::now();
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(args.timeout_seconds))
|
||||
.build()?;
|
||||
let mut batches = 0_u64;
|
||||
let mut rows = 0_u64;
|
||||
let mut http_statuses = Vec::new();
|
||||
|
||||
let chunk = args.chunk_series_count.max(1);
|
||||
for first in (0..args.series_count).step_by(chunk as usize) {
|
||||
let last = (first + chunk).min(args.series_count);
|
||||
let timeseries = (first..last)
|
||||
.map(|series_idx| make_series(&args, series_idx))
|
||||
.collect::<Vec<_>>();
|
||||
rows += timeseries
|
||||
.iter()
|
||||
.map(|ts| ts.samples.len() as u64)
|
||||
.sum::<u64>();
|
||||
let body = snappy_compress(
|
||||
&WriteRequest {
|
||||
timeseries,
|
||||
..Default::default()
|
||||
}
|
||||
.encode_to_vec(),
|
||||
)?;
|
||||
let response = client
|
||||
.post(&args.endpoint)
|
||||
.header("Content-Encoding", "snappy")
|
||||
.body(body)
|
||||
.send()
|
||||
.await?;
|
||||
let status = response.status();
|
||||
http_statuses.push(status.as_u16());
|
||||
if !status.is_success() {
|
||||
let body_text = response.text().await.unwrap_or_default();
|
||||
return Err(format!("remote-write failed with status {status}: {body_text}").into());
|
||||
}
|
||||
batches += 1;
|
||||
}
|
||||
|
||||
println!(
|
||||
"{}",
|
||||
json!({
|
||||
"status": "ok",
|
||||
"endpoint": args.endpoint,
|
||||
"database": args.database,
|
||||
"metric": args.metric,
|
||||
"physical_table": args.physical_table,
|
||||
"series_count": args.series_count,
|
||||
"samples_per_series": args.samples_per_series,
|
||||
"rows": rows,
|
||||
"samples_written": rows,
|
||||
"batches": batches,
|
||||
"elapsed_seconds": started.elapsed().as_secs_f64(),
|
||||
"http_statuses": http_statuses,
|
||||
})
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn make_series(args: &Args, series_idx: u64) -> TimeSeries {
|
||||
let mut labels = vec![
|
||||
label("__name__", &args.metric),
|
||||
label("x_greptime_database", &args.database),
|
||||
label("x_greptime_physical_table", &args.physical_table),
|
||||
label("host", &format!("host{:04}", series_idx % 1024)),
|
||||
label("instance", &format!("instance{:06}", series_idx)),
|
||||
];
|
||||
labels.sort_unstable_by(|left, right| left.name.cmp(&right.name));
|
||||
let samples = (0..args.samples_per_series)
|
||||
.map(|sample_idx| Sample {
|
||||
value: deterministic_value(series_idx, sample_idx),
|
||||
timestamp: args.start_unix_millis + (sample_idx as i64 * args.step_millis),
|
||||
})
|
||||
.collect();
|
||||
TimeSeries {
|
||||
labels,
|
||||
samples,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn label(name: &str, value: &str) -> Label {
|
||||
Label {
|
||||
name: name.to_string(),
|
||||
value: value.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn deterministic_value(series_idx: u64, sample_idx: u64) -> f64 {
|
||||
((series_idx % 97) as f64) + (sample_idx as f64 * 0.125)
|
||||
}
|
||||
@@ -14,656 +14,10 @@
|
||||
|
||||
#![allow(clippy::print_stderr, clippy::print_stdout)]
|
||||
|
||||
//! Generic direct-readable-SST fixture generator for query performance labs.
|
||||
//!
|
||||
//! This is a bounded MVP: it reads one table/one region case TOML, builds
|
||||
//! synthetic region metadata, writes real Mito readable SST parquet files, and
|
||||
//! emits a manifest checkpoint plus validation metadata. The first case uses
|
||||
//! `sst_format = "flat"`; primary-key output is also accepted for early lab use
|
||||
//! because the writer can convert the same flat input batch to primary-key SSTs.
|
||||
//!
|
||||
//! TODO(query-perf): add multi-region generation and seed-manifest/catalog DB
|
||||
//! integration so the generated fixture can be mounted directly by a full
|
||||
//! GreptimeDB open/query path without manual metadata wiring.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs;
|
||||
use std::io::{BufWriter, Write};
|
||||
use std::num::{NonZeroU64, NonZeroUsize};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use api::v1::{OpType, SemanticType};
|
||||
use async_trait::async_trait;
|
||||
use clap::Parser;
|
||||
use datatypes::arrow::array::{
|
||||
ArrayRef, BinaryDictionaryBuilder, Float64Array, RecordBatch, StringDictionaryBuilder,
|
||||
TimestampMillisecondArray, TimestampNanosecondArray, UInt8Array, UInt64Array,
|
||||
};
|
||||
use datatypes::arrow::datatypes::UInt32Type;
|
||||
use datatypes::prelude::ConcreteDataType;
|
||||
use datatypes::schema::{ColumnSchema, SkippingIndexOptions};
|
||||
use mito_codec::row_converter::{DensePrimaryKeyCodec, PrimaryKeyCodecExt, SortField};
|
||||
use mito2::access_layer::{FilePathProvider, Metrics, WriteType};
|
||||
use mito2::config::IndexConfig;
|
||||
use mito2::manifest::action::{RegionCheckpoint, RegionManifest, RemovedFilesRecord};
|
||||
use mito2::read::FlatSource;
|
||||
use mito2::sst::file::{FileMeta, RegionFileId};
|
||||
use mito2::sst::index::{Indexer, IndexerBuilder};
|
||||
use mito2::sst::parquet::writer::ParquetWriter;
|
||||
use mito2::sst::parquet::{SstInfo, WriteOptions};
|
||||
use mito2::sst::{
|
||||
DEFAULT_WRITE_BUFFER_SIZE, FlatSchemaOptions, FormatType, to_flat_sst_arrow_schema,
|
||||
};
|
||||
use object_store::ObjectStore;
|
||||
use object_store::services::Fs as FsBuilder;
|
||||
use serde::Deserialize;
|
||||
use store_api::codec::PrimaryKeyEncoding;
|
||||
use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataRef};
|
||||
use store_api::path_utils::region_name;
|
||||
use store_api::region_request::PathType;
|
||||
use store_api::storage::{FileId, RegionId};
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(name = "query_perf_fixture")]
|
||||
#[command(about = "Generate direct-readable SST fixtures for query performance cases")]
|
||||
struct Args {
|
||||
/// TOML case file, for example tests/perf/query_cases/.../case.toml.
|
||||
#[arg(long, value_name = "PATH")]
|
||||
case: PathBuf,
|
||||
|
||||
/// Output directory (creates object-store/, manifest/, files.jsonl, summary.json).
|
||||
#[arg(long, value_name = "DIR")]
|
||||
out_dir: PathBuf,
|
||||
|
||||
/// Region ID to synthesize. Defaults to table id 1024, region 0.
|
||||
#[arg(long, default_value = "4398046511104")]
|
||||
region_id: u64,
|
||||
|
||||
/// Table directory relative to object-store root. Defaults to data/{database}/{table}/.
|
||||
#[arg(long, value_name = "DIR")]
|
||||
table_dir: Option<String>,
|
||||
|
||||
/// Table name to generate when the case contains multiple [[tables]].
|
||||
#[arg(long, value_name = "NAME")]
|
||||
table: Option<String>,
|
||||
|
||||
/// Manifest/checkpoint version.
|
||||
#[arg(long, default_value = "1000000")]
|
||||
checkpoint_version: u64,
|
||||
|
||||
/// Safety flag when sst_count exceeds 1000.
|
||||
#[arg(long)]
|
||||
allow_large: bool,
|
||||
|
||||
/// Print plan only.
|
||||
#[arg(long)]
|
||||
dry_run: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct CaseFile {
|
||||
scenario: Scenario,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "kind")]
|
||||
enum Scenario {
|
||||
#[serde(rename = "direct_readable_sst")]
|
||||
DirectReadableSst(DirectReadableSstScenario),
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct DirectReadableSstScenario {
|
||||
#[serde(default)]
|
||||
seed: Option<u64>,
|
||||
tables: Vec<TableConfig>,
|
||||
layout: LayoutConfig,
|
||||
}
|
||||
|
||||
impl Scenario {
|
||||
fn kind(&self) -> &'static str {
|
||||
match self {
|
||||
Scenario::DirectReadableSst(_) => "direct_readable_sst",
|
||||
}
|
||||
}
|
||||
|
||||
fn direct_readable_sst(&self) -> &DirectReadableSstScenario {
|
||||
match self {
|
||||
Scenario::DirectReadableSst(scenario) => scenario,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct TableConfig {
|
||||
database: String,
|
||||
name: String,
|
||||
engine: String,
|
||||
#[serde(default)]
|
||||
append_mode: Option<bool>,
|
||||
#[serde(default)]
|
||||
sst_format: Option<String>,
|
||||
primary_key: Vec<String>,
|
||||
time_index: String,
|
||||
columns: Vec<ColumnConfig>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct ColumnConfig {
|
||||
name: String,
|
||||
#[serde(rename = "type")]
|
||||
ty: String,
|
||||
semantic: String,
|
||||
distribution: Option<Distribution>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(tag = "kind")]
|
||||
enum Distribution {
|
||||
#[serde(rename = "cardinality")]
|
||||
Cardinality {
|
||||
values: NonZeroUsize,
|
||||
prefix: String,
|
||||
},
|
||||
#[serde(rename = "deterministic_wave")]
|
||||
DeterministicWave { min: f64, max: f64 },
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct LayoutConfig {
|
||||
regions: usize,
|
||||
sst_count: usize,
|
||||
rows_per_sst: usize,
|
||||
row_group_size: usize,
|
||||
series_count: NonZeroUsize,
|
||||
start_unix_nanos: i64,
|
||||
step_nanos: i64,
|
||||
time_range_layout: String,
|
||||
series_layout: String,
|
||||
}
|
||||
|
||||
struct NoopIndexBuilder;
|
||||
|
||||
#[async_trait]
|
||||
impl IndexerBuilder for NoopIndexBuilder {
|
||||
async fn build(&self, _file_id: FileId, _index_version: u64) -> Indexer {
|
||||
Indexer::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct FixedPathProvider {
|
||||
table_dir: String,
|
||||
}
|
||||
|
||||
impl FilePathProvider for FixedPathProvider {
|
||||
fn build_index_file_path(&self, file_id: RegionFileId) -> String {
|
||||
mito2::sst::location::index_file_path_legacy(&self.table_dir, file_id, PathType::Bare)
|
||||
}
|
||||
fn build_index_file_path_with_version(
|
||||
&self,
|
||||
index_id: mito2::sst::file::RegionIndexId,
|
||||
) -> String {
|
||||
mito2::sst::location::index_file_path(&self.table_dir, index_id, PathType::Bare)
|
||||
}
|
||||
fn build_sst_file_path(&self, file_id: RegionFileId) -> String {
|
||||
mito2::sst::location::sst_file_path(&self.table_dir, file_id, PathType::Bare)
|
||||
}
|
||||
}
|
||||
|
||||
fn semantic_type(s: &str) -> SemanticType {
|
||||
match s.to_ascii_lowercase().as_str() {
|
||||
"tag" => SemanticType::Tag,
|
||||
"field" => SemanticType::Field,
|
||||
"timestamp" => SemanticType::Timestamp,
|
||||
other => panic!("unsupported semantic {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn concrete_type(ty: &str) -> ConcreteDataType {
|
||||
match ty.to_ascii_uppercase().as_str() {
|
||||
"STRING" => ConcreteDataType::string_datatype(),
|
||||
"DOUBLE" => ConcreteDataType::float64_datatype(),
|
||||
"UINT64" => ConcreteDataType::uint64_datatype(),
|
||||
"TIMESTAMP(9)" => ConcreteDataType::timestamp_nanosecond_datatype(),
|
||||
"TIMESTAMP(3)" => ConcreteDataType::timestamp_millisecond_datatype(),
|
||||
other => panic!("unsupported column type {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn build_region_metadata(table: &TableConfig, region_id: RegionId) -> RegionMetadata {
|
||||
let mut builder = store_api::metadata::RegionMetadataBuilder::new(region_id);
|
||||
for (idx, col) in table.columns.iter().enumerate() {
|
||||
let mut schema = ColumnSchema::new(
|
||||
&col.name,
|
||||
concrete_type(&col.ty),
|
||||
col.semantic != "timestamp",
|
||||
);
|
||||
if col.semantic == "tag" {
|
||||
schema = schema.with_inverted_index(true);
|
||||
}
|
||||
if col.semantic == "tag" || col.semantic == "field" {
|
||||
schema = schema
|
||||
.with_skipping_options(SkippingIndexOptions {
|
||||
granularity: 1,
|
||||
..Default::default()
|
||||
})
|
||||
.expect("valid skipping index options for query perf fixture");
|
||||
}
|
||||
if col.name == table.time_index {
|
||||
schema = schema.with_time_index(true);
|
||||
}
|
||||
builder.push_column_metadata(ColumnMetadata {
|
||||
column_schema: schema,
|
||||
semantic_type: semantic_type(&col.semantic),
|
||||
column_id: idx as u32,
|
||||
});
|
||||
}
|
||||
let pk_ids = table
|
||||
.primary_key
|
||||
.iter()
|
||||
.map(|name| {
|
||||
table
|
||||
.columns
|
||||
.iter()
|
||||
.position(|c| c.name == *name)
|
||||
.unwrap_or_else(|| panic!("primary key column {name} not found")) as u32
|
||||
})
|
||||
.collect();
|
||||
builder.primary_key(pk_ids);
|
||||
builder.primary_key_encoding(PrimaryKeyEncoding::Dense);
|
||||
builder
|
||||
.build()
|
||||
.expect("region metadata should be valid for fixture table config")
|
||||
}
|
||||
|
||||
fn encode_dense_primary_key(table: &TableConfig, values: &HashMap<String, String>) -> Vec<u8> {
|
||||
let fields = table
|
||||
.primary_key
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, _)| {
|
||||
(
|
||||
idx as u32,
|
||||
SortField::new(ConcreteDataType::string_datatype()),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let converter = DensePrimaryKeyCodec::with_fields(fields);
|
||||
converter
|
||||
.encode(
|
||||
table
|
||||
.primary_key
|
||||
.iter()
|
||||
.map(|name| datatypes::value::ValueRef::String(values[name].as_str())),
|
||||
)
|
||||
.expect("dense primary key encoding should match fixture primary key fields")
|
||||
}
|
||||
|
||||
fn tag_value(col: &ColumnConfig, series: usize) -> String {
|
||||
match col
|
||||
.distribution
|
||||
.as_ref()
|
||||
.expect("tag distribution is required")
|
||||
{
|
||||
Distribution::Cardinality { values, prefix } => {
|
||||
format!("{}{}", prefix, series % values.get())
|
||||
}
|
||||
_ => panic!("tag column {} requires cardinality distribution", col.name),
|
||||
}
|
||||
}
|
||||
|
||||
fn series_for_row(layout: &LayoutConfig, sst_idx: usize, base_row: usize, row: usize) -> usize {
|
||||
match layout.series_layout.as_str() {
|
||||
"round_robin" | "timestamp_major" => (base_row + row) % layout.series_count.get(),
|
||||
"per_sst" => sst_idx % layout.series_count.get(),
|
||||
other => panic!("unsupported series_layout {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn timestamp_for_row(layout: &LayoutConfig, base_row: usize, row: usize) -> i64 {
|
||||
let logical_row = base_row + row;
|
||||
let timestamp_step = if layout.series_layout == "timestamp_major" {
|
||||
logical_row / layout.series_count.get()
|
||||
} else {
|
||||
logical_row
|
||||
};
|
||||
|
||||
layout.start_unix_nanos + (timestamp_step as i64 * layout.step_nanos)
|
||||
}
|
||||
|
||||
fn wave_value(min: f64, max: f64, row: usize) -> f64 {
|
||||
let span = max - min;
|
||||
let phase = (row % 1024) as f64 / 1023.0;
|
||||
min + span * (0.5 - 0.5 * (std::f64::consts::TAU * phase).cos())
|
||||
}
|
||||
|
||||
fn generate_record_batch(
|
||||
table: &TableConfig,
|
||||
metadata: &RegionMetadataRef,
|
||||
layout: &LayoutConfig,
|
||||
sst_idx: usize,
|
||||
sequence: u64,
|
||||
) -> RecordBatch {
|
||||
let flat_schema = to_flat_sst_arrow_schema(metadata, &FlatSchemaOptions::default());
|
||||
let rows = layout.rows_per_sst;
|
||||
let mut columns = Vec::new();
|
||||
let base_row = sst_idx * rows;
|
||||
|
||||
for col in &table.columns {
|
||||
match (col.semantic.as_str(), col.ty.to_ascii_uppercase().as_str()) {
|
||||
("tag", "STRING") => {
|
||||
let mut b = StringDictionaryBuilder::<UInt32Type>::new();
|
||||
for row in 0..rows {
|
||||
let series = series_for_row(layout, sst_idx, base_row, row);
|
||||
let value = tag_value(col, series);
|
||||
b.append_value(value);
|
||||
}
|
||||
columns.push(Arc::new(b.finish()) as ArrayRef);
|
||||
}
|
||||
("field", "DOUBLE") => {
|
||||
let (min, max) = match col.distribution.as_ref() {
|
||||
Some(Distribution::DeterministicWave { min, max }) => (*min, *max),
|
||||
_ => (0.0, 1.0),
|
||||
};
|
||||
columns.push(Arc::new(Float64Array::from(
|
||||
(0..rows)
|
||||
.map(|r| wave_value(min, max, base_row + r))
|
||||
.collect::<Vec<_>>(),
|
||||
)) as ArrayRef);
|
||||
}
|
||||
("field", "UINT64") => columns.push(Arc::new(UInt64Array::from(
|
||||
(0..rows).map(|r| (base_row + r) as u64).collect::<Vec<_>>(),
|
||||
)) as ArrayRef),
|
||||
("timestamp", "TIMESTAMP(9)") => columns.push(Arc::new(TimestampNanosecondArray::from(
|
||||
(0..rows)
|
||||
.map(|r| timestamp_for_row(layout, base_row, r))
|
||||
.collect::<Vec<_>>(),
|
||||
)) as ArrayRef),
|
||||
("timestamp", "TIMESTAMP(3)") => {
|
||||
columns.push(Arc::new(TimestampMillisecondArray::from(
|
||||
(0..rows)
|
||||
.map(|r| timestamp_for_row(layout, base_row, r) / 1_000_000)
|
||||
.collect::<Vec<_>>(),
|
||||
)) as ArrayRef)
|
||||
}
|
||||
other => panic!("unsupported column combination {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
let mut pk_builder = BinaryDictionaryBuilder::<UInt32Type>::new();
|
||||
for row in 0..rows {
|
||||
let series = series_for_row(layout, sst_idx, base_row, row);
|
||||
let values = table
|
||||
.columns
|
||||
.iter()
|
||||
.filter(|c| c.semantic == "tag")
|
||||
.map(|c| (c.name.clone(), tag_value(c, series)))
|
||||
.collect();
|
||||
pk_builder
|
||||
.append(encode_dense_primary_key(table, &values))
|
||||
.expect("append generated dense primary key to Arrow dictionary builder");
|
||||
}
|
||||
columns.push(Arc::new(pk_builder.finish()));
|
||||
columns.push(Arc::new(UInt64Array::from_value(sequence, rows)));
|
||||
columns.push(Arc::new(UInt8Array::from_value(OpType::Put as u8, rows)));
|
||||
RecordBatch::try_new(flat_schema, columns)
|
||||
.expect("generated fixture columns should match flat SST Arrow schema")
|
||||
}
|
||||
|
||||
fn file_meta_from_sst_info(
|
||||
info: &SstInfo,
|
||||
region_id: RegionId,
|
||||
file_id: FileId,
|
||||
sequence: u64,
|
||||
) -> FileMeta {
|
||||
FileMeta {
|
||||
region_id,
|
||||
file_id,
|
||||
time_range: info.time_range,
|
||||
level: 0,
|
||||
file_size: info.file_size,
|
||||
max_row_group_uncompressed_size: info.max_row_group_uncompressed_size,
|
||||
available_indexes: Default::default(),
|
||||
indexes: Default::default(),
|
||||
index_file_size: 0,
|
||||
index_version: 0,
|
||||
num_rows: info.num_rows as u64,
|
||||
num_row_groups: info.num_row_groups,
|
||||
sequence: NonZeroU64::new(sequence),
|
||||
partition_expr: None,
|
||||
num_series: info.num_series,
|
||||
primary_key_min: None,
|
||||
primary_key_max: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn deterministic_file_id(seed: u64, index: usize) -> FileId {
|
||||
let seed_hi = ((seed >> 16) & 0xffff) as u16;
|
||||
let seed_lo = (seed & 0xffff) as u16;
|
||||
FileId::parse_str(&format!(
|
||||
"00000000-0000-{seed_hi:04x}-{seed_lo:04x}-{index:012x}"
|
||||
))
|
||||
.expect("deterministic file id must be a valid UUID")
|
||||
}
|
||||
|
||||
fn remap_sst_file(
|
||||
object_store_dir: &Path,
|
||||
table_dir: &str,
|
||||
region_id: RegionId,
|
||||
old_file_id: FileId,
|
||||
new_file_id: FileId,
|
||||
) {
|
||||
if old_file_id == new_file_id {
|
||||
return;
|
||||
}
|
||||
let old_path = object_store_dir.join(mito2::sst::location::sst_file_path(
|
||||
table_dir,
|
||||
RegionFileId::new(region_id, old_file_id),
|
||||
PathType::Bare,
|
||||
));
|
||||
let new_path = object_store_dir.join(mito2::sst::location::sst_file_path(
|
||||
table_dir,
|
||||
RegionFileId::new(region_id, new_file_id),
|
||||
PathType::Bare,
|
||||
));
|
||||
if let Some(parent) = new_path.parent() {
|
||||
fs::create_dir_all(parent).expect("failed to create remapped SST parent directory");
|
||||
}
|
||||
fs::rename(&old_path, &new_path).unwrap_or_else(|err| {
|
||||
panic!(
|
||||
"failed to remap SST file {} -> {}: {err}",
|
||||
old_path.display(),
|
||||
new_path.display()
|
||||
)
|
||||
});
|
||||
}
|
||||
|
||||
fn case_name_from_path(path: &Path) -> String {
|
||||
path.parent()
|
||||
.and_then(Path::file_name)
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("query_perf_case")
|
||||
.to_string()
|
||||
}
|
||||
#[path = "query_perf_fixture/mod.rs"]
|
||||
mod query_perf_fixture;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let args = Args::parse();
|
||||
let case_text = fs::read_to_string(&args.case).expect("failed to read query perf case TOML");
|
||||
let case: CaseFile = toml::from_str(&case_text).expect("failed to parse query perf case TOML");
|
||||
let case_name = case_name_from_path(&args.case);
|
||||
let scenario = case.scenario.direct_readable_sst();
|
||||
if scenario.tables.is_empty() || scenario.layout.regions != 1 {
|
||||
panic!("fixture generator supports one or more tables and exactly one region per table");
|
||||
}
|
||||
if scenario.layout.time_range_layout != "non_overlapping_per_sst" {
|
||||
panic!("MVP supports time_range_layout=non_overlapping_per_sst");
|
||||
}
|
||||
if scenario.layout.series_layout == "timestamp_major"
|
||||
&& !scenario
|
||||
.layout
|
||||
.rows_per_sst
|
||||
.is_multiple_of(scenario.layout.series_count.get())
|
||||
{
|
||||
panic!(
|
||||
"series_layout=timestamp_major requires rows_per_sst to be divisible by series_count"
|
||||
);
|
||||
}
|
||||
if scenario.layout.sst_count > 1000 && !args.allow_large {
|
||||
panic!("sst_count exceeds 1000; pass --allow-large");
|
||||
}
|
||||
let seed = scenario.seed.unwrap_or(0);
|
||||
let table_index = match args.table.as_deref() {
|
||||
Some(name) => scenario
|
||||
.tables
|
||||
.iter()
|
||||
.position(|table| table.name == name)
|
||||
.unwrap_or_else(|| panic!("--table {name} was not found in case {case_name}")),
|
||||
None if scenario.tables.len() == 1 => 0,
|
||||
None => panic!(
|
||||
"case {} contains {} tables; pass --table <name> to choose one",
|
||||
case_name,
|
||||
scenario.tables.len()
|
||||
),
|
||||
};
|
||||
let table = &scenario.tables[table_index];
|
||||
if table.engine != "mito" {
|
||||
panic!("MVP supports engine=mito");
|
||||
}
|
||||
let region_id = RegionId::from(args.region_id);
|
||||
let table_dir = args
|
||||
.table_dir
|
||||
.unwrap_or_else(|| format!("data/{}/{}/", table.database, table.name));
|
||||
if Path::new(&table_dir).is_absolute() {
|
||||
panic!("--table-dir must be relative");
|
||||
}
|
||||
if table_dir.trim_end_matches('/').rsplit('/').next()
|
||||
== Some(region_name(region_id.table_id(), region_id.region_sequence()).as_str())
|
||||
{
|
||||
panic!("--table-dir must be table-level directory");
|
||||
}
|
||||
let region_dir =
|
||||
mito2::sst::location::region_dir_from_table_dir(&table_dir, region_id, PathType::Bare);
|
||||
let format = match table.sst_format.as_deref().unwrap_or("flat") {
|
||||
"flat" => FormatType::Flat,
|
||||
"primary_key" | "primary-key" => FormatType::PrimaryKey,
|
||||
other => panic!("unsupported sst_format {other}"),
|
||||
};
|
||||
println!(
|
||||
"query_perf_fixture case={} scenario={} table={}.{} ssts={} rows_per_sst={} format={format:?}",
|
||||
case_name,
|
||||
case.scenario.kind(),
|
||||
table.database,
|
||||
table.name,
|
||||
scenario.layout.sst_count,
|
||||
scenario.layout.rows_per_sst
|
||||
);
|
||||
println!(
|
||||
"out_dir={} table_dir={} region_dir={}",
|
||||
args.out_dir.display(),
|
||||
table_dir,
|
||||
region_dir
|
||||
);
|
||||
if args.dry_run {
|
||||
return;
|
||||
}
|
||||
|
||||
let obj_store_dir = args.out_dir.join("object-store");
|
||||
let manifest_dir = args.out_dir.join("manifest");
|
||||
fs::create_dir_all(&obj_store_dir).expect("failed to create fixture object-store directory");
|
||||
fs::create_dir_all(&manifest_dir).expect("failed to create fixture manifest directory");
|
||||
let ostorage = ObjectStore::new(FsBuilder::default().root(&obj_store_dir.to_string_lossy()))
|
||||
.expect("failed to create filesystem object store for fixture output")
|
||||
.finish();
|
||||
let metadata: RegionMetadataRef = Arc::new(build_region_metadata(table, region_id));
|
||||
let mut files = HashMap::with_capacity(scenario.layout.sst_count);
|
||||
let mut next_file_index = 1;
|
||||
for i in 0..scenario.layout.sst_count {
|
||||
let sequence = 1000 + i as u64;
|
||||
let batch = generate_record_batch(table, &metadata, &scenario.layout, i, sequence);
|
||||
let source =
|
||||
FlatSource::new_iter(batch.schema(), Box::new(vec![batch].into_iter().map(Ok)));
|
||||
let mut metrics = Metrics::new(WriteType::Flush);
|
||||
let mut writer = ParquetWriter::new_with_object_store(
|
||||
ostorage.clone(),
|
||||
metadata.clone(),
|
||||
IndexConfig::default(),
|
||||
NoopIndexBuilder,
|
||||
FixedPathProvider {
|
||||
table_dir: table_dir.clone(),
|
||||
},
|
||||
&mut metrics,
|
||||
)
|
||||
.await;
|
||||
let opts = WriteOptions {
|
||||
write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
|
||||
row_group_size: scenario.layout.row_group_size,
|
||||
max_file_size: None,
|
||||
};
|
||||
let infos = match format {
|
||||
FormatType::Flat => writer.write_all_flat(source, Some(sequence), &opts).await,
|
||||
FormatType::PrimaryKey => {
|
||||
writer
|
||||
.write_all_flat_as_primary_key(source, Some(sequence), &opts)
|
||||
.await
|
||||
}
|
||||
}
|
||||
.expect("failed to write fixture SST through Mito parquet writer");
|
||||
for info in infos {
|
||||
let file_id = deterministic_file_id(
|
||||
seed.wrapping_add((table_index as u64) << 16),
|
||||
next_file_index,
|
||||
);
|
||||
next_file_index += 1;
|
||||
remap_sst_file(&obj_store_dir, &table_dir, region_id, info.file_id, file_id);
|
||||
files.insert(
|
||||
file_id,
|
||||
file_meta_from_sst_info(&info, region_id, file_id, sequence),
|
||||
);
|
||||
}
|
||||
}
|
||||
let overall_seq = files
|
||||
.values()
|
||||
.filter_map(|m| m.sequence)
|
||||
.map(|s| s.get())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
let manifest = RegionManifest {
|
||||
metadata,
|
||||
files,
|
||||
removed_files: RemovedFilesRecord::default(),
|
||||
flushed_entry_id: overall_seq,
|
||||
flushed_sequence: overall_seq,
|
||||
committed_sequence: Some(overall_seq),
|
||||
manifest_version: args.checkpoint_version,
|
||||
truncated_entry_id: None,
|
||||
compaction_time_window: None,
|
||||
sst_format: format,
|
||||
append_mode: table.append_mode,
|
||||
};
|
||||
let checkpoint = RegionCheckpoint {
|
||||
last_version: args.checkpoint_version,
|
||||
compacted_actions: manifest.files.len(),
|
||||
checkpoint: Some(manifest.clone()),
|
||||
};
|
||||
let checkpoint_bytes = checkpoint
|
||||
.encode()
|
||||
.expect("failed to encode fixture region checkpoint");
|
||||
let checkpoint_path = manifest_dir.join(format!("{:020}.checkpoint", args.checkpoint_version));
|
||||
fs::write(&checkpoint_path, &checkpoint_bytes).expect("failed to write checkpoint file");
|
||||
fs::write(manifest_dir.join("_last_checkpoint"), serde_json::to_vec_pretty(&serde_json::json!({ "size": checkpoint_bytes.len(), "version": args.checkpoint_version, "checksum": null, "extend_metadata": {} })).expect("failed to serialize _last_checkpoint metadata")).expect("failed to write _last_checkpoint file");
|
||||
let files_jsonl_path = args.out_dir.join("files.jsonl");
|
||||
let mut jsonl = BufWriter::new(
|
||||
fs::File::create(&files_jsonl_path).expect("failed to create fixture files.jsonl"),
|
||||
);
|
||||
let mut entries: Vec<_> = manifest.files.iter().collect();
|
||||
entries.sort_by_key(|(id, _)| id.to_string());
|
||||
for (file_id, meta) in entries {
|
||||
writeln!(jsonl, "{}", serde_json::to_string(&serde_json::json!({ "file_id": file_id.to_string(), "region_id": meta.region_id.as_u64(), "object_path": mito2::sst::location::sst_file_path(&table_dir, RegionFileId::new(meta.region_id, *file_id), PathType::Bare), "time_range_start": meta.time_range.0.value(), "time_range_end": meta.time_range.1.value(), "num_rows": meta.num_rows, "num_row_groups": meta.num_row_groups, "file_size": meta.file_size, "num_series": meta.num_series, "sequence": meta.sequence.map(|s| s.get()) })).expect("failed to serialize fixture SST metadata JSONL entry")).expect("failed to write fixture SST metadata JSONL entry");
|
||||
}
|
||||
jsonl.flush().expect("failed to flush fixture files.jsonl");
|
||||
fs::write(args.out_dir.join("summary.json"), serde_json::to_vec_pretty(&serde_json::json!({ "case": case_name, "seed": seed, "table_index": table_index, "table": table.name, "database": table.database, "region_id": region_id.as_u64(), "table_dir": table_dir, "region_dir": region_dir, "sst_format": format!("{format:?}"), "sst_count": scenario.layout.sst_count, "rows_per_sst": scenario.layout.rows_per_sst, "row_group_size": scenario.layout.row_group_size, "total_rows": scenario.layout.sst_count * scenario.layout.rows_per_sst, "checkpoint_path": checkpoint_path, "files_jsonl_path": files_jsonl_path, "readback_validated": false, "metadata_source": "synthetic" })).expect("failed to serialize fixture summary")).expect("failed to write fixture summary.json");
|
||||
println!("Done. wrote {} SST file entries", manifest.files.len());
|
||||
query_perf_fixture::run().await;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,470 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::num::NonZeroUsize;
|
||||
|
||||
use clap::ValueEnum;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub(super) struct CaseFile {
|
||||
pub(super) scenario: Scenario,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
#[serde(tag = "kind")]
|
||||
pub(super) enum Scenario {
|
||||
#[serde(rename = "direct_readable_sst")]
|
||||
DirectReadableSst(DirectReadableSstScenario),
|
||||
#[serde(rename = "prom_remote_write_then_query")]
|
||||
PromRemoteWriteThenQuery(PromRemoteWriteThenQueryScenario),
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub(super) struct DirectReadableSstScenario {
|
||||
#[serde(default)]
|
||||
pub(super) seed: Option<u64>,
|
||||
pub(super) tables: Vec<TableConfig>,
|
||||
pub(super) layout: LayoutConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub(super) struct PromRemoteWriteThenQueryScenario {
|
||||
#[serde(default)]
|
||||
pub(super) queries: Vec<serde_json::Value>,
|
||||
pub(super) remote_write: PromRemoteWritePlan,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub(super) struct PromRemoteWritePlan {
|
||||
#[serde(default = "default_database")]
|
||||
pub(super) database: String,
|
||||
#[serde(alias = "metric_name")]
|
||||
pub(super) metric: String,
|
||||
#[serde(default = "default_physical_table")]
|
||||
pub(super) physical_table: String,
|
||||
#[serde(default = "default_series_count")]
|
||||
pub(super) series_count: u64,
|
||||
#[serde(default = "default_samples_per_series")]
|
||||
pub(super) samples_per_series: u64,
|
||||
#[serde(default = "default_start_unix_millis")]
|
||||
pub(super) start_unix_millis: i64,
|
||||
#[serde(default = "default_step_millis")]
|
||||
pub(super) step_millis: i64,
|
||||
#[serde(default = "default_chunk_series_count", alias = "batch_size")]
|
||||
pub(super) chunk_series_count: u64,
|
||||
#[serde(default = "default_timeout_seconds")]
|
||||
pub(super) timeout_seconds: u64,
|
||||
#[serde(default)]
|
||||
pub(super) sample_chunk_size: Option<u64>,
|
||||
#[serde(default = "default_flush_every_sample_chunks")]
|
||||
pub(super) flush_every_sample_chunks: u64,
|
||||
#[serde(default = "default_visibility_timeout_seconds")]
|
||||
pub(super) visibility_timeout_seconds: u64,
|
||||
#[serde(default)]
|
||||
pub(super) prom_store: PromStoreConfig,
|
||||
#[serde(default)]
|
||||
pub(super) value: ValueConfig,
|
||||
#[serde(default)]
|
||||
pub(super) storage: Option<StorageConfig>,
|
||||
#[serde(default)]
|
||||
pub(super) read_bench: Option<ReadBenchConfig>,
|
||||
}
|
||||
|
||||
pub(super) fn default_database() -> String {
|
||||
"public".to_string()
|
||||
}
|
||||
pub(super) fn default_physical_table() -> String {
|
||||
"greptime_physical_table".to_string()
|
||||
}
|
||||
pub(super) fn default_series_count() -> u64 {
|
||||
8
|
||||
}
|
||||
pub(super) fn default_samples_per_series() -> u64 {
|
||||
30
|
||||
}
|
||||
pub(super) fn default_start_unix_millis() -> i64 {
|
||||
1_704_067_200_000
|
||||
}
|
||||
pub(super) fn default_step_millis() -> i64 {
|
||||
15_000
|
||||
}
|
||||
pub(super) fn default_chunk_series_count() -> u64 {
|
||||
8
|
||||
}
|
||||
pub(super) fn default_timeout_seconds() -> u64 {
|
||||
60
|
||||
}
|
||||
pub(super) fn default_flush_every_sample_chunks() -> u64 {
|
||||
1
|
||||
}
|
||||
pub(super) fn default_visibility_timeout_seconds() -> u64 {
|
||||
30
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub(super) struct PromStoreConfig {
|
||||
#[serde(default = "default_pending_rows_flush_interval")]
|
||||
pub(super) pending_rows_flush_interval: String,
|
||||
#[serde(default = "default_max_batch_rows")]
|
||||
pub(super) max_batch_rows: u64,
|
||||
#[serde(default = "default_max_concurrent_flushes")]
|
||||
pub(super) max_concurrent_flushes: u64,
|
||||
#[serde(default = "default_worker_channel_capacity")]
|
||||
pub(super) worker_channel_capacity: u64,
|
||||
#[serde(default = "default_max_inflight_requests")]
|
||||
pub(super) max_inflight_requests: u64,
|
||||
}
|
||||
|
||||
impl Default for PromStoreConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
pending_rows_flush_interval: default_pending_rows_flush_interval(),
|
||||
max_batch_rows: default_max_batch_rows(),
|
||||
max_concurrent_flushes: default_max_concurrent_flushes(),
|
||||
worker_channel_capacity: default_worker_channel_capacity(),
|
||||
max_inflight_requests: default_max_inflight_requests(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn default_pending_rows_flush_interval() -> String {
|
||||
"1s".to_string()
|
||||
}
|
||||
pub(super) fn default_max_batch_rows() -> u64 {
|
||||
100000
|
||||
}
|
||||
pub(super) fn default_max_concurrent_flushes() -> u64 {
|
||||
256
|
||||
}
|
||||
pub(super) fn default_worker_channel_capacity() -> u64 {
|
||||
65526
|
||||
}
|
||||
pub(super) fn default_max_inflight_requests() -> u64 {
|
||||
3000
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Deserialize, Serialize, ValueEnum, Eq, PartialEq)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
#[value(rename_all = "snake_case")]
|
||||
pub(super) enum ValuePattern {
|
||||
Linear,
|
||||
Constant,
|
||||
Modulo,
|
||||
Unique,
|
||||
SeededRandom,
|
||||
RunLength,
|
||||
QuantizedSignal,
|
||||
SignalWithSporadicStalls,
|
||||
MixedSignalRepeated,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ValuePattern {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"{}",
|
||||
serde_json::to_value(self).unwrap().as_str().unwrap()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub(super) struct ValueConfig {
|
||||
#[serde(default = "default_value_pattern")]
|
||||
pub(super) pattern: ValuePattern,
|
||||
#[serde(default)]
|
||||
pub(super) base: f64,
|
||||
#[serde(default = "default_value_step")]
|
||||
pub(super) step: f64,
|
||||
#[serde(default = "default_value_cardinality")]
|
||||
pub(super) cardinality: u64,
|
||||
#[serde(default)]
|
||||
pub(super) seed: u64,
|
||||
#[serde(default = "default_value_run_length")]
|
||||
pub(super) run_length: u64,
|
||||
#[serde(default = "default_value_stall_every")]
|
||||
pub(super) stall_every: u64,
|
||||
#[serde(default = "default_value_stall_length")]
|
||||
pub(super) stall_length: u64,
|
||||
#[serde(default = "default_value_mixed_every")]
|
||||
pub(super) mixed_every: u64,
|
||||
}
|
||||
impl Default for ValueConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
pattern: default_value_pattern(),
|
||||
base: 0.0,
|
||||
step: default_value_step(),
|
||||
cardinality: default_value_cardinality(),
|
||||
seed: 0,
|
||||
run_length: default_value_run_length(),
|
||||
stall_every: default_value_stall_every(),
|
||||
stall_length: default_value_stall_length(),
|
||||
mixed_every: default_value_mixed_every(),
|
||||
}
|
||||
}
|
||||
}
|
||||
pub(super) fn default_value_pattern() -> ValuePattern {
|
||||
ValuePattern::Linear
|
||||
}
|
||||
pub(super) fn default_value_step() -> f64 {
|
||||
0.125
|
||||
}
|
||||
pub(super) fn default_value_cardinality() -> u64 {
|
||||
97
|
||||
}
|
||||
pub(super) fn default_value_run_length() -> u64 {
|
||||
8
|
||||
}
|
||||
pub(super) fn default_value_stall_every() -> u64 {
|
||||
100
|
||||
}
|
||||
pub(super) fn default_value_stall_length() -> u64 {
|
||||
16
|
||||
}
|
||||
pub(super) fn default_value_mixed_every() -> u64 {
|
||||
5
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub(super) struct StorageConfig {
|
||||
#[serde(default = "default_true")]
|
||||
pub(super) inspect: bool,
|
||||
#[serde(default = "default_storage_column")]
|
||||
pub(super) column: String,
|
||||
#[serde(default)]
|
||||
pub(super) root_suffix: Option<String>,
|
||||
#[serde(default)]
|
||||
pub(super) include_metadata_files: bool,
|
||||
#[serde(default = "default_min_files")]
|
||||
pub(super) min_files: u64,
|
||||
#[serde(default = "default_min_files")]
|
||||
pub(super) min_files_with_column: u64,
|
||||
#[serde(default)]
|
||||
pub(super) require_encodings: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub(super) forbid_encodings: Vec<String>,
|
||||
#[serde(default)]
|
||||
pub(super) max_total_file_size_bytes: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub(super) max_column_compressed_size_bytes: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub(super) max_column_uncompressed_size_bytes: Option<u64>,
|
||||
#[serde(default)]
|
||||
pub(super) max_candidate_total_file_size_regression_pct: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub(super) max_candidate_column_compressed_size_regression_pct: Option<f64>,
|
||||
#[serde(default)]
|
||||
pub(super) max_candidate_column_uncompressed_size_regression_pct: Option<f64>,
|
||||
#[serde(skip_deserializing, default)]
|
||||
pub(super) planned_thresholds: Vec<StorageThresholdPlan>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(super) struct StorageThresholdPlan {
|
||||
pub(super) threshold: String,
|
||||
pub(super) status: String,
|
||||
pub(super) value: serde_json::Value,
|
||||
}
|
||||
|
||||
impl StorageConfig {
|
||||
pub(super) fn populate_planned_thresholds(&mut self) {
|
||||
let mut planned = Vec::new();
|
||||
planned.push(StorageThresholdPlan::new("min_files", self.min_files));
|
||||
planned.push(StorageThresholdPlan::new(
|
||||
"min_files_with_column",
|
||||
self.min_files_with_column,
|
||||
));
|
||||
if !self.require_encodings.is_empty() {
|
||||
planned.push(StorageThresholdPlan::new(
|
||||
"require_encodings",
|
||||
&self.require_encodings,
|
||||
));
|
||||
}
|
||||
if !self.forbid_encodings.is_empty() {
|
||||
planned.push(StorageThresholdPlan::new(
|
||||
"forbid_encodings",
|
||||
&self.forbid_encodings,
|
||||
));
|
||||
}
|
||||
macro_rules! push_optional {
|
||||
($name:literal, $value:expr) => {
|
||||
if let Some(value) = $value {
|
||||
planned.push(StorageThresholdPlan::new($name, value));
|
||||
}
|
||||
};
|
||||
}
|
||||
push_optional!("max_total_file_size_bytes", self.max_total_file_size_bytes);
|
||||
push_optional!(
|
||||
"max_column_compressed_size_bytes",
|
||||
self.max_column_compressed_size_bytes
|
||||
);
|
||||
push_optional!(
|
||||
"max_column_uncompressed_size_bytes",
|
||||
self.max_column_uncompressed_size_bytes
|
||||
);
|
||||
push_optional!(
|
||||
"max_candidate_total_file_size_regression_pct",
|
||||
self.max_candidate_total_file_size_regression_pct
|
||||
);
|
||||
push_optional!(
|
||||
"max_candidate_column_compressed_size_regression_pct",
|
||||
self.max_candidate_column_compressed_size_regression_pct
|
||||
);
|
||||
push_optional!(
|
||||
"max_candidate_column_uncompressed_size_regression_pct",
|
||||
self.max_candidate_column_uncompressed_size_regression_pct
|
||||
);
|
||||
self.planned_thresholds = planned;
|
||||
}
|
||||
}
|
||||
|
||||
impl StorageThresholdPlan {
|
||||
fn new<T: Serialize>(threshold: &str, value: T) -> Self {
|
||||
Self {
|
||||
threshold: threshold.to_string(),
|
||||
status: "planned".to_string(),
|
||||
value: serde_json::to_value(value).expect("storage threshold value must serialize"),
|
||||
}
|
||||
}
|
||||
}
|
||||
pub(super) fn default_true() -> bool {
|
||||
true
|
||||
}
|
||||
pub(super) fn default_storage_column() -> String {
|
||||
"greptime_value".to_string()
|
||||
}
|
||||
pub(super) fn default_min_files() -> u64 {
|
||||
1
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub(super) struct ReadBenchConfig {
|
||||
#[serde(default = "default_true")]
|
||||
pub(super) enabled: bool,
|
||||
#[serde(default = "default_true")]
|
||||
pub(super) parquetbench: bool,
|
||||
#[serde(default = "default_true")]
|
||||
pub(super) scanbench: bool,
|
||||
#[serde(default = "default_iterations")]
|
||||
pub(super) iterations: u64,
|
||||
#[serde(default)]
|
||||
pub(super) projection: Vec<String>,
|
||||
#[serde(default = "default_parquet_reader")]
|
||||
pub(super) parquet_reader: String,
|
||||
#[serde(default = "default_scan_scanner")]
|
||||
pub(super) scan_scanner: String,
|
||||
#[serde(default = "default_parallelism")]
|
||||
pub(super) parallelism: u64,
|
||||
#[serde(default)]
|
||||
pub(super) max_files: Option<usize>,
|
||||
#[serde(flatten)]
|
||||
pub(super) thresholds: HashMap<String, serde_json::Value>,
|
||||
}
|
||||
|
||||
impl Default for ReadBenchConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled: true,
|
||||
parquetbench: true,
|
||||
scanbench: true,
|
||||
iterations: default_iterations(),
|
||||
projection: vec![],
|
||||
parquet_reader: default_parquet_reader(),
|
||||
scan_scanner: default_scan_scanner(),
|
||||
parallelism: default_parallelism(),
|
||||
max_files: None,
|
||||
thresholds: HashMap::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn default_iterations() -> u64 {
|
||||
7
|
||||
}
|
||||
pub(super) fn default_parquet_reader() -> String {
|
||||
"direct".to_string()
|
||||
}
|
||||
pub(super) fn default_scan_scanner() -> String {
|
||||
"seq".to_string()
|
||||
}
|
||||
pub(super) fn default_parallelism() -> u64 {
|
||||
1
|
||||
}
|
||||
|
||||
impl Scenario {
|
||||
pub(super) fn kind(&self) -> &'static str {
|
||||
match self {
|
||||
Scenario::DirectReadableSst(_) => "direct_readable_sst",
|
||||
Scenario::PromRemoteWriteThenQuery(_) => "prom_remote_write_then_query",
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn direct_readable_sst(&self) -> &DirectReadableSstScenario {
|
||||
match self {
|
||||
Scenario::DirectReadableSst(scenario) => scenario,
|
||||
_ => panic!("scenario is not direct_readable_sst"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub(super) struct TableConfig {
|
||||
pub(super) database: String,
|
||||
pub(super) name: String,
|
||||
pub(super) engine: String,
|
||||
#[serde(default)]
|
||||
pub(super) append_mode: Option<bool>,
|
||||
#[serde(default)]
|
||||
pub(super) sst_format: Option<String>,
|
||||
pub(super) primary_key: Vec<String>,
|
||||
pub(super) time_index: String,
|
||||
pub(super) columns: Vec<ColumnConfig>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub(super) struct ColumnConfig {
|
||||
pub(super) name: String,
|
||||
#[serde(rename = "type")]
|
||||
pub(super) ty: String,
|
||||
pub(super) semantic: String,
|
||||
pub(super) distribution: Option<Distribution>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
#[serde(tag = "kind")]
|
||||
pub(super) enum Distribution {
|
||||
#[serde(rename = "cardinality")]
|
||||
Cardinality {
|
||||
values: NonZeroUsize,
|
||||
prefix: String,
|
||||
},
|
||||
#[serde(rename = "deterministic_wave")]
|
||||
DeterministicWave { min: f64, max: f64 },
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
pub(super) struct LayoutConfig {
|
||||
pub(super) regions: usize,
|
||||
pub(super) sst_count: usize,
|
||||
pub(super) rows_per_sst: usize,
|
||||
pub(super) row_group_size: usize,
|
||||
pub(super) series_count: NonZeroUsize,
|
||||
pub(super) start_unix_nanos: i64,
|
||||
pub(super) step_nanos: i64,
|
||||
pub(super) time_range_layout: String,
|
||||
pub(super) series_layout: String,
|
||||
}
|
||||
@@ -0,0 +1,529 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs::{self};
|
||||
use std::io::{BufWriter, Write};
|
||||
use std::num::NonZeroU64;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use api::v1::{OpType, SemanticType};
|
||||
use async_trait::async_trait;
|
||||
use datatypes::arrow::array::{
|
||||
ArrayRef, BinaryDictionaryBuilder, Float64Array, RecordBatch, StringDictionaryBuilder,
|
||||
TimestampMillisecondArray, TimestampNanosecondArray, UInt8Array, UInt64Array,
|
||||
};
|
||||
use datatypes::arrow::datatypes::UInt32Type;
|
||||
use datatypes::prelude::ConcreteDataType;
|
||||
use datatypes::schema::{ColumnSchema, SkippingIndexOptions};
|
||||
use mito_codec::row_converter::{DensePrimaryKeyCodec, PrimaryKeyCodecExt, SortField};
|
||||
use mito2::access_layer::{FilePathProvider, Metrics, WriteType};
|
||||
use mito2::config::IndexConfig;
|
||||
use mito2::manifest::action::{RegionCheckpoint, RegionManifest, RemovedFilesRecord};
|
||||
use mito2::read::FlatSource;
|
||||
use mito2::sst::file::{FileMeta, RegionFileId};
|
||||
use mito2::sst::index::{Indexer, IndexerBuilder};
|
||||
use mito2::sst::parquet::writer::ParquetWriter;
|
||||
use mito2::sst::parquet::{SstInfo, WriteOptions};
|
||||
use mito2::sst::{
|
||||
DEFAULT_WRITE_BUFFER_SIZE, FlatSchemaOptions, FormatType, to_flat_sst_arrow_schema,
|
||||
};
|
||||
use object_store::ObjectStore;
|
||||
use object_store::services::Fs as FsBuilder;
|
||||
use store_api::codec::PrimaryKeyEncoding;
|
||||
use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataRef};
|
||||
use store_api::path_utils::region_name;
|
||||
use store_api::region_request::PathType;
|
||||
use store_api::storage::{FileId, RegionId};
|
||||
|
||||
use crate::query_perf_fixture::DirectArgs;
|
||||
use crate::query_perf_fixture::case::*;
|
||||
use crate::query_perf_fixture::util::case_name_from_path;
|
||||
|
||||
struct NoopIndexBuilder;
|
||||
|
||||
#[async_trait]
|
||||
impl IndexerBuilder for NoopIndexBuilder {
|
||||
async fn build(&self, _file_id: FileId, _index_version: u64) -> Indexer {
|
||||
Indexer::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct FixedPathProvider {
|
||||
table_dir: String,
|
||||
}
|
||||
|
||||
impl FilePathProvider for FixedPathProvider {
|
||||
fn build_index_file_path(&self, file_id: RegionFileId) -> String {
|
||||
mito2::sst::location::index_file_path_legacy(&self.table_dir, file_id, PathType::Bare)
|
||||
}
|
||||
fn build_index_file_path_with_version(
|
||||
&self,
|
||||
index_id: mito2::sst::file::RegionIndexId,
|
||||
) -> String {
|
||||
mito2::sst::location::index_file_path(&self.table_dir, index_id, PathType::Bare)
|
||||
}
|
||||
fn build_sst_file_path(&self, file_id: RegionFileId) -> String {
|
||||
mito2::sst::location::sst_file_path(&self.table_dir, file_id, PathType::Bare)
|
||||
}
|
||||
}
|
||||
|
||||
fn semantic_type(s: &str) -> SemanticType {
|
||||
match s.to_ascii_lowercase().as_str() {
|
||||
"tag" => SemanticType::Tag,
|
||||
"field" => SemanticType::Field,
|
||||
"timestamp" => SemanticType::Timestamp,
|
||||
other => panic!("unsupported semantic {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn concrete_type(ty: &str) -> ConcreteDataType {
|
||||
match ty.to_ascii_uppercase().as_str() {
|
||||
"STRING" => ConcreteDataType::string_datatype(),
|
||||
"DOUBLE" => ConcreteDataType::float64_datatype(),
|
||||
"UINT64" => ConcreteDataType::uint64_datatype(),
|
||||
"TIMESTAMP(9)" => ConcreteDataType::timestamp_nanosecond_datatype(),
|
||||
"TIMESTAMP(3)" => ConcreteDataType::timestamp_millisecond_datatype(),
|
||||
other => panic!("unsupported column type {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn build_region_metadata(table: &TableConfig, region_id: RegionId) -> RegionMetadata {
|
||||
let mut builder = store_api::metadata::RegionMetadataBuilder::new(region_id);
|
||||
for (idx, col) in table.columns.iter().enumerate() {
|
||||
let mut schema = ColumnSchema::new(
|
||||
&col.name,
|
||||
concrete_type(&col.ty),
|
||||
col.semantic != "timestamp",
|
||||
);
|
||||
if col.semantic == "tag" {
|
||||
schema = schema.with_inverted_index(true);
|
||||
}
|
||||
if col.semantic == "tag" || col.semantic == "field" {
|
||||
schema = schema
|
||||
.with_skipping_options(SkippingIndexOptions {
|
||||
granularity: 1,
|
||||
..Default::default()
|
||||
})
|
||||
.expect("valid skipping index options for query perf fixture");
|
||||
}
|
||||
if col.name == table.time_index {
|
||||
schema = schema.with_time_index(true);
|
||||
}
|
||||
builder.push_column_metadata(ColumnMetadata {
|
||||
column_schema: schema,
|
||||
semantic_type: semantic_type(&col.semantic),
|
||||
column_id: idx as u32,
|
||||
});
|
||||
}
|
||||
let pk_ids = table
|
||||
.primary_key
|
||||
.iter()
|
||||
.map(|name| {
|
||||
table
|
||||
.columns
|
||||
.iter()
|
||||
.position(|c| c.name == *name)
|
||||
.unwrap_or_else(|| panic!("primary key column {name} not found")) as u32
|
||||
})
|
||||
.collect();
|
||||
builder.primary_key(pk_ids);
|
||||
builder.primary_key_encoding(PrimaryKeyEncoding::Dense);
|
||||
builder
|
||||
.build()
|
||||
.expect("region metadata should be valid for fixture table config")
|
||||
}
|
||||
|
||||
fn encode_dense_primary_key(table: &TableConfig, values: &HashMap<String, String>) -> Vec<u8> {
|
||||
let fields = table
|
||||
.primary_key
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, _)| {
|
||||
(
|
||||
idx as u32,
|
||||
SortField::new(ConcreteDataType::string_datatype()),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
let converter = DensePrimaryKeyCodec::with_fields(fields);
|
||||
converter
|
||||
.encode(
|
||||
table
|
||||
.primary_key
|
||||
.iter()
|
||||
.map(|name| datatypes::value::ValueRef::String(values[name].as_str())),
|
||||
)
|
||||
.expect("dense primary key encoding should match fixture primary key fields")
|
||||
}
|
||||
|
||||
fn tag_value(col: &ColumnConfig, series: usize) -> String {
|
||||
match col
|
||||
.distribution
|
||||
.as_ref()
|
||||
.expect("tag distribution is required")
|
||||
{
|
||||
Distribution::Cardinality { values, prefix } => {
|
||||
format!("{}{}", prefix, series % values.get())
|
||||
}
|
||||
_ => panic!("tag column {} requires cardinality distribution", col.name),
|
||||
}
|
||||
}
|
||||
|
||||
fn series_for_row(layout: &LayoutConfig, sst_idx: usize, base_row: usize, row: usize) -> usize {
|
||||
match layout.series_layout.as_str() {
|
||||
"round_robin" | "timestamp_major" => (base_row + row) % layout.series_count.get(),
|
||||
"per_sst" => sst_idx % layout.series_count.get(),
|
||||
other => panic!("unsupported series_layout {other}"),
|
||||
}
|
||||
}
|
||||
|
||||
fn timestamp_for_row(layout: &LayoutConfig, base_row: usize, row: usize) -> i64 {
|
||||
let logical_row = base_row + row;
|
||||
let timestamp_step = if layout.series_layout == "timestamp_major" {
|
||||
logical_row / layout.series_count.get()
|
||||
} else {
|
||||
logical_row
|
||||
};
|
||||
|
||||
layout.start_unix_nanos + (timestamp_step as i64 * layout.step_nanos)
|
||||
}
|
||||
|
||||
fn wave_value(min: f64, max: f64, row: usize) -> f64 {
|
||||
let span = max - min;
|
||||
let phase = (row % 1024) as f64 / 1023.0;
|
||||
min + span * (0.5 - 0.5 * (std::f64::consts::TAU * phase).cos())
|
||||
}
|
||||
|
||||
fn generate_record_batch(
|
||||
table: &TableConfig,
|
||||
metadata: &RegionMetadataRef,
|
||||
layout: &LayoutConfig,
|
||||
sst_idx: usize,
|
||||
sequence: u64,
|
||||
) -> RecordBatch {
|
||||
let flat_schema = to_flat_sst_arrow_schema(metadata, &FlatSchemaOptions::default());
|
||||
let rows = layout.rows_per_sst;
|
||||
let mut columns = Vec::new();
|
||||
let base_row = sst_idx * rows;
|
||||
|
||||
for col in &table.columns {
|
||||
match (col.semantic.as_str(), col.ty.to_ascii_uppercase().as_str()) {
|
||||
("tag", "STRING") => {
|
||||
let mut b = StringDictionaryBuilder::<UInt32Type>::new();
|
||||
for row in 0..rows {
|
||||
let series = series_for_row(layout, sst_idx, base_row, row);
|
||||
let value = tag_value(col, series);
|
||||
b.append_value(value);
|
||||
}
|
||||
columns.push(Arc::new(b.finish()) as ArrayRef);
|
||||
}
|
||||
("field", "DOUBLE") => {
|
||||
let (min, max) = match col.distribution.as_ref() {
|
||||
Some(Distribution::DeterministicWave { min, max }) => (*min, *max),
|
||||
_ => (0.0, 1.0),
|
||||
};
|
||||
columns.push(Arc::new(Float64Array::from(
|
||||
(0..rows)
|
||||
.map(|r| wave_value(min, max, base_row + r))
|
||||
.collect::<Vec<_>>(),
|
||||
)) as ArrayRef);
|
||||
}
|
||||
("field", "UINT64") => columns.push(Arc::new(UInt64Array::from(
|
||||
(0..rows).map(|r| (base_row + r) as u64).collect::<Vec<_>>(),
|
||||
)) as ArrayRef),
|
||||
("timestamp", "TIMESTAMP(9)") => columns.push(Arc::new(TimestampNanosecondArray::from(
|
||||
(0..rows)
|
||||
.map(|r| timestamp_for_row(layout, base_row, r))
|
||||
.collect::<Vec<_>>(),
|
||||
)) as ArrayRef),
|
||||
("timestamp", "TIMESTAMP(3)") => {
|
||||
columns.push(Arc::new(TimestampMillisecondArray::from(
|
||||
(0..rows)
|
||||
.map(|r| timestamp_for_row(layout, base_row, r) / 1_000_000)
|
||||
.collect::<Vec<_>>(),
|
||||
)) as ArrayRef)
|
||||
}
|
||||
other => panic!("unsupported column combination {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
let mut pk_builder = BinaryDictionaryBuilder::<UInt32Type>::new();
|
||||
for row in 0..rows {
|
||||
let series = series_for_row(layout, sst_idx, base_row, row);
|
||||
let values = table
|
||||
.columns
|
||||
.iter()
|
||||
.filter(|c| c.semantic == "tag")
|
||||
.map(|c| (c.name.clone(), tag_value(c, series)))
|
||||
.collect();
|
||||
pk_builder
|
||||
.append(encode_dense_primary_key(table, &values))
|
||||
.expect("append generated dense primary key to Arrow dictionary builder");
|
||||
}
|
||||
columns.push(Arc::new(pk_builder.finish()));
|
||||
columns.push(Arc::new(UInt64Array::from_value(sequence, rows)));
|
||||
columns.push(Arc::new(UInt8Array::from_value(OpType::Put as u8, rows)));
|
||||
RecordBatch::try_new(flat_schema, columns)
|
||||
.expect("generated fixture columns should match flat SST Arrow schema")
|
||||
}
|
||||
|
||||
fn file_meta_from_sst_info(
|
||||
info: &SstInfo,
|
||||
region_id: RegionId,
|
||||
file_id: FileId,
|
||||
sequence: u64,
|
||||
) -> FileMeta {
|
||||
FileMeta {
|
||||
region_id,
|
||||
file_id,
|
||||
time_range: info.time_range,
|
||||
level: 0,
|
||||
file_size: info.file_size,
|
||||
max_row_group_uncompressed_size: info.max_row_group_uncompressed_size,
|
||||
available_indexes: Default::default(),
|
||||
indexes: Default::default(),
|
||||
index_file_size: 0,
|
||||
index_version: 0,
|
||||
num_rows: info.num_rows as u64,
|
||||
num_row_groups: info.num_row_groups,
|
||||
sequence: NonZeroU64::new(sequence),
|
||||
partition_expr: None,
|
||||
num_series: info.num_series,
|
||||
primary_key_min: None,
|
||||
primary_key_max: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn deterministic_file_id(seed: u64, index: usize) -> FileId {
|
||||
let seed_hi = ((seed >> 16) & 0xffff) as u16;
|
||||
let seed_lo = (seed & 0xffff) as u16;
|
||||
FileId::parse_str(&format!(
|
||||
"00000000-0000-{seed_hi:04x}-{seed_lo:04x}-{index:012x}"
|
||||
))
|
||||
.expect("deterministic file id must be a valid UUID")
|
||||
}
|
||||
|
||||
fn remap_sst_file(
|
||||
object_store_dir: &Path,
|
||||
table_dir: &str,
|
||||
region_id: RegionId,
|
||||
old_file_id: FileId,
|
||||
new_file_id: FileId,
|
||||
) {
|
||||
if old_file_id == new_file_id {
|
||||
return;
|
||||
}
|
||||
let old_path = object_store_dir.join(mito2::sst::location::sst_file_path(
|
||||
table_dir,
|
||||
RegionFileId::new(region_id, old_file_id),
|
||||
PathType::Bare,
|
||||
));
|
||||
let new_path = object_store_dir.join(mito2::sst::location::sst_file_path(
|
||||
table_dir,
|
||||
RegionFileId::new(region_id, new_file_id),
|
||||
PathType::Bare,
|
||||
));
|
||||
if let Some(parent) = new_path.parent() {
|
||||
fs::create_dir_all(parent).expect("failed to create remapped SST parent directory");
|
||||
}
|
||||
fs::rename(&old_path, &new_path).unwrap_or_else(|err| {
|
||||
panic!(
|
||||
"failed to remap SST file {} -> {}: {err}",
|
||||
old_path.display(),
|
||||
new_path.display()
|
||||
)
|
||||
});
|
||||
}
|
||||
pub(super) async fn run_direct_sst(args: DirectArgs) {
|
||||
let case_path = args.case.as_ref().expect("--case is required");
|
||||
let out_dir = args.out_dir.as_ref().expect("--out-dir is required");
|
||||
let case_text = fs::read_to_string(case_path).expect("failed to read query perf case TOML");
|
||||
let case: CaseFile = toml::from_str(&case_text).expect("failed to parse query perf case TOML");
|
||||
let case_name = case_name_from_path(case_path);
|
||||
let scenario = case.scenario.direct_readable_sst();
|
||||
if scenario.tables.is_empty() || scenario.layout.regions != 1 {
|
||||
panic!("fixture generator supports one or more tables and exactly one region per table");
|
||||
}
|
||||
if scenario.layout.time_range_layout != "non_overlapping_per_sst" {
|
||||
panic!("MVP supports time_range_layout=non_overlapping_per_sst");
|
||||
}
|
||||
if scenario.layout.series_layout == "timestamp_major"
|
||||
&& !scenario
|
||||
.layout
|
||||
.rows_per_sst
|
||||
.is_multiple_of(scenario.layout.series_count.get())
|
||||
{
|
||||
panic!(
|
||||
"series_layout=timestamp_major requires rows_per_sst to be divisible by series_count"
|
||||
);
|
||||
}
|
||||
if scenario.layout.sst_count > 1000 && !args.allow_large {
|
||||
panic!("sst_count exceeds 1000; pass --allow-large");
|
||||
}
|
||||
let seed = scenario.seed.unwrap_or(0);
|
||||
let table_index = match args.table.as_deref() {
|
||||
Some(name) => scenario
|
||||
.tables
|
||||
.iter()
|
||||
.position(|table| table.name == name)
|
||||
.unwrap_or_else(|| panic!("--table {name} was not found in case {case_name}")),
|
||||
None if scenario.tables.len() == 1 => 0,
|
||||
None => panic!(
|
||||
"case {} contains {} tables; pass --table <name> to choose one",
|
||||
case_name,
|
||||
scenario.tables.len()
|
||||
),
|
||||
};
|
||||
let table = &scenario.tables[table_index];
|
||||
if table.engine != "mito" {
|
||||
panic!("MVP supports engine=mito");
|
||||
}
|
||||
let region_id = RegionId::from(args.region_id);
|
||||
let table_dir = args
|
||||
.table_dir
|
||||
.unwrap_or_else(|| format!("data/{}/{}/", table.database, table.name));
|
||||
if Path::new(&table_dir).is_absolute() {
|
||||
panic!("--table-dir must be relative");
|
||||
}
|
||||
if table_dir.trim_end_matches('/').rsplit('/').next()
|
||||
== Some(region_name(region_id.table_id(), region_id.region_sequence()).as_str())
|
||||
{
|
||||
panic!("--table-dir must be table-level directory");
|
||||
}
|
||||
let region_dir =
|
||||
mito2::sst::location::region_dir_from_table_dir(&table_dir, region_id, PathType::Bare);
|
||||
let format = match table.sst_format.as_deref().unwrap_or("flat") {
|
||||
"flat" => FormatType::Flat,
|
||||
"primary_key" | "primary-key" => FormatType::PrimaryKey,
|
||||
other => panic!("unsupported sst_format {other}"),
|
||||
};
|
||||
println!(
|
||||
"query_perf_fixture case={} scenario={} table={}.{} ssts={} rows_per_sst={} format={format:?}",
|
||||
case_name,
|
||||
case.scenario.kind(),
|
||||
table.database,
|
||||
table.name,
|
||||
scenario.layout.sst_count,
|
||||
scenario.layout.rows_per_sst
|
||||
);
|
||||
println!(
|
||||
"out_dir={} table_dir={} region_dir={}",
|
||||
out_dir.display(),
|
||||
table_dir,
|
||||
region_dir
|
||||
);
|
||||
if args.dry_run {
|
||||
return;
|
||||
}
|
||||
|
||||
let obj_store_dir = out_dir.join("object-store");
|
||||
let manifest_dir = out_dir.join("manifest");
|
||||
fs::create_dir_all(&obj_store_dir).expect("failed to create fixture object-store directory");
|
||||
fs::create_dir_all(&manifest_dir).expect("failed to create fixture manifest directory");
|
||||
let ostorage = ObjectStore::new(FsBuilder::default().root(&obj_store_dir.to_string_lossy()))
|
||||
.expect("failed to create filesystem object store for fixture output")
|
||||
.finish();
|
||||
let metadata: RegionMetadataRef = Arc::new(build_region_metadata(table, region_id));
|
||||
let mut files = HashMap::with_capacity(scenario.layout.sst_count);
|
||||
let mut next_file_index = 1;
|
||||
for i in 0..scenario.layout.sst_count {
|
||||
let sequence = 1000 + i as u64;
|
||||
let batch = generate_record_batch(table, &metadata, &scenario.layout, i, sequence);
|
||||
let source =
|
||||
FlatSource::new_iter(batch.schema(), Box::new(vec![batch].into_iter().map(Ok)));
|
||||
let mut metrics = Metrics::new(WriteType::Flush);
|
||||
let mut writer = ParquetWriter::new_with_object_store(
|
||||
ostorage.clone(),
|
||||
metadata.clone(),
|
||||
IndexConfig::default(),
|
||||
NoopIndexBuilder,
|
||||
FixedPathProvider {
|
||||
table_dir: table_dir.clone(),
|
||||
},
|
||||
&mut metrics,
|
||||
)
|
||||
.await;
|
||||
let opts = WriteOptions {
|
||||
write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
|
||||
row_group_size: scenario.layout.row_group_size,
|
||||
max_file_size: None,
|
||||
};
|
||||
let infos = match format {
|
||||
FormatType::Flat => writer.write_all_flat(source, Some(sequence), &opts).await,
|
||||
FormatType::PrimaryKey => {
|
||||
writer
|
||||
.write_all_flat_as_primary_key(source, Some(sequence), &opts)
|
||||
.await
|
||||
}
|
||||
}
|
||||
.expect("failed to write fixture SST through Mito parquet writer");
|
||||
for info in infos {
|
||||
let file_id = deterministic_file_id(
|
||||
seed.wrapping_add((table_index as u64) << 16),
|
||||
next_file_index,
|
||||
);
|
||||
next_file_index += 1;
|
||||
remap_sst_file(&obj_store_dir, &table_dir, region_id, info.file_id, file_id);
|
||||
files.insert(
|
||||
file_id,
|
||||
file_meta_from_sst_info(&info, region_id, file_id, sequence),
|
||||
);
|
||||
}
|
||||
}
|
||||
let overall_seq = files
|
||||
.values()
|
||||
.filter_map(|m| m.sequence)
|
||||
.map(|s| s.get())
|
||||
.max()
|
||||
.unwrap_or(0);
|
||||
let manifest = RegionManifest {
|
||||
metadata,
|
||||
files,
|
||||
removed_files: RemovedFilesRecord::default(),
|
||||
flushed_entry_id: overall_seq,
|
||||
flushed_sequence: overall_seq,
|
||||
committed_sequence: Some(overall_seq),
|
||||
manifest_version: args.checkpoint_version,
|
||||
truncated_entry_id: None,
|
||||
compaction_time_window: None,
|
||||
sst_format: format,
|
||||
append_mode: table.append_mode,
|
||||
};
|
||||
let checkpoint = RegionCheckpoint {
|
||||
last_version: args.checkpoint_version,
|
||||
compacted_actions: manifest.files.len(),
|
||||
checkpoint: Some(manifest.clone()),
|
||||
};
|
||||
let checkpoint_bytes = checkpoint
|
||||
.encode()
|
||||
.expect("failed to encode fixture region checkpoint");
|
||||
let checkpoint_path = manifest_dir.join(format!("{:020}.checkpoint", args.checkpoint_version));
|
||||
fs::write(&checkpoint_path, &checkpoint_bytes).expect("failed to write checkpoint file");
|
||||
fs::write(manifest_dir.join("_last_checkpoint"), serde_json::to_vec_pretty(&serde_json::json!({ "size": checkpoint_bytes.len(), "version": args.checkpoint_version, "checksum": null, "extend_metadata": {} })).expect("failed to serialize _last_checkpoint metadata")).expect("failed to write _last_checkpoint file");
|
||||
let files_jsonl_path = out_dir.join("files.jsonl");
|
||||
let mut jsonl = BufWriter::new(
|
||||
fs::File::create(&files_jsonl_path).expect("failed to create fixture files.jsonl"),
|
||||
);
|
||||
let mut entries: Vec<_> = manifest.files.iter().collect();
|
||||
entries.sort_by_key(|(id, _)| id.to_string());
|
||||
for (file_id, meta) in entries {
|
||||
writeln!(jsonl, "{}", serde_json::to_string(&serde_json::json!({ "file_id": file_id.to_string(), "region_id": meta.region_id.as_u64(), "object_path": mito2::sst::location::sst_file_path(&table_dir, RegionFileId::new(meta.region_id, *file_id), PathType::Bare), "time_range_start": meta.time_range.0.value(), "time_range_end": meta.time_range.1.value(), "num_rows": meta.num_rows, "num_row_groups": meta.num_row_groups, "file_size": meta.file_size, "num_series": meta.num_series, "sequence": meta.sequence.map(|s| s.get()) })).expect("failed to serialize fixture SST metadata JSONL entry")).expect("failed to write fixture SST metadata JSONL entry");
|
||||
}
|
||||
jsonl.flush().expect("failed to flush fixture files.jsonl");
|
||||
fs::write(out_dir.join("summary.json"), serde_json::to_vec_pretty(&serde_json::json!({ "case": case_name, "seed": seed, "table_index": table_index, "table": table.name, "database": table.database, "region_id": region_id.as_u64(), "table_dir": table_dir, "region_dir": region_dir, "sst_format": format!("{format:?}"), "sst_count": scenario.layout.sst_count, "rows_per_sst": scenario.layout.rows_per_sst, "row_group_size": scenario.layout.row_group_size, "total_rows": scenario.layout.sst_count * scenario.layout.rows_per_sst, "checkpoint_path": checkpoint_path, "files_jsonl_path": files_jsonl_path, "readback_validated": false, "metadata_source": "synthetic" })).expect("failed to serialize fixture summary")).expect("failed to write fixture summary.json");
|
||||
println!("Done. wrote {} SST file entries", manifest.files.len());
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::BTreeSet;
|
||||
use std::fs::{self, File};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use clap::Args as ClapArgs;
|
||||
use parquet::file::reader::{FileReader, SerializedFileReader};
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Debug, ClapArgs)]
|
||||
pub(super) struct InspectFooterArgs {
|
||||
#[arg(long)]
|
||||
root: PathBuf,
|
||||
#[arg(long, default_value = "greptime_value")]
|
||||
column: String,
|
||||
#[arg(long)]
|
||||
include_metadata_files: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct FooterReport {
|
||||
root: String,
|
||||
summary: FooterSummary,
|
||||
files: Vec<FooterFileReport>,
|
||||
}
|
||||
#[derive(Debug, Default, Serialize)]
|
||||
struct FooterSummary {
|
||||
column: String,
|
||||
file_count: usize,
|
||||
files_with_column: usize,
|
||||
total_file_size: u64,
|
||||
total_rows: i64,
|
||||
column_compressed_size: i64,
|
||||
column_uncompressed_size: i64,
|
||||
unique_encodings: Vec<String>,
|
||||
}
|
||||
#[derive(Debug, Serialize)]
|
||||
struct FooterFileReport {
|
||||
path: String,
|
||||
relative_path: String,
|
||||
file_size: u64,
|
||||
num_rows: i64,
|
||||
num_row_groups: usize,
|
||||
columns: Vec<FooterColumnChunkReport>,
|
||||
}
|
||||
#[derive(Debug, Serialize)]
|
||||
struct FooterColumnChunkReport {
|
||||
row_group_index: usize,
|
||||
column_index: usize,
|
||||
column_path: String,
|
||||
encodings: Vec<String>,
|
||||
compression: String,
|
||||
compressed_size: i64,
|
||||
uncompressed_size: i64,
|
||||
num_values: i64,
|
||||
}
|
||||
|
||||
pub(super) fn run_inspect_footer(
|
||||
args: InspectFooterArgs,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let files = collect_parquet_files(&args.root, args.include_metadata_files)?;
|
||||
let reports = files
|
||||
.iter()
|
||||
.map(|p| inspect_file(&args.root, p, &args.column))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&FooterReport {
|
||||
root: args.root.display().to_string(),
|
||||
summary: summarize_footer(&args.column, &reports),
|
||||
files: reports
|
||||
})?
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
fn collect_parquet_files(
|
||||
root: &Path,
|
||||
include_metadata_files: bool,
|
||||
) -> Result<Vec<PathBuf>, Box<dyn std::error::Error>> {
|
||||
let mut files = Vec::new();
|
||||
collect_parquet_files_inner(root, root, include_metadata_files, &mut files)?;
|
||||
files.sort();
|
||||
Ok(files)
|
||||
}
|
||||
fn collect_parquet_files_inner(
|
||||
root: &Path,
|
||||
path: &Path,
|
||||
include_metadata_files: bool,
|
||||
files: &mut Vec<PathBuf>,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
if path.is_dir() {
|
||||
for entry in fs::read_dir(path)? {
|
||||
collect_parquet_files_inner(root, &entry?.path(), include_metadata_files, files)?;
|
||||
}
|
||||
} else if path.extension().is_some_and(|ext| ext == "parquet")
|
||||
&& (include_metadata_files
|
||||
|| !path
|
||||
.strip_prefix(root)
|
||||
.unwrap_or(path)
|
||||
.components()
|
||||
.any(|c| c.as_os_str() == "metadata"))
|
||||
{
|
||||
files.push(path.to_path_buf());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn inspect_file(
|
||||
root: &Path,
|
||||
path: &Path,
|
||||
column: &str,
|
||||
) -> Result<FooterFileReport, Box<dyn std::error::Error>> {
|
||||
let file_size = fs::metadata(path)?.len();
|
||||
let reader = SerializedFileReader::new(File::open(path)?)?;
|
||||
let metadata = reader.metadata().file_metadata();
|
||||
let row_groups = reader.metadata().row_groups();
|
||||
let mut columns = Vec::new();
|
||||
for (row_group_index, rg) in row_groups.iter().enumerate() {
|
||||
for (column_index, chunk) in rg.columns().iter().enumerate() {
|
||||
let column_path = chunk.column_path().string();
|
||||
if column_path == column {
|
||||
columns.push(FooterColumnChunkReport {
|
||||
row_group_index,
|
||||
column_index,
|
||||
column_path,
|
||||
encodings: chunk.encodings().map(|e| format!("{e:?}")).collect(),
|
||||
compression: format!("{:?}", chunk.compression()),
|
||||
compressed_size: chunk.compressed_size(),
|
||||
uncompressed_size: chunk.uncompressed_size(),
|
||||
num_values: chunk.num_values(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(FooterFileReport {
|
||||
path: path.display().to_string(),
|
||||
relative_path: path
|
||||
.strip_prefix(root)
|
||||
.unwrap_or(path)
|
||||
.display()
|
||||
.to_string(),
|
||||
file_size,
|
||||
num_rows: metadata.num_rows(),
|
||||
num_row_groups: row_groups.len(),
|
||||
columns,
|
||||
})
|
||||
}
|
||||
fn summarize_footer(column: &str, files: &[FooterFileReport]) -> FooterSummary {
|
||||
let mut encodings = BTreeSet::new();
|
||||
let mut s = FooterSummary {
|
||||
column: column.to_string(),
|
||||
file_count: files.len(),
|
||||
..Default::default()
|
||||
};
|
||||
for file in files {
|
||||
s.total_file_size += file.file_size;
|
||||
s.total_rows += file.num_rows;
|
||||
if !file.columns.is_empty() {
|
||||
s.files_with_column += 1;
|
||||
}
|
||||
for c in &file.columns {
|
||||
s.column_compressed_size += c.compressed_size;
|
||||
s.column_uncompressed_size += c.uncompressed_size;
|
||||
encodings.extend(c.encodings.iter().cloned());
|
||||
}
|
||||
}
|
||||
s.unique_encodings = encodings.into_iter().collect();
|
||||
s
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod case;
|
||||
mod direct_sst;
|
||||
mod inspect_footer;
|
||||
mod prom_remote_write;
|
||||
mod util;
|
||||
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use case::*;
|
||||
use clap::{Args as ClapArgs, Parser, Subcommand};
|
||||
use direct_sst::run_direct_sst;
|
||||
use inspect_footer::{InspectFooterArgs, run_inspect_footer};
|
||||
use prom_remote_write::{PromRemoteWriteArgs, run_prom_remote_write};
|
||||
use serde_json::json;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(name = "query_perf_fixture")]
|
||||
#[command(about = "Query performance fixture helper")]
|
||||
struct Args {
|
||||
#[command(subcommand)]
|
||||
command: Option<Command>,
|
||||
|
||||
#[command(flatten)]
|
||||
legacy: DirectArgs,
|
||||
}
|
||||
|
||||
#[derive(Subcommand, Debug)]
|
||||
enum Command {
|
||||
DirectSst(DirectArgs),
|
||||
PromRemoteWrite(PromRemoteWriteArgs),
|
||||
InspectFooter(InspectFooterArgs),
|
||||
Plan(PlanArgs),
|
||||
}
|
||||
|
||||
#[derive(ClapArgs, Debug, Clone)]
|
||||
struct DirectArgs {
|
||||
/// TOML case file, for example tests/perf/query_cases/.../case.toml.
|
||||
#[arg(long, value_name = "PATH")]
|
||||
case: Option<PathBuf>,
|
||||
|
||||
/// Output directory (creates object-store/, manifest/, files.jsonl, summary.json).
|
||||
#[arg(long, value_name = "DIR")]
|
||||
out_dir: Option<PathBuf>,
|
||||
|
||||
/// Region ID to synthesize. Defaults to table id 1024, region 0.
|
||||
#[arg(long, default_value = "4398046511104")]
|
||||
region_id: u64,
|
||||
|
||||
/// Table directory relative to object-store root. Defaults to data/{database}/{table}/.
|
||||
#[arg(long, value_name = "DIR")]
|
||||
table_dir: Option<String>,
|
||||
|
||||
/// Table name to generate when the case contains multiple [[tables]].
|
||||
#[arg(long, value_name = "NAME")]
|
||||
table: Option<String>,
|
||||
|
||||
/// Manifest/checkpoint version.
|
||||
#[arg(long, default_value = "1000000")]
|
||||
checkpoint_version: u64,
|
||||
|
||||
/// Safety flag when sst_count exceeds 1000.
|
||||
#[arg(long)]
|
||||
allow_large: bool,
|
||||
|
||||
/// Print plan only.
|
||||
#[arg(long)]
|
||||
dry_run: bool,
|
||||
}
|
||||
|
||||
#[derive(ClapArgs, Debug)]
|
||||
struct PlanArgs {
|
||||
#[arg(long, value_name = "PATH")]
|
||||
case: PathBuf,
|
||||
}
|
||||
|
||||
pub async fn run() {
|
||||
let args = Args::parse();
|
||||
match args.command {
|
||||
Some(Command::DirectSst(direct)) => run_direct_sst(direct).await,
|
||||
Some(Command::PromRemoteWrite(rw)) => run_prom_remote_write(rw)
|
||||
.await
|
||||
.expect("prom remote write failed"),
|
||||
Some(Command::InspectFooter(inspect)) => {
|
||||
run_inspect_footer(inspect).expect("inspect footer failed")
|
||||
}
|
||||
Some(Command::Plan(plan)) => run_plan(plan).expect("plan failed"),
|
||||
None => run_direct_sst(args.legacy).await,
|
||||
}
|
||||
}
|
||||
|
||||
fn run_plan(args: PlanArgs) -> Result<(), Box<dyn std::error::Error>> {
|
||||
let case_text = fs::read_to_string(&args.case)?;
|
||||
let mut case: CaseFile = toml::from_str(&case_text)?;
|
||||
if let Scenario::PromRemoteWriteThenQuery(s) = &mut case.scenario
|
||||
&& let Some(storage) = &mut s.remote_write.storage
|
||||
{
|
||||
storage.populate_planned_thresholds();
|
||||
if storage.inspect {
|
||||
let mut rb = s.remote_write.read_bench.clone().unwrap_or_default();
|
||||
if rb.enabled && rb.projection.is_empty() {
|
||||
rb.projection = vec![storage.column.clone()];
|
||||
}
|
||||
s.remote_write.read_bench = Some(rb);
|
||||
} else if s
|
||||
.remote_write
|
||||
.read_bench
|
||||
.as_ref()
|
||||
.is_some_and(|rb| rb.enabled)
|
||||
{
|
||||
return Err("scenario.remote_write.read_bench requires scenario.remote_write.storage.inspect = true".into());
|
||||
}
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&json!({"schema_version": 1, "scenario": case.scenario}))?
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,333 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use clap::Args as ClapArgs;
|
||||
use prost::Message;
|
||||
use serde_json::json;
|
||||
use servers::prom_store::snappy_compress;
|
||||
|
||||
use crate::query_perf_fixture::case::ValuePattern;
|
||||
|
||||
#[derive(Debug, ClapArgs)]
|
||||
pub(super) struct PromRemoteWriteArgs {
|
||||
#[arg(long, default_value = "http://127.0.0.1:4000/v1/prometheus/write")]
|
||||
endpoint: String,
|
||||
#[arg(long, default_value = "public")]
|
||||
database: String,
|
||||
#[arg(long)]
|
||||
metric: String,
|
||||
#[arg(long, default_value = "greptime_physical_table")]
|
||||
physical_table: String,
|
||||
#[arg(long, default_value_t = 8)]
|
||||
series_count: u64,
|
||||
#[arg(long, default_value_t = 30)]
|
||||
samples_per_series: u64,
|
||||
#[arg(long, default_value_t = 1_704_067_200_000)]
|
||||
start_unix_millis: i64,
|
||||
#[arg(long, default_value_t = 15_000)]
|
||||
step_millis: i64,
|
||||
#[arg(long, alias = "batch-size", default_value_t = 8)]
|
||||
chunk_series_count: u64,
|
||||
#[arg(long, default_value_t = 60)]
|
||||
timeout_seconds: u64,
|
||||
#[arg(long, default_value_t = ValuePattern::Linear)]
|
||||
value_pattern: ValuePattern,
|
||||
#[arg(long, default_value_t = 0.0)]
|
||||
value_base: f64,
|
||||
#[arg(long, default_value_t = 0.125)]
|
||||
value_step: f64,
|
||||
#[arg(long, default_value_t = 97)]
|
||||
value_cardinality: u64,
|
||||
#[arg(long, default_value_t = 0)]
|
||||
value_seed: u64,
|
||||
#[arg(long, default_value_t = 8)]
|
||||
value_run_length: u64,
|
||||
#[arg(long, default_value_t = 100)]
|
||||
value_stall_every: u64,
|
||||
#[arg(long, default_value_t = 16)]
|
||||
value_stall_length: u64,
|
||||
#[arg(long, default_value_t = 5)]
|
||||
value_mixed_every: u64,
|
||||
#[arg(long, default_value_t = 0)]
|
||||
value_sample_offset: u64,
|
||||
#[arg(long)]
|
||||
value_total_samples_per_series: Option<u64>,
|
||||
}
|
||||
|
||||
pub(super) async fn run_prom_remote_write(
|
||||
args: PromRemoteWriteArgs,
|
||||
) -> Result<(), Box<dyn std::error::Error>> {
|
||||
use api::prom_store::remote::WriteRequest;
|
||||
let started = std::time::Instant::now();
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(args.timeout_seconds))
|
||||
.build()?;
|
||||
let mut batches = 0_u64;
|
||||
let mut rows = 0_u64;
|
||||
let mut http_statuses = Vec::new();
|
||||
for first in (0..args.series_count).step_by(args.chunk_series_count.max(1) as usize) {
|
||||
let last = (first + args.chunk_series_count.max(1)).min(args.series_count);
|
||||
let timeseries = (first..last)
|
||||
.map(|series_idx| make_prom_series(&args, series_idx))
|
||||
.collect::<Vec<_>>();
|
||||
rows += timeseries
|
||||
.iter()
|
||||
.map(|ts| ts.samples.len() as u64)
|
||||
.sum::<u64>();
|
||||
let body = snappy_compress(
|
||||
&WriteRequest {
|
||||
timeseries,
|
||||
..Default::default()
|
||||
}
|
||||
.encode_to_vec(),
|
||||
)?;
|
||||
let response = client
|
||||
.post(&args.endpoint)
|
||||
.header("Content-Encoding", "snappy")
|
||||
.body(body)
|
||||
.send()
|
||||
.await?;
|
||||
let status = response.status();
|
||||
http_statuses.push(status.as_u16());
|
||||
if !status.is_success() {
|
||||
return Err(format!(
|
||||
"remote-write failed with status {status}: {}",
|
||||
response.text().await.unwrap_or_default()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
batches += 1;
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
json!({"status":"ok","endpoint":args.endpoint,"database":args.database,"metric":args.metric,"physical_table":args.physical_table,"series_count":args.series_count,"samples_per_series":args.samples_per_series,"rows":rows,"samples_written":rows,"batches":batches,"elapsed_seconds":started.elapsed().as_secs_f64(),"http_statuses":http_statuses,"value":{"pattern":args.value_pattern.to_string(),"base":args.value_base,"step":args.value_step,"cardinality":args.value_cardinality,"seed":args.value_seed,"run_length":args.value_run_length,"stall_every":args.value_stall_every,"stall_length":args.value_stall_length,"mixed_every":args.value_mixed_every,"sample_offset":args.value_sample_offset,"total_samples_per_series":args.value_total_samples_per_series.unwrap_or(args.samples_per_series)}})
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn make_prom_series(
|
||||
args: &PromRemoteWriteArgs,
|
||||
series_idx: u64,
|
||||
) -> api::prom_store::remote::TimeSeries {
|
||||
let mut labels = vec![
|
||||
prom_label("__name__", &args.metric),
|
||||
prom_label("x_greptime_database", &args.database),
|
||||
prom_label("x_greptime_physical_table", &args.physical_table),
|
||||
prom_label("host", &format!("host{:04}", series_idx % 1024)),
|
||||
prom_label("instance", &format!("instance{:06}", series_idx)),
|
||||
];
|
||||
labels.sort_unstable_by(|l, r| l.name.cmp(&r.name));
|
||||
let samples = (0..args.samples_per_series)
|
||||
.map(|sample_idx| api::prom_store::remote::Sample {
|
||||
value: deterministic_prom_value(args, series_idx, sample_idx),
|
||||
timestamp: args.start_unix_millis + (sample_idx as i64 * args.step_millis),
|
||||
})
|
||||
.collect();
|
||||
api::prom_store::remote::TimeSeries {
|
||||
labels,
|
||||
samples,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
fn prom_label(name: &str, value: &str) -> api::prom_store::remote::Label {
|
||||
api::prom_store::remote::Label {
|
||||
name: name.to_string(),
|
||||
value: value.to_string(),
|
||||
}
|
||||
}
|
||||
fn deterministic_prom_value(args: &PromRemoteWriteArgs, series_idx: u64, sample_idx: u64) -> f64 {
|
||||
let ordinal = series_idx
|
||||
* args
|
||||
.value_total_samples_per_series
|
||||
.unwrap_or(args.samples_per_series)
|
||||
+ args.value_sample_offset
|
||||
+ sample_idx;
|
||||
let local = args.value_sample_offset + sample_idx;
|
||||
match args.value_pattern {
|
||||
ValuePattern::Linear => {
|
||||
args.value_base + (series_idx % 97) as f64 + local as f64 * args.value_step
|
||||
}
|
||||
ValuePattern::Constant => args.value_base,
|
||||
ValuePattern::Modulo => {
|
||||
args.value_base + (ordinal % args.value_cardinality.max(1)) as f64 * args.value_step
|
||||
}
|
||||
ValuePattern::Unique => args.value_base + ordinal as f64 * args.value_step,
|
||||
ValuePattern::SeededRandom => {
|
||||
args.value_base
|
||||
+ (splitmix64(ordinal ^ args.value_seed) % args.value_cardinality.max(1)) as f64
|
||||
* args.value_step
|
||||
}
|
||||
ValuePattern::RunLength => {
|
||||
args.value_base
|
||||
+ ((ordinal / args.value_run_length.max(1)) % args.value_cardinality.max(1)) as f64
|
||||
* args.value_step
|
||||
}
|
||||
ValuePattern::QuantizedSignal => {
|
||||
args.value_base
|
||||
+ (((series_idx % args.value_cardinality.max(1))
|
||||
+ (local / args.value_run_length.max(1)))
|
||||
% args.value_cardinality.max(1)) as f64
|
||||
* args.value_step
|
||||
}
|
||||
ValuePattern::SignalWithSporadicStalls => {
|
||||
let every = args.value_stall_every.max(1);
|
||||
let phase = local % every;
|
||||
let effective = if phase < args.value_stall_length.min(every) {
|
||||
local - phase
|
||||
} else {
|
||||
local
|
||||
};
|
||||
args.value_base + (series_idx % 97) as f64 + effective as f64 * args.value_step
|
||||
}
|
||||
ValuePattern::MixedSignalRepeated => {
|
||||
if local.is_multiple_of(args.value_mixed_every.max(1)) {
|
||||
args.value_base
|
||||
} else {
|
||||
args.value_base + (series_idx % 97) as f64 + local as f64 * args.value_step
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
fn splitmix64(mut value: u64) -> u64 {
|
||||
value = value.wrapping_add(0x9e3779b97f4a7c15);
|
||||
let mut mixed = value;
|
||||
mixed = (mixed ^ (mixed >> 30)).wrapping_mul(0xbf58476d1ce4e5b9);
|
||||
mixed = (mixed ^ (mixed >> 27)).wrapping_mul(0x94d049bb133111eb);
|
||||
mixed ^ (mixed >> 31)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::collections::HashSet;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn args(pattern: ValuePattern) -> PromRemoteWriteArgs {
|
||||
PromRemoteWriteArgs {
|
||||
endpoint: "http://127.0.0.1:4000/v1/prometheus/write".to_string(),
|
||||
database: "public".to_string(),
|
||||
metric: "test_metric".to_string(),
|
||||
physical_table: "greptime_physical_table".to_string(),
|
||||
series_count: 2,
|
||||
samples_per_series: 64,
|
||||
start_unix_millis: 1_704_067_200_000,
|
||||
step_millis: 60_000,
|
||||
chunk_series_count: 2,
|
||||
timeout_seconds: 60,
|
||||
value_pattern: pattern,
|
||||
value_base: 0.0,
|
||||
value_step: 1.0,
|
||||
value_cardinality: 1_024,
|
||||
value_seed: 8_444,
|
||||
value_run_length: 16,
|
||||
value_stall_every: 100,
|
||||
value_stall_length: 16,
|
||||
value_mixed_every: 5,
|
||||
value_sample_offset: 0,
|
||||
value_total_samples_per_series: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn seeded_random_is_deterministic_in_range_and_highly_distinct() {
|
||||
let mut args = args(ValuePattern::SeededRandom);
|
||||
args.samples_per_series = 4_096;
|
||||
args.value_cardinality = 29_491_200;
|
||||
|
||||
let values = (0..args.samples_per_series)
|
||||
.map(|sample_idx| deterministic_prom_value(&args, 0, sample_idx))
|
||||
.collect::<Vec<_>>();
|
||||
let repeated = (0..args.samples_per_series)
|
||||
.map(|sample_idx| deterministic_prom_value(&args, 0, sample_idx))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
assert_eq!(values, repeated);
|
||||
assert!(values.iter().all(|value| {
|
||||
(args.value_base..=args.value_base + (args.value_cardinality - 1) as f64)
|
||||
.contains(value)
|
||||
}));
|
||||
assert!(
|
||||
values
|
||||
.iter()
|
||||
.map(|value| value.to_bits())
|
||||
.collect::<HashSet<_>>()
|
||||
.len()
|
||||
> 4_000
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_length_values_have_exact_sixteen_sample_runs() {
|
||||
let args = args(ValuePattern::RunLength);
|
||||
|
||||
for sample_idx in 0..64 {
|
||||
assert_eq!(
|
||||
deterministic_prom_value(&args, 0, sample_idx),
|
||||
(sample_idx / 16) as f64
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mixed_every_five_repeats_only_the_expected_positions() {
|
||||
let mut args = args(ValuePattern::MixedSignalRepeated);
|
||||
args.value_step = 0.125;
|
||||
|
||||
for sample_idx in 0..20 {
|
||||
let expected = if sample_idx % 5 == 0 {
|
||||
0.0
|
||||
} else {
|
||||
1.0 + sample_idx as f64 * 0.125
|
||||
};
|
||||
assert_eq!(deterministic_prom_value(&args, 1, sample_idx), expected);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunked_values_match_monolithic_values_through_offsets() {
|
||||
let mut monolithic = args(ValuePattern::SeededRandom);
|
||||
monolithic.samples_per_series = 14_400;
|
||||
monolithic.value_total_samples_per_series = Some(14_400);
|
||||
|
||||
for series_idx in 0..2 {
|
||||
let expected = (0..monolithic.samples_per_series)
|
||||
.map(|sample_idx| deterministic_prom_value(&monolithic, series_idx, sample_idx))
|
||||
.collect::<Vec<_>>();
|
||||
for chunk_idx in 0..10 {
|
||||
let mut chunk = args(ValuePattern::SeededRandom);
|
||||
chunk.samples_per_series = 1_440;
|
||||
chunk.value_sample_offset = chunk_idx * 1_440;
|
||||
chunk.value_total_samples_per_series = Some(14_400);
|
||||
for sample_idx in 0..chunk.samples_per_series {
|
||||
assert_eq!(
|
||||
deterministic_prom_value(&chunk, series_idx, sample_idx),
|
||||
expected[(chunk.value_sample_offset + sample_idx) as usize]
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn values_continue_across_series_boundaries() {
|
||||
let mut args = args(ValuePattern::Unique);
|
||||
args.samples_per_series = 16;
|
||||
args.value_total_samples_per_series = Some(16);
|
||||
args.value_base = 1.0;
|
||||
args.value_step = 0.5;
|
||||
|
||||
assert_eq!(deterministic_prom_value(&args, 0, 15), 8.5);
|
||||
assert_eq!(deterministic_prom_value(&args, 1, 0), 9.0);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// Copyright 2023 Greptime Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
pub(super) fn case_name_from_path(path: &Path) -> String {
|
||||
path.parent()
|
||||
.and_then(Path::file_name)
|
||||
.and_then(|name| name.to_str())
|
||||
.unwrap_or("query_perf_case")
|
||||
.to_string()
|
||||
}
|
||||
@@ -15,11 +15,10 @@
|
||||
pub mod builder;
|
||||
#[allow(clippy::print_stdout)]
|
||||
pub(crate) mod objbench;
|
||||
#[cfg(feature = "dev-tools")]
|
||||
#[allow(clippy::print_stdout)]
|
||||
mod parquetbench;
|
||||
pub mod parquetbench;
|
||||
#[allow(clippy::print_stdout)]
|
||||
mod scanbench;
|
||||
pub mod scanbench;
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
@@ -40,7 +39,6 @@ use tracing_appender::non_blocking::WorkerGuard;
|
||||
use crate::App;
|
||||
use crate::datanode::builder::InstanceBuilder;
|
||||
use crate::datanode::objbench::ObjbenchCommand;
|
||||
#[cfg(feature = "dev-tools")]
|
||||
use crate::datanode::parquetbench::ParquetbenchCommand;
|
||||
use crate::datanode::scanbench::ScanbenchCommand;
|
||||
use crate::error::{
|
||||
@@ -115,7 +113,6 @@ impl Command {
|
||||
SubCommand::Start(cmd) => cmd.load_options(global_options),
|
||||
// Bench commands are standalone utilities and don't need to load DatanodeOptions.
|
||||
SubCommand::Objbench(_) | SubCommand::Scanbench(_) => Self::default_bench_options(),
|
||||
#[cfg(feature = "dev-tools")]
|
||||
SubCommand::Parquetbench(_) => Self::default_bench_options(),
|
||||
}
|
||||
}
|
||||
@@ -141,7 +138,6 @@ pub enum SubCommand {
|
||||
/// Scan benchmark tool - benchmarks scanning a region directly from storage
|
||||
Scanbench(ScanbenchCommand),
|
||||
/// Benchmark scanning a single parquet SST.
|
||||
#[cfg(feature = "dev-tools")]
|
||||
Parquetbench(ParquetbenchCommand),
|
||||
}
|
||||
|
||||
@@ -160,7 +156,6 @@ impl SubCommand {
|
||||
cmd.run().await?;
|
||||
std::process::exit(0);
|
||||
}
|
||||
#[cfg(feature = "dev-tools")]
|
||||
SubCommand::Parquetbench(cmd) => {
|
||||
cmd.run().await?;
|
||||
std::process::exit(0);
|
||||
|
||||
@@ -12,11 +12,17 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
use common_base::Plugins;
|
||||
use store_api::region_engine::RegionEngine;
|
||||
use store_api::region_request::{RegionCloseRequest, RegionRequest};
|
||||
use store_api::storage::RegionId;
|
||||
|
||||
use crate::config::MitoConfig;
|
||||
use crate::engine::flush_test::MockRegionHook;
|
||||
use crate::engine::region_hook::RegionHookRef;
|
||||
use crate::test_util::{CreateRequestBuilder, TestEnv};
|
||||
|
||||
#[tokio::test]
|
||||
@@ -60,3 +66,40 @@ async fn test_engine_close_region_with_format(flat_format: bool) {
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_region_hook_on_close() {
|
||||
common_telemetry::init_default_ut_logging();
|
||||
let mut env = TestEnv::new().await;
|
||||
|
||||
let hook = Arc::new(MockRegionHook::new());
|
||||
let plugins = Plugins::new();
|
||||
plugins.insert(hook.clone() as RegionHookRef);
|
||||
|
||||
let engine = env
|
||||
.create_engine_with_plugins(MitoConfig::default(), plugins)
|
||||
.await;
|
||||
|
||||
let region_id = RegionId::new(1, 1);
|
||||
let request = CreateRequestBuilder::new().build();
|
||||
engine
|
||||
.handle_request(region_id, RegionRequest::Create(request))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Sanity: no lifecycle events before closing besides the create open.
|
||||
assert_eq!(hook.opened_count.load(Ordering::Relaxed), 1);
|
||||
assert_eq!(hook.closed_count.load(Ordering::Relaxed), 0);
|
||||
assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 0);
|
||||
|
||||
engine
|
||||
.handle_request(region_id, RegionRequest::Close(RegionCloseRequest {}))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Closing fires on_region_closed exactly once.
|
||||
assert_eq!(hook.closed_count.load(Ordering::Relaxed), 1);
|
||||
// Close must not be confused with drop.
|
||||
assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 0);
|
||||
assert_eq!(hook.files_removed_count.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
@@ -13,9 +13,11 @@
|
||||
// limitations under the License.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::Ordering;
|
||||
use std::time::Duration;
|
||||
|
||||
use api::v1::Rows;
|
||||
use common_base::Plugins;
|
||||
use common_meta::key::SchemaMetadataManager;
|
||||
use common_meta::kv_backend::KvBackendRef;
|
||||
use object_store::util::join_path;
|
||||
@@ -25,7 +27,9 @@ use store_api::storage::RegionId;
|
||||
|
||||
use crate::config::MitoConfig;
|
||||
use crate::engine::MitoEngine;
|
||||
use crate::engine::flush_test::MockRegionHook;
|
||||
use crate::engine::listener::DropListener;
|
||||
use crate::engine::region_hook::RegionHookRef;
|
||||
use crate::test_util::{
|
||||
CreateRequestBuilder, TestEnv, build_rows_for_key, flush_region, put_rows, rows_schema,
|
||||
};
|
||||
@@ -302,3 +306,75 @@ async fn test_engine_drop_region_for_custom_store_with_format(flat_format: bool)
|
||||
.unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_region_hook_on_drop() {
|
||||
common_telemetry::init_default_ut_logging();
|
||||
let mut env = TestEnv::with_prefix("drop_hook").await;
|
||||
|
||||
let hook = Arc::new(MockRegionHook::new());
|
||||
let plugins = Plugins::new();
|
||||
plugins.insert(hook.clone() as RegionHookRef);
|
||||
|
||||
let engine = env
|
||||
.create_engine_with_plugins(MitoConfig::default(), plugins)
|
||||
.await;
|
||||
|
||||
let region_id = RegionId::new(1, 1);
|
||||
env.get_schema_metadata_manager()
|
||||
.register_region_table_info(
|
||||
region_id.table_id(),
|
||||
"test_table",
|
||||
"test_catalog",
|
||||
"test_schema",
|
||||
None,
|
||||
env.get_kv_backend(),
|
||||
)
|
||||
.await;
|
||||
|
||||
let request = CreateRequestBuilder::new().build();
|
||||
let column_schemas = rows_schema(&request);
|
||||
engine
|
||||
.handle_request(region_id, RegionRequest::Create(request))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Put some rows so the region has active data (kept in memtable; we
|
||||
// intentionally do not flush so the drop GC worker finds no parquet files
|
||||
// and removes the directory immediately, avoiding the multi-minute GC wait).
|
||||
let rows = Rows {
|
||||
schema: column_schemas,
|
||||
rows: build_rows_for_key("a", 0, 2, 0),
|
||||
};
|
||||
put_rows(&engine, region_id, rows).await;
|
||||
|
||||
// Sanity: nothing fired yet besides the open from create.
|
||||
assert_eq!(hook.opened_count.load(Ordering::Relaxed), 1);
|
||||
assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 0);
|
||||
assert_eq!(hook.closed_count.load(Ordering::Relaxed), 0);
|
||||
|
||||
// full drop (partial_drop = false) so the GC worker physically deletes the
|
||||
// directory and fires on_region_files_removed.
|
||||
engine
|
||||
.handle_request(
|
||||
region_id,
|
||||
RegionRequest::Drop(RegionDropRequest {
|
||||
fast_path: false,
|
||||
force: false,
|
||||
partial_drop: false,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// on_region_dropped fires inline while handling the drop request.
|
||||
hook.wait_for_dropped().await;
|
||||
assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 1);
|
||||
// Drop must not be confused with close.
|
||||
assert_eq!(hook.closed_count.load(Ordering::Relaxed), 0);
|
||||
|
||||
// on_region_files_removed fires from the background GC worker once the
|
||||
// directory is physically deleted.
|
||||
hook.wait_for_files_removed().await;
|
||||
assert_eq!(hook.files_removed_count.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
@@ -895,24 +895,46 @@ async fn test_update_topic_latest_entry_id(factory: Option<LogStoreFactory>) {
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct MockRegionHook {
|
||||
sst_written_count: AtomicUsize,
|
||||
manifest_updated_count: AtomicUsize,
|
||||
pub(super) struct MockRegionHook {
|
||||
pub(super) sst_written_count: AtomicUsize,
|
||||
pub(super) manifest_updated_count: AtomicUsize,
|
||||
pub(super) opened_count: AtomicUsize,
|
||||
pub(super) closed_count: AtomicUsize,
|
||||
pub(super) dropped_count: AtomicUsize,
|
||||
pub(super) files_removed_count: AtomicUsize,
|
||||
notify: Notify,
|
||||
dropped_notify: Notify,
|
||||
files_removed_notify: Notify,
|
||||
}
|
||||
|
||||
impl MockRegionHook {
|
||||
fn new() -> Self {
|
||||
pub(super) fn new() -> Self {
|
||||
Self {
|
||||
sst_written_count: AtomicUsize::new(0),
|
||||
manifest_updated_count: AtomicUsize::new(0),
|
||||
opened_count: AtomicUsize::new(0),
|
||||
closed_count: AtomicUsize::new(0),
|
||||
dropped_count: AtomicUsize::new(0),
|
||||
files_removed_count: AtomicUsize::new(0),
|
||||
notify: Notify::new(),
|
||||
dropped_notify: Notify::new(),
|
||||
files_removed_notify: Notify::new(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_manifest_update(&self) {
|
||||
self.notify.notified().await;
|
||||
}
|
||||
|
||||
/// Waits until `on_region_dropped` has fired at least once.
|
||||
pub(super) async fn wait_for_dropped(&self) {
|
||||
self.dropped_notify.notified().await;
|
||||
}
|
||||
|
||||
/// Waits until `on_region_files_removed` has fired at least once.
|
||||
pub(super) async fn wait_for_files_removed(&self) {
|
||||
self.files_removed_notify.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
@@ -966,6 +988,35 @@ impl RegionHook for MockRegionHook {
|
||||
);
|
||||
self.notify.notify_one();
|
||||
}
|
||||
|
||||
async fn on_region_opened(&self, region_id: RegionId, _region_metadata: &RegionMetadataRef) {
|
||||
self.opened_count.fetch_add(1, Ordering::Relaxed);
|
||||
common_telemetry::info!("MockRegionHook::on_region_opened: region={}", region_id);
|
||||
}
|
||||
|
||||
async fn on_region_closed(&self, region_id: RegionId, _region_metadata: &RegionMetadataRef) {
|
||||
self.closed_count.fetch_add(1, Ordering::Relaxed);
|
||||
common_telemetry::info!("MockRegionHook::on_region_closed: region={}", region_id);
|
||||
}
|
||||
|
||||
async fn on_region_dropped(&self, region_id: RegionId, _region_metadata: &RegionMetadataRef) {
|
||||
self.dropped_count.fetch_add(1, Ordering::Relaxed);
|
||||
common_telemetry::info!("MockRegionHook::on_region_dropped: region={}", region_id);
|
||||
self.dropped_notify.notify_one();
|
||||
}
|
||||
|
||||
async fn on_region_files_removed(
|
||||
&self,
|
||||
region_id: RegionId,
|
||||
_region_metadata: &RegionMetadataRef,
|
||||
) {
|
||||
self.files_removed_count.fetch_add(1, Ordering::Relaxed);
|
||||
common_telemetry::info!(
|
||||
"MockRegionHook::on_region_files_removed: region={}",
|
||||
region_id
|
||||
);
|
||||
self.files_removed_notify.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -32,6 +32,8 @@ use tokio::sync::oneshot;
|
||||
|
||||
use crate::compaction::compactor::{OpenCompactionRegionRequest, open_compaction_region};
|
||||
use crate::config::MitoConfig;
|
||||
use crate::engine::flush_test::MockRegionHook;
|
||||
use crate::engine::region_hook::RegionHookRef;
|
||||
use crate::error;
|
||||
use crate::region::opener::{PartitionExprFetcher, PartitionExprFetcherRef};
|
||||
use crate::region::options::RegionOptions;
|
||||
@@ -805,3 +807,36 @@ async fn test_open_keeps_none_without_fetcher() {
|
||||
let meta = engine.get_region(region_id).unwrap().metadata();
|
||||
assert!(meta.partition_expr.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_region_hook_on_open() {
|
||||
common_telemetry::init_default_ut_logging();
|
||||
let mut env = TestEnv::with_prefix("open_hook").await;
|
||||
|
||||
let hook = Arc::new(MockRegionHook::new());
|
||||
let plugins = Plugins::new();
|
||||
plugins.insert(hook.clone() as RegionHookRef);
|
||||
|
||||
let engine = env
|
||||
.create_engine_with_plugins(MitoConfig::default(), plugins)
|
||||
.await;
|
||||
|
||||
let region_id = RegionId::new(1, 1);
|
||||
let request = CreateRequestBuilder::new().build();
|
||||
let table_dir = request.table_dir.clone();
|
||||
engine
|
||||
.handle_request(region_id, RegionRequest::Create(request))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Creating a region fires on_region_opened exactly once.
|
||||
assert_eq!(hook.opened_count.load(Ordering::Relaxed), 1);
|
||||
assert_eq!(hook.closed_count.load(Ordering::Relaxed), 0);
|
||||
|
||||
// `reopen_region` closes then opens; the open must fire the hook again.
|
||||
reopen_region(&engine, region_id, table_dir, false, Default::default()).await;
|
||||
|
||||
assert_eq!(hook.opened_count.load(Ordering::Relaxed), 2);
|
||||
assert_eq!(hook.closed_count.load(Ordering::Relaxed), 1);
|
||||
assert_eq!(hook.dropped_count.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
@@ -16,15 +16,24 @@
|
||||
//!
|
||||
//! ## Design
|
||||
//!
|
||||
//! The [`RegionHook`] trait provides two methods with clear separation of concerns:
|
||||
//! The [`RegionHook`] trait observes region activity through two categories of
|
||||
//! callbacks — manifest/file observation and region lifecycle:
|
||||
//!
|
||||
//! - [`on_sst_files_written`]: Fires when mito2 physically writes SST **data files**.
|
||||
//! Provides per-file [`SstInfo`] + [`FileMeta`]; metadata richness varies by path
|
||||
//! (see [`SstFileInfo`] and the coverage footnote).
|
||||
//!
|
||||
//! - [`on_manifest_updated`]: Fires after **any** manifest write is successfully committed.
|
||||
//! Receives the full [`RegionMetaActionList`] so consumers can inspect what changed
|
||||
//! (file additions/removals, schema changes, truncation, partition expression changes, etc.).
|
||||
//! - [`on_manifest_updated`]: Fires after a manifest write is committed to the **live**
|
||||
//! (normal) manifest directory. Writes to the staging directory (enter staging,
|
||||
//! operations during staging, the intermediate apply-staging edit) are suppressed —
|
||||
//! their effects are accumulated and delivered in a single notification when the
|
||||
//! staged actions are promoted to the live manifest. Receives the full
|
||||
//! [`RegionMetaActionList`] so consumers can inspect what changed (file additions/
|
||||
//! removals, schema changes, truncation, partition expression changes, etc.).
|
||||
//!
|
||||
//! - [`on_region_opened`] / [`on_region_closed`] / [`on_region_dropped`] / [`on_region_files_removed`]:
|
||||
//! Region **lifecycle** callbacks for open, close, logical drop, and physical file removal.
|
||||
//! See [Region lifecycle](#region-lifecycle) below.
|
||||
//!
|
||||
//! Hook implementations are registered via the [`Plugins`](common_base::Plugins) system:
|
||||
//! ```ignore
|
||||
@@ -33,6 +42,12 @@
|
||||
//!
|
||||
//! ## Coverage
|
||||
//!
|
||||
//! Only manifest writes to the **normal** (live) manifest directory trigger
|
||||
//! `on_manifest_updated`. Writes to the staging manifest directory (operations
|
||||
//! that happen while the region is in staging mode) are intentionally suppressed:
|
||||
//! their effects are accumulated and delivered in a single notification when
|
||||
//! the staged actions are promoted to the live manifest via `exit_staging_on_success`.
|
||||
//!
|
||||
//! | Scenario | `on_sst_files_written` | `on_manifest_updated` |
|
||||
//! |------------------------------|:----------------------:|:---------------------:|
|
||||
//! | Flush (memtable → SST) | ✅ Yes | ✅ Yes |
|
||||
@@ -40,23 +55,57 @@
|
||||
//! | Remote compaction | ✅ (compactor node) ¹ | ✅ (compactor node) ¹ |
|
||||
//! | RegionEdit / bulk ingestion | ❌ (files pre-written) | ✅ Yes |
|
||||
//! | Copy region | ❌ (object-store copy) | ✅ Yes |
|
||||
//! | Apply staging | ❌ (delegates to edit) | ✅ Yes ² |
|
||||
//! | Apply staging (promote) | ❌ (delegates to edit) | ✅ Yes ² |
|
||||
//! | Alter (schema change) | ❌ (no SST files) | ✅ Yes |
|
||||
//! | Truncate | ❌ (removes files) | ✅ Yes |
|
||||
//! | Enter staging | ❌ (no SST files) | ✅ Yes |
|
||||
//! | Enter staging | ❌ (no SST files) | ❌ (staging dir) |
|
||||
//! | Operations during staging | N/A | ❌ (staging dir) |
|
||||
//! | Async index build | ❌ (index files only) | ✅ Yes |
|
||||
//!
|
||||
//! ¹ Remote compaction runs on a dedicated compactor node via `open_compaction_region()`;
|
||||
//! pass plugins via `OpenCompactionRegionRequest` to enable hooks there. `sst_infos` is
|
||||
//! `#[serde(skip)]` over the wire, so each [`SstInfo`] is rebuilt from [`FileMeta`] with
|
||||
//! empty footer/index — see [`SstFileInfo`] for field-level detail.
|
||||
//! ² Apply staging fires `on_manifest_updated` twice: once when the staging SST files are
|
||||
//! committed via `RegionEdit`, and once when `exit_staging_on_success` merges all staged
|
||||
//! manifest actions into the live manifest.
|
||||
//! ² Apply staging fires `on_manifest_updated` once when `exit_staging_on_success` promotes
|
||||
//! all staged manifest actions (including the SST file additions) into the live manifest.
|
||||
//! The intermediate staging `RegionEdit` is written to the staging directory and does not
|
||||
//! fire the hook — its file list is included in the promote notification.
|
||||
//!
|
||||
//! The following paths do **not** trigger any hook:
|
||||
//! - Follower region sync / catchup (manifest read-only; followers don't author changes)
|
||||
//! - GC / checkpoint / drop / remap (internal bookkeeping, not logical state changes)
|
||||
//! - GC / checkpoint / remap (internal bookkeeping, not logical state changes)
|
||||
//!
|
||||
//! An explicit region **drop** does fire lifecycle hooks — see
|
||||
//! [Region lifecycle](#region-lifecycle).
|
||||
//!
|
||||
//! ## Region lifecycle
|
||||
//!
|
||||
//! Beyond manifest/SST observation, the hook observes the high-level lifecycle of an
|
||||
//! active region:
|
||||
//!
|
||||
//! | Event | Method | When |
|
||||
//! |-------|--------|------|
|
||||
//! | Open | [`on_region_opened`] | A create or open request registers the region as active (the counterpart to close/drop). Does not fire for the compactor's transient regions or the catch-up reopen. |
|
||||
//! | Close | [`on_region_closed`] | A close request (or a close-after-flush) removes the region from the active set. Data files, manifest and WAL state are **preserved**; the region may be reopened. |
|
||||
//! | Logical drop | [`on_region_dropped`] | A drop request has been handled: the region leaves the active set and its WAL entries are marked obsolete. Data files are **not yet deleted**. |
|
||||
//! | Physical file removal | [`on_region_files_removed`] | The drop GC worker has deleted the region directory. Terminal file-lifecycle event. |
|
||||
//!
|
||||
//! Notes:
|
||||
//! - `on_region_closed` / `on_region_dropped` run **inline in the region worker loop**,
|
||||
//! so implementations must be fast (same contract as `on_manifest_updated`).
|
||||
//! - `on_region_opened` runs inline in the worker loop on the **create** path, but on the
|
||||
//! **open** path it fires inside the spawned open task (`common_runtime::spawn_global`),
|
||||
//! i.e. concurrently with the worker loop — after WAL replay, before the region is
|
||||
//! registered and its open request is acknowledged. Implementations must still be fast
|
||||
//! and must not assume worker-loop-thread affinity or strict ordering against concurrent
|
||||
//! requests to other regions.
|
||||
//! - `on_region_files_removed` runs on the background drop GC task, outside the worker loop.
|
||||
//! - When global GC is enabled and a normal table region is dropped with `partial_drop`, its
|
||||
//! directory is left for global reclamation and `on_region_files_removed` is **not** fired
|
||||
//! by the drop worker (observe it via the global GC path instead).
|
||||
//! - Logical file removal (compaction, region edit, truncate) is already observable via
|
||||
//! [`on_manifest_updated`] (`Edit.files_to_remove` / `Truncate` action); only the drop
|
||||
//! worker's physical directory deletion needs a dedicated file hook.
|
||||
//!
|
||||
//! ## Invocation points
|
||||
//!
|
||||
@@ -77,11 +126,18 @@
|
||||
//!
|
||||
//! ## Future work
|
||||
//!
|
||||
//! A future `on_files_removed` hook may be added to observe file lifecycle end
|
||||
//! (GC, drop, truncate, compaction removal). This is not yet implemented.
|
||||
//! `on_region_files_removed` currently covers only the **drop** GC worker's physical
|
||||
//! directory removal. A broader per-file `on_files_removed` hook covering compaction
|
||||
//! removal, truncate, and the global GC reclamation path is not yet implemented
|
||||
//! (though logical file removal is already observable via `on_manifest_updated`).
|
||||
//! Role/leadership transitions (`on_region_role_changed`) are also not hooked.
|
||||
//!
|
||||
//! [`on_sst_files_written`]: RegionHook::on_sst_files_written
|
||||
//! [`on_manifest_updated`]: RegionHook::on_manifest_updated
|
||||
//! [`on_region_opened`]: RegionHook::on_region_opened
|
||||
//! [`on_region_closed`]: RegionHook::on_region_closed
|
||||
//! [`on_region_dropped`]: RegionHook::on_region_dropped
|
||||
//! [`on_region_files_removed`]: RegionHook::on_region_files_removed
|
||||
//! [`RegionManifestManager::update`]: crate::manifest::manager::RegionManifestManager::update
|
||||
//! [`ManifestContext::update_locked`]: crate::region::ManifestContext::update_locked
|
||||
//! [`ManifestContext::update_manifest`]: crate::region::ManifestContext::update_manifest
|
||||
@@ -103,6 +159,13 @@ use crate::sst::parquet::SstInfo;
|
||||
///
|
||||
/// Must be [`fire`](Self::fire)d **after** the manifest write lock is released
|
||||
/// (the hook may read the manifest). `#[must_use]` so a forgotten receipt warns.
|
||||
///
|
||||
/// ## Staging suppression
|
||||
///
|
||||
/// When `is_staging` is `true`, [`fire`](Self::fire) is a no-op — the write went to
|
||||
/// the staging manifest directory. The hook only observes writes to the live (normal)
|
||||
/// manifest directory. Staging actions are accumulated and delivered in a single
|
||||
/// notification when `exit_staging_on_success` promotes them (`is_staging = false`).
|
||||
#[must_use = "the region hook must be fired after releasing the manifest write lock"]
|
||||
pub(crate) struct PendingManifestHook {
|
||||
region_id: RegionId,
|
||||
@@ -110,6 +173,9 @@ pub(crate) struct PendingManifestHook {
|
||||
action_list: Option<RegionMetaActionList>,
|
||||
version: ManifestVersion,
|
||||
hook: Option<RegionHookRef>,
|
||||
/// Whether the manifest write went to the staging directory.
|
||||
/// When `true`, `fire()` is suppressed — the hook only observes live manifest writes.
|
||||
is_staging: bool,
|
||||
}
|
||||
|
||||
impl PendingManifestHook {
|
||||
@@ -118,12 +184,14 @@ impl PendingManifestHook {
|
||||
action_list: Option<RegionMetaActionList>,
|
||||
version: ManifestVersion,
|
||||
hook: Option<RegionHookRef>,
|
||||
is_staging: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
region_id,
|
||||
action_list,
|
||||
version,
|
||||
hook,
|
||||
is_staging,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,9 +200,13 @@ impl PendingManifestHook {
|
||||
self.version
|
||||
}
|
||||
|
||||
/// Fires the hook if one is registered. Safe to call unconditionally: it is
|
||||
/// a no-op when no hook is registered.
|
||||
/// Fires the hook if one is registered, **unless** the write went to the staging
|
||||
/// manifest directory (`is_staging = true`). Safe to call unconditionally:
|
||||
/// it is a no-op when no hook is registered or when the write is staging-only.
|
||||
pub(crate) async fn fire(self) {
|
||||
if self.is_staging {
|
||||
return;
|
||||
}
|
||||
if let (Some(hook), Some(action_list)) = (self.hook, self.action_list) {
|
||||
hook.on_manifest_updated(self.region_id, &action_list, self.version)
|
||||
.await;
|
||||
@@ -146,6 +218,9 @@ impl PendingManifestHook {
|
||||
/// keeps `self`'s actions followed by `other`'s, and the *later* manifest
|
||||
/// version wins. Used when a sequence of writes (e.g. staging-exit followed
|
||||
/// by metadata backfill) should notify the hook exactly once.
|
||||
///
|
||||
/// `is_staging` is `true` only if **both** sides are staging; if either side
|
||||
/// is a live write, the merged result is also live.
|
||||
pub(crate) fn merge(self, other: PendingManifestHook) -> PendingManifestHook {
|
||||
debug_assert_eq!(
|
||||
self.region_id, other.region_id,
|
||||
@@ -164,6 +239,7 @@ impl PendingManifestHook {
|
||||
},
|
||||
version: self.version.max(other.version),
|
||||
hook: self.hook.or(other.hook),
|
||||
is_staging: self.is_staging && other.is_staging,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -211,10 +287,14 @@ pub trait RegionHook: Send + Sync + Debug {
|
||||
let _ = (region_id, region_metadata, files);
|
||||
}
|
||||
|
||||
/// Called after the region manifest is successfully committed.
|
||||
/// Called after the region manifest is successfully committed to the **live**
|
||||
/// (normal) manifest directory.
|
||||
///
|
||||
/// Fires for **all** manifest write paths: flush, compaction, region edit,
|
||||
/// copy region, alter, truncate, enter staging, index build, etc.
|
||||
/// Fires for: flush, compaction, region edit, copy region, alter, truncate,
|
||||
/// async index build, and apply-staging promote. Does **not** fire for writes
|
||||
/// to the staging manifest directory (enter staging, operations during staging,
|
||||
/// the intermediate apply-staging edit) — those are suppressed because their
|
||||
/// effects are accumulated and delivered in a single promote notification.
|
||||
///
|
||||
/// Does **not** fire for:
|
||||
/// - Manifest reads / follower sync (no write)
|
||||
@@ -228,6 +308,80 @@ pub trait RegionHook: Send + Sync + Debug {
|
||||
) {
|
||||
let _ = (region_id, action_list, manifest_version);
|
||||
}
|
||||
|
||||
/// Called once a region **open** or **create** succeeds, but **before** the
|
||||
/// region is registered in the engine's active set (`insert_region` runs
|
||||
/// immediately afterwards).
|
||||
///
|
||||
/// Fires once when a region becomes active via a create or open request —
|
||||
/// the natural counterpart to [`on_region_closed`] / [`on_region_dropped`].
|
||||
/// It does **not** fire for the compactor's transient compaction regions
|
||||
/// (`open_compaction_region`), nor for the internal reopen performed during
|
||||
/// follower catch-up / leadership promotion.
|
||||
///
|
||||
/// On the **create** path it runs inline in the region worker loop; on the
|
||||
/// **open** path it runs inside the spawned open task
|
||||
/// (`common_runtime::spawn_global`), concurrently with the worker loop
|
||||
/// (after WAL replay, before the region is registered/acknowledged).
|
||||
/// Implementations must be fast and must **not** assume worker-loop-thread
|
||||
/// affinity or strict ordering against concurrent requests to other regions.
|
||||
///
|
||||
/// [`on_region_closed`]: RegionHook::on_region_closed
|
||||
/// [`on_region_dropped`]: RegionHook::on_region_dropped
|
||||
async fn on_region_opened(&self, region_id: RegionId, region_metadata: &RegionMetadataRef) {
|
||||
let _ = (region_id, region_metadata);
|
||||
}
|
||||
|
||||
/// Called after a region is **closed** via a close request.
|
||||
///
|
||||
/// The region is removed from the engine's active set, but its data files,
|
||||
/// manifest, and WAL state are **preserved**; the region may be reopened
|
||||
/// later. Fires once per successful close, after the region's background
|
||||
/// tasks (flush/compaction) have been stopped.
|
||||
///
|
||||
/// Fires for a region of **any** role (leader or follower) that is closed.
|
||||
/// Does **not** fire when a region is dropped (see [`on_region_dropped`]).
|
||||
///
|
||||
/// Runs inline in the region worker loop; implementations should be fast.
|
||||
///
|
||||
/// [`on_region_dropped`]: RegionHook::on_region_dropped
|
||||
async fn on_region_closed(&self, region_id: RegionId, region_metadata: &RegionMetadataRef) {
|
||||
let _ = (region_id, region_metadata);
|
||||
}
|
||||
|
||||
/// Called after a region is **logically dropped** (a drop request has been
|
||||
/// handled).
|
||||
///
|
||||
/// The region is removed from the active set and its WAL entries are marked
|
||||
/// obsolete. Its data files are **not yet deleted** — they are scheduled for
|
||||
/// asynchronous removal by the GC worker. Observe physical deletion via
|
||||
/// [`on_region_files_removed`].
|
||||
///
|
||||
/// Runs inline in the region worker loop; implementations should be fast.
|
||||
///
|
||||
/// [`on_region_files_removed`]: RegionHook::on_region_files_removed
|
||||
async fn on_region_dropped(&self, region_id: RegionId, region_metadata: &RegionMetadataRef) {
|
||||
let _ = (region_id, region_metadata);
|
||||
}
|
||||
|
||||
/// Called after a dropped region's data files are **physically removed** by
|
||||
/// the drop GC worker (the region directory has been deleted).
|
||||
///
|
||||
/// This is the terminal event in a region's file lifecycle; no further
|
||||
/// callbacks fire for this region id afterwards. Fires only when the drop
|
||||
/// worker itself deletes the directory. When global GC is enabled and the
|
||||
/// region is a normal table region dropped with `partial_drop`, the
|
||||
/// directory is left for global reclamation and this hook is **not** fired
|
||||
/// by the drop worker.
|
||||
///
|
||||
/// Runs on a background task, outside the region worker loop.
|
||||
async fn on_region_files_removed(
|
||||
&self,
|
||||
region_id: RegionId,
|
||||
region_metadata: &RegionMetadataRef,
|
||||
) {
|
||||
let _ = (region_id, region_metadata);
|
||||
}
|
||||
}
|
||||
|
||||
pub type RegionHookRef = Arc<dyn RegionHook>;
|
||||
|
||||
@@ -1236,6 +1236,7 @@ impl ManifestContext {
|
||||
action_list_for_hook,
|
||||
version,
|
||||
self.hook.clone(),
|
||||
is_staging,
|
||||
))
|
||||
}
|
||||
|
||||
|
||||
@@ -89,5 +89,13 @@ impl<S: LogStore> RegionWorkerLoop<S> {
|
||||
// clean index build status.
|
||||
self.index_build_scheduler.on_region_closed(region_id).await;
|
||||
self.region_count.dec();
|
||||
|
||||
// Notify the region hook that the region has been closed. The region is
|
||||
// fully stopped and unregistered, but its files/manifest are preserved.
|
||||
// Runs inline; the hook contract requires it to be fast.
|
||||
if let Some(hook) = region.manifest_ctx.hook() {
|
||||
let metadata = region.metadata();
|
||||
hook.on_region_closed(region_id, &metadata).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,6 +86,12 @@ impl<S: LogStore> RegionWorkerLoop<S> {
|
||||
|
||||
self.region_count.inc();
|
||||
|
||||
// Notify the region hook that the region has been opened (created).
|
||||
// Fires before registration; allocates nothing when no hook is registered.
|
||||
if let Some(hook) = region.manifest_ctx.hook() {
|
||||
hook.on_region_opened(region_id, ®ion.metadata()).await;
|
||||
}
|
||||
|
||||
// Insert the MitoRegion into the RegionMap.
|
||||
self.regions.insert_region(region);
|
||||
|
||||
|
||||
@@ -23,10 +23,12 @@ use object_store::util::join_path;
|
||||
use object_store::{EntryMode, ObjectStore};
|
||||
use snafu::ResultExt;
|
||||
use store_api::logstore::LogStore;
|
||||
use store_api::metadata::RegionMetadataRef;
|
||||
use store_api::region_request::{AffectedRows, PathType};
|
||||
use store_api::storage::RegionId;
|
||||
use tokio::time::sleep;
|
||||
|
||||
use crate::engine::region_hook::RegionHookRef;
|
||||
use crate::error::{OpenDalSnafu, Result};
|
||||
use crate::region::{RegionLeaderState, RegionMapRef};
|
||||
use crate::worker::{DROPPING_MARKER_FILE, RegionWorkerLoop};
|
||||
@@ -106,6 +108,22 @@ where
|
||||
|
||||
self.region_count.dec();
|
||||
|
||||
// Notify registered hooks that the region has been logically dropped, and
|
||||
// prepare a payload for the background GC task to fire the terminal
|
||||
// `on_region_files_removed`. When no hook is registered this allocates
|
||||
// nothing — no metadata snapshot is taken and no payload is built.
|
||||
let hook_payload = match region.manifest_ctx.hook() {
|
||||
Some(hook) => {
|
||||
let region_metadata = region.metadata();
|
||||
hook.on_region_dropped(region_id, ®ion_metadata).await;
|
||||
Some(DropHookPayload {
|
||||
hook,
|
||||
metadata: region_metadata,
|
||||
})
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
let object_store = region.access_layer.object_store().clone();
|
||||
let dropping_regions = self.dropping_regions.clone();
|
||||
let listener = self.listener.clone();
|
||||
@@ -122,6 +140,7 @@ where
|
||||
object_store,
|
||||
dropping_regions,
|
||||
partial_drop,
|
||||
hook_payload.as_ref(),
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
@@ -135,6 +154,7 @@ where
|
||||
object_store,
|
||||
dropping_regions,
|
||||
gc_duration,
|
||||
hook_payload.as_ref(),
|
||||
)
|
||||
.await
|
||||
};
|
||||
@@ -156,6 +176,19 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Carries the region hook and region metadata into the background GC task so
|
||||
/// it can fire [`RegionHook::on_region_files_removed`] once the dropped region's
|
||||
/// directory is physically deleted.
|
||||
///
|
||||
/// Only constructed when a hook is registered; the task receives it as
|
||||
/// `Option<&DropHookPayload>` so the no-hook path allocates nothing.
|
||||
///
|
||||
/// [`RegionHook::on_region_files_removed`]: crate::engine::region_hook::RegionHook::on_region_files_removed
|
||||
struct DropHookPayload {
|
||||
hook: RegionHookRef,
|
||||
metadata: RegionMetadataRef,
|
||||
}
|
||||
|
||||
/// Background GC task to remove the entire region path once one of the following
|
||||
/// conditions is true:
|
||||
/// - It finds there is no parquet file left.
|
||||
@@ -172,6 +205,7 @@ async fn later_drop_task_without_global_gc(
|
||||
object_store: ObjectStore,
|
||||
dropping_regions: RegionMapRef,
|
||||
gc_duration: Duration,
|
||||
hook_payload: Option<&DropHookPayload>,
|
||||
) -> bool {
|
||||
remove_region_with_retry(
|
||||
region_id,
|
||||
@@ -180,6 +214,7 @@ async fn later_drop_task_without_global_gc(
|
||||
dropping_regions,
|
||||
Some(gc_duration),
|
||||
false,
|
||||
hook_payload,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -191,6 +226,7 @@ async fn remove_region_with_retry(
|
||||
dropping_regions: std::sync::Arc<crate::region::RegionMap>,
|
||||
gc_duration: Option<Duration>,
|
||||
mut force: bool,
|
||||
hook_payload: Option<&DropHookPayload>,
|
||||
) -> bool {
|
||||
for _ in 0..MAX_RETRY_TIMES {
|
||||
let result = remove_region_dir_once(®ion_path, &object_store, force).await;
|
||||
@@ -204,6 +240,15 @@ async fn remove_region_with_retry(
|
||||
Ok(true) => {
|
||||
dropping_regions.remove_region(region_id);
|
||||
info!("Region {} is dropped, force: {}", region_path, force);
|
||||
// The region directory has been physically deleted; fire the
|
||||
// terminal file-lifecycle event. The partial-drop/global-GC path
|
||||
// never reaches here (it does not delete the directory itself).
|
||||
if let Some(hook_payload) = hook_payload {
|
||||
hook_payload
|
||||
.hook
|
||||
.on_region_files_removed(region_id, &hook_payload.metadata)
|
||||
.await;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
Ok(false) => (),
|
||||
@@ -230,6 +275,7 @@ async fn later_drop_task_with_global_gc(
|
||||
object_store: ObjectStore,
|
||||
dropping_regions: RegionMapRef,
|
||||
partial_drop: bool,
|
||||
hook_payload: Option<&DropHookPayload>,
|
||||
) -> bool {
|
||||
// For metadata regions or regions marked for full deletion (such as when dropping a table)
|
||||
// the region directory is forcefully removed immediately.
|
||||
@@ -243,6 +289,7 @@ async fn later_drop_task_with_global_gc(
|
||||
dropping_regions,
|
||||
None,
|
||||
true,
|
||||
hook_payload,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
|
||||
@@ -149,6 +149,13 @@ impl<S: LogStore> RegionWorkerLoop<S> {
|
||||
);
|
||||
region_count.inc();
|
||||
|
||||
// Notify the region hook that the region has been opened.
|
||||
// Fires before registration; allocates nothing when no hook
|
||||
// is registered.
|
||||
if let Some(hook) = region.manifest_ctx.hook() {
|
||||
hook.on_region_opened(region_id, ®ion.metadata()).await;
|
||||
}
|
||||
|
||||
// Insert the Region into the RegionMap.
|
||||
regions.insert_region(region);
|
||||
|
||||
|
||||
+30
-19
@@ -52,45 +52,37 @@ pub enum QueryStatement {
|
||||
}
|
||||
|
||||
impl QueryStatement {
|
||||
pub fn post_process(&self, params: HashMap<String, String>) -> Result<QueryStatement> {
|
||||
pub fn post_process(self, params: HashMap<String, String>) -> Result<QueryStatement> {
|
||||
match self {
|
||||
QueryStatement::Sql(_) => UnimplementedSnafu {
|
||||
operation: "sql post process",
|
||||
}
|
||||
.fail(),
|
||||
QueryStatement::Promql(eval_stmt, alias) => {
|
||||
QueryStatement::Promql(mut eval_stmt, alias) => {
|
||||
let node_name = match params.get("name") {
|
||||
Some(name) => name.as_str(),
|
||||
None => "",
|
||||
};
|
||||
let extension_node = Self::create_extension_node(node_name, &eval_stmt.expr);
|
||||
Ok(QueryStatement::Promql(
|
||||
EvalStmt {
|
||||
expr: Extension(extension_node.unwrap()),
|
||||
start: eval_stmt.start,
|
||||
end: eval_stmt.end,
|
||||
interval: eval_stmt.interval,
|
||||
lookback_delta: eval_stmt.lookback_delta,
|
||||
},
|
||||
alias.clone(),
|
||||
))
|
||||
let extension_node = Self::create_extension_node(node_name, eval_stmt.expr);
|
||||
eval_stmt.expr = Extension(extension_node.unwrap());
|
||||
Ok(QueryStatement::Promql(eval_stmt, alias))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn create_extension_node(node_name: &str, expr: &Expr) -> Option<NodeExtension> {
|
||||
fn create_extension_node(node_name: &str, expr: Expr) -> Option<NodeExtension> {
|
||||
match node_name {
|
||||
ANALYZE_NODE_NAME => Some(NodeExtension {
|
||||
expr: Arc::new(AnalyzeExpr { expr: expr.clone() }),
|
||||
expr: Arc::new(AnalyzeExpr { expr }),
|
||||
}),
|
||||
ANALYZE_VERBOSE_NODE_NAME => Some(NodeExtension {
|
||||
expr: Arc::new(AnalyzeVerboseExpr { expr: expr.clone() }),
|
||||
expr: Arc::new(AnalyzeVerboseExpr { expr }),
|
||||
}),
|
||||
EXPLAIN_NODE_NAME => Some(NodeExtension {
|
||||
expr: Arc::new(ExplainExpr { expr: expr.clone() }),
|
||||
expr: Arc::new(ExplainExpr { expr }),
|
||||
}),
|
||||
EXPLAIN_VERBOSE_NODE_NAME => Some(NodeExtension {
|
||||
expr: Arc::new(ExplainVerboseExpr { expr: expr.clone() }),
|
||||
expr: Arc::new(ExplainVerboseExpr { expr }),
|
||||
}),
|
||||
_ => None,
|
||||
}
|
||||
@@ -204,9 +196,10 @@ impl QueryLanguageParser {
|
||||
}
|
||||
|
||||
pub(crate) fn apply_alias_extension(mut eval_stmt: EvalStmt, alias: &str) -> EvalStmt {
|
||||
let expr = eval_stmt.expr;
|
||||
eval_stmt.expr = Extension(NodeExtension {
|
||||
expr: Arc::new(AliasExpr {
|
||||
expr: eval_stmt.expr.clone(),
|
||||
expr,
|
||||
alias: alias.to_string(),
|
||||
}),
|
||||
});
|
||||
@@ -263,6 +256,14 @@ macro_rules! define_node_ast_extension {
|
||||
fn children(&self) -> &[Expr] {
|
||||
std::slice::from_ref(&self.expr)
|
||||
}
|
||||
|
||||
fn with_new_children(&self, children: Vec<Expr>) -> Arc<dyn ExtensionExpr> {
|
||||
let mut iter = children.into_iter();
|
||||
match (iter.next(), iter.next()) {
|
||||
(Some(expr), None) => Arc::new($name_expr { expr }),
|
||||
_ => Arc::new(self.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(rustdoc::broken_intra_doc_links)]
|
||||
@@ -313,6 +314,16 @@ impl ExtensionExpr for AliasExpr {
|
||||
fn children(&self) -> &[Expr] {
|
||||
std::slice::from_ref(&self.expr)
|
||||
}
|
||||
fn with_new_children(&self, children: Vec<Expr>) -> Arc<dyn ExtensionExpr> {
|
||||
let mut iter = children.into_iter();
|
||||
match (iter.next(), iter.next()) {
|
||||
(Some(expr), None) => Arc::new(Self {
|
||||
expr,
|
||||
alias: self.alias.clone(),
|
||||
}),
|
||||
_ => Arc::new(self.clone()),
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Alias {
|
||||
|
||||
@@ -229,6 +229,8 @@ impl IslandExpr {
|
||||
!modifier.return_bool
|
||||
&& modifier.matching.is_none()
|
||||
&& matches!(modifier.card, VectorMatchCardinality::OneToOne)
|
||||
&& modifier.fill_values.lhs.is_none()
|
||||
&& modifier.fill_values.rhs.is_none()
|
||||
}) =>
|
||||
{
|
||||
let lhs = Self::try_new(lhs, env)?;
|
||||
@@ -1093,6 +1095,18 @@ impl PromPlanner {
|
||||
query_engine_state: &QueryEngineState,
|
||||
binary_expr: &PromBinaryExpr,
|
||||
) -> Result<LogicalPlan> {
|
||||
// promql-parser accepts fill modifiers, but Greptime does not implement the
|
||||
// required outer joins and missing-value substitution. Reject them before the
|
||||
// binary-island fast path so they cannot silently behave like normal inner joins.
|
||||
if let Some(modifier) = &binary_expr.modifier {
|
||||
ensure!(
|
||||
modifier.fill_values.lhs.is_none() && modifier.fill_values.rhs.is_none(),
|
||||
UnsupportedExprSnafu {
|
||||
name: "PromQL fill modifiers"
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(plan) = self.try_plan_binary_island(binary_expr).await? {
|
||||
return Ok(plan);
|
||||
}
|
||||
@@ -5542,6 +5556,33 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reject_binary_fill_modifiers() {
|
||||
let state = build_query_engine_state();
|
||||
|
||||
for query in [
|
||||
"some_metric + fill(0) some_alt_metric",
|
||||
"some_metric + fill_left(0) some_alt_metric",
|
||||
"some_metric + fill_right(0) some_alt_metric",
|
||||
"(some_metric + fill(0) some_alt_metric) + some_metric",
|
||||
] {
|
||||
let eval_stmt = build_eval_stmt(query);
|
||||
let table_provider = build_test_table_provider(&[], 0, 0).await;
|
||||
let err = PromPlanner::stmt_to_plan(table_provider, &eval_stmt, &state)
|
||||
.await
|
||||
.unwrap_err();
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
&err,
|
||||
crate::promql::error::Error::UnsupportedExpr { name, .. }
|
||||
if name == "PromQL fill modifiers"
|
||||
),
|
||||
"{err}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timestamp_binary_join_falls_back_when_tsid_is_projected_out() {
|
||||
for query in [
|
||||
|
||||
@@ -1098,6 +1098,10 @@ async fn test_execute_copy_from_csv_strict_headers(instance: Arc<dyn MockInstanc
|
||||
let unknown_path = tmp_dir.path().join("unknown.csv");
|
||||
let missing_path = tmp_dir.path().join("missing.csv");
|
||||
let duplicate_path = tmp_dir.path().join("duplicate.csv");
|
||||
let matching_location = prepare_path(matching_path.to_str().unwrap());
|
||||
let unknown_location = prepare_path(unknown_path.to_str().unwrap());
|
||||
let missing_location = prepare_path(missing_path.to_str().unwrap());
|
||||
let duplicate_location = prepare_path(duplicate_path.to_str().unwrap());
|
||||
std::fs::write(
|
||||
&matching_path,
|
||||
"ts,host_id,reading_value\n2024-01-01T00:00:00,1,10.5\n",
|
||||
@@ -1127,7 +1131,7 @@ async fn test_execute_copy_from_csv_strict_headers(instance: Arc<dyn MockInstanc
|
||||
&instance,
|
||||
&format!(
|
||||
"COPY csv_strict_headers FROM '{}' WITH (FORMAT='csv', STRICT_HEADERS='true');",
|
||||
matching_path.display()
|
||||
matching_location
|
||||
),
|
||||
)
|
||||
.await
|
||||
@@ -1149,7 +1153,7 @@ async fn test_execute_copy_from_csv_strict_headers(instance: Arc<dyn MockInstanc
|
||||
&instance,
|
||||
&format!(
|
||||
"COPY csv_strict_headers FROM '{}' WITH (FORMAT='csv', STRICT_HEADERS='true');",
|
||||
unknown_path.display()
|
||||
unknown_location
|
||||
),
|
||||
)
|
||||
.await
|
||||
@@ -1173,7 +1177,7 @@ async fn test_execute_copy_from_csv_strict_headers(instance: Arc<dyn MockInstanc
|
||||
&instance,
|
||||
&format!(
|
||||
"COPY csv_strict_headers FROM '{}' WITH (FORMAT='csv', STRICT_HEADERS='true');",
|
||||
missing_path.display()
|
||||
missing_location
|
||||
),
|
||||
)
|
||||
.await
|
||||
@@ -1197,7 +1201,7 @@ async fn test_execute_copy_from_csv_strict_headers(instance: Arc<dyn MockInstanc
|
||||
&instance,
|
||||
&format!(
|
||||
"COPY csv_strict_headers FROM '{}' WITH (FORMAT='csv', STRICT_HEADERS='true');",
|
||||
duplicate_path.display()
|
||||
duplicate_location
|
||||
),
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -62,16 +62,6 @@ tql eval (10, 10, '1s') double_exponential_smoothing(prom_series[10s], 0.4 + 0.1
|
||||
| 1970-01-01T00:00:10 | 47.0806953125 | p |
|
||||
+---------------------+------------------------------------------------------------------------------------------+------+
|
||||
|
||||
-- holt_winters (backward compatibility)
|
||||
-- SQLNESS SORT_RESULT 3 1
|
||||
tql eval (10, 10, '1s') holt_winters(prom_series[10s], 0.4 + 0.1, 0.1);
|
||||
|
||||
+---------------------+------------------------------------------------------------------------------------------+------+
|
||||
| ts | prom_double_exponential_smoothing(ts_range,val,Float64(0.4) + Float64(0.1),Float64(0.1)) | host |
|
||||
+---------------------+------------------------------------------------------------------------------------------+------+
|
||||
| 1970-01-01T00:00:10 | 47.0806953125 | p |
|
||||
+---------------------+------------------------------------------------------------------------------------------+------+
|
||||
|
||||
DROP TABLE prom_series;
|
||||
|
||||
Affected Rows: 0
|
||||
|
||||
@@ -34,10 +34,6 @@ tql eval (3, 3, '1s') predict_linear(prom_series[3s], 40 + 2);
|
||||
-- SQLNESS SORT_RESULT 3 1
|
||||
tql eval (10, 10, '1s') double_exponential_smoothing(prom_series[10s], 0.4 + 0.1, 0.1);
|
||||
|
||||
-- holt_winters (backward compatibility)
|
||||
-- SQLNESS SORT_RESULT 3 1
|
||||
tql eval (10, 10, '1s') holt_winters(prom_series[10s], 0.4 + 0.1, 0.1);
|
||||
|
||||
DROP TABLE prom_series;
|
||||
|
||||
CREATE TABLE
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
name = "legacy_json2_variant_payload"
|
||||
reason = "Verify current binaries can read JSON2 variant payloads written by old binaries after the JSON2 encoding change."
|
||||
introduced_by = "PR #8435"
|
||||
topologies = ["distributed"]
|
||||
from_range = ["=v1.1.0", "=v1.1.1", "=v1.1.2"]
|
||||
to_range = [">v1.1.1"]
|
||||
features = ["json", "json2", "table", "read"]
|
||||
owner = "query"
|
||||
@@ -0,0 +1,29 @@
|
||||
CREATE TABLE t_legacy_json2_variant_payload (
|
||||
ts TIMESTAMP TIME INDEX,
|
||||
j JSON2
|
||||
) WITH (
|
||||
'append_mode' = 'true'
|
||||
);
|
||||
|
||||
INSERT INTO t_legacy_json2_variant_payload (ts, j) VALUES
|
||||
('2026-07-08 00:00:00+0000', '{"a": {"b": 1}, "c": "s1", "d": [{"e": {"f": 0.1}}]}'),
|
||||
('2026-07-08 00:01:00+0000', '{"a": {"b": -2}, "c": [1], "d": [{"e": {"f": 0.2}}]}'),
|
||||
('2026-07-08 00:02:00+0000', '{"a": {"b": "s3"}, "c": "s3", "d": [{"e": {"g": -0.3}}]}'),
|
||||
('2026-07-08 00:03:00+0000', '{"a": {"b": null}, "c": null}'),
|
||||
('2026-07-08 00:04:00+0000', '{"c": "s5", "d": {"e": true}}'),
|
||||
('2026-07-08 00:05:00+0000', '{"a": {"b": 6}}');
|
||||
|
||||
ADMIN FLUSH_TABLE('t_legacy_json2_variant_payload');
|
||||
|
||||
CREATE TABLE t_legacy_json2_variant_payload_object_d (
|
||||
ts TIMESTAMP TIME INDEX,
|
||||
j JSON2
|
||||
) WITH (
|
||||
'append_mode' = 'true'
|
||||
);
|
||||
|
||||
INSERT INTO t_legacy_json2_variant_payload_object_d (ts, j) VALUES
|
||||
('2026-07-08 01:00:00+0000', '{"d": {"e": true}}'),
|
||||
('2026-07-08 01:01:00+0000', '{"d": {"e": false}}');
|
||||
|
||||
ADMIN FLUSH_TABLE('t_legacy_json2_variant_payload_object_d');
|
||||
@@ -0,0 +1,60 @@
|
||||
SELECT ts, j.a.b AS a_b, j.c AS c
|
||||
FROM t_legacy_json2_variant_payload
|
||||
ORDER BY ts;
|
||||
|
||||
+---------------------+-----+-----+
|
||||
| ts | a_b | c |
|
||||
+---------------------+-----+-----+
|
||||
| 2026-07-08T00:00:00 | 1 | s1 |
|
||||
| 2026-07-08T00:01:00 | -2 | [1] |
|
||||
| 2026-07-08T00:02:00 | s3 | s3 |
|
||||
| 2026-07-08T00:03:00 | | |
|
||||
| 2026-07-08T00:04:00 | | s5 |
|
||||
| 2026-07-08T00:05:00 | 6 | |
|
||||
+---------------------+-----+-----+
|
||||
|
||||
SELECT
|
||||
ts,
|
||||
j.d.e AS d_e
|
||||
FROM t_legacy_json2_variant_payload_object_d
|
||||
ORDER BY ts;
|
||||
|
||||
+---------------------+-------+
|
||||
| ts | d_e |
|
||||
+---------------------+-------+
|
||||
| 2026-07-08T01:00:00 | true |
|
||||
| 2026-07-08T01:01:00 | false |
|
||||
+---------------------+-------+
|
||||
|
||||
-- JSON2 projection rendering is a user-visible compatibility contract.
|
||||
SELECT ts, j.d AS d
|
||||
FROM t_legacy_json2_variant_payload
|
||||
ORDER BY ts;
|
||||
|
||||
+---------------------+--------------------+
|
||||
| ts | d |
|
||||
+---------------------+--------------------+
|
||||
| 2026-07-08T00:00:00 | [{"e":{"f":0.1}}] |
|
||||
| 2026-07-08T00:01:00 | [{"e":{"f":0.2}}] |
|
||||
| 2026-07-08T00:02:00 | [{"e":{"g":-0.3}}] |
|
||||
| 2026-07-08T00:03:00 | |
|
||||
| 2026-07-08T00:04:00 | {"e":true} |
|
||||
| 2026-07-08T00:05:00 | |
|
||||
+---------------------+--------------------+
|
||||
|
||||
SHOW CREATE TABLE t_legacy_json2_variant_payload;
|
||||
|
||||
+--------------------------------+---------------------------------------------------------------+
|
||||
| Table | Create Table |
|
||||
+--------------------------------+---------------------------------------------------------------+
|
||||
| t_legacy_json2_variant_payload | CREATE TABLE IF NOT EXISTS "t_legacy_json2_variant_payload" ( |
|
||||
| | "ts" TIMESTAMP(3) NOT NULL, |
|
||||
| | "j" JSON2 NULL, |
|
||||
| | TIME INDEX ("ts") |
|
||||
| | ) |
|
||||
| | |
|
||||
| | ENGINE=mito |
|
||||
| | WITH( |
|
||||
| | append_mode = 'true' |
|
||||
| | ) |
|
||||
+--------------------------------+---------------------------------------------------------------+
|
||||
@@ -0,0 +1,16 @@
|
||||
SELECT ts, j.a.b AS a_b, j.c AS c
|
||||
FROM t_legacy_json2_variant_payload
|
||||
ORDER BY ts;
|
||||
|
||||
SELECT
|
||||
ts,
|
||||
j.d.e AS d_e
|
||||
FROM t_legacy_json2_variant_payload_object_d
|
||||
ORDER BY ts;
|
||||
|
||||
-- JSON2 projection rendering is a user-visible compatibility contract.
|
||||
SELECT ts, j.d AS d
|
||||
FROM t_legacy_json2_variant_payload
|
||||
ORDER BY ts;
|
||||
|
||||
SHOW CREATE TABLE t_legacy_json2_variant_payload;
|
||||
@@ -11,14 +11,14 @@
|
||||
docs together.
|
||||
- `[case]` is report metadata only. `[scenario]` is the executable regression
|
||||
configuration and must include `kind`, data layout, tables, queries, and
|
||||
thresholds. Keep scenario parsing explicit in both
|
||||
`tests/perf/query_regression_runner.py` and
|
||||
`src/cmd/src/bin/query_perf_fixture.rs`.
|
||||
thresholds. Rust owns case schema, defaults, validation, and normalized plan
|
||||
output through `query_perf_fixture plan`; `tests/perf/query_regression_runner.py`
|
||||
should only orchestrate from that normalized JSON.
|
||||
- Keep the direct-SST generator generic. Issue-specific behavior belongs in case
|
||||
files and thresholds, not in Rust generator logic.
|
||||
- Before pushing perf harness changes, run at least:
|
||||
- `uv run --no-project python -m py_compile .github/scripts/query-regression-run.py .github/scripts/query-regression-summary.py .github/scripts/query-regression-pr-metadata.py tests/perf/query_regression_runner.py`
|
||||
- `cargo fmt --all -- --check`
|
||||
- `cargo build -p cmd --bin query_perf_fixture`
|
||||
- `cargo build -p cmd --bin query_perf_fixture --features dev-tools`
|
||||
- dry-run the Python runner and Rust fixture generator against all built-in
|
||||
cases when the DSL or workflow case selection changes.
|
||||
|
||||
+144
-22
@@ -4,6 +4,11 @@ This directory is for query performance cases that compare a base build with a
|
||||
candidate build. It is not a replacement for sqlness: the goal is to measure the
|
||||
effect of optimizer/query-engine changes on realistic scan work.
|
||||
|
||||
The Prometheus remote-write cases are end-to-end regression coverage for the
|
||||
automatic write, flush, SST, and query path. They do not replace controlled
|
||||
encoding experiments that explicitly compare `plain`/dictionary,
|
||||
`no_dictionary`, BYTE_STREAM_SPLIT, or Auto policies.
|
||||
|
||||
## Phase 1: direct readable SST fixtures
|
||||
|
||||
Phase 1 should generate data by writing readable Mito SST files and matching
|
||||
@@ -40,8 +45,10 @@ The runner also supports `scenario.kind = "prom_remote_write_then_query"` for a
|
||||
bounded write-path smoke/regression flow. This path is explicit and separate from
|
||||
the direct-SST fixture path: it starts the base and candidate distributed clusters,
|
||||
writes deterministic Prometheus remote-write v1 samples through
|
||||
`/v1/prometheus/write`, flushes the configured physical metric table, then runs
|
||||
the configured SQL/TQL queries against the logical metric table.
|
||||
`/v1/prometheus/write`, flushes the configured physical metric table, checks
|
||||
visibility, runs the configured SQL/TQL queries, stops and awaits the datanode,
|
||||
then optionally inspects the landed SST/Parquet footer, encoding, and size and
|
||||
runs the read-bench against the quiescent data directory.
|
||||
|
||||
Remote-write cases configure one database, one logical metric, and one physical
|
||||
table under `[scenario.remote_write]`:
|
||||
@@ -52,34 +59,121 @@ kind = "prom_remote_write_then_query"
|
||||
|
||||
[scenario.remote_write]
|
||||
database = "public"
|
||||
metric = "prom_remote_write_smoke"
|
||||
metric = "prom_remote_write_seeded_random"
|
||||
physical_table = "greptime_physical_table"
|
||||
series_count = 8
|
||||
samples_per_series = 30
|
||||
series_count = 512
|
||||
samples_per_series = 1440
|
||||
sample_chunk_size = 480
|
||||
flush_every_sample_chunks = 1
|
||||
start_unix_millis = 1_704_067_200_000
|
||||
step_millis = 15_000
|
||||
chunk_series_count = 4
|
||||
visibility_timeout_seconds = 30
|
||||
step_millis = 60_000
|
||||
chunk_series_count = 128
|
||||
timeout_seconds = 180
|
||||
visibility_timeout_seconds = 120
|
||||
# Optional: split by time so each helper invocation writes only this many samples
|
||||
# per series, then periodically flush the physical metric table to produce
|
||||
# multiple time-interval SSTs.
|
||||
# sample_chunk_size = 300
|
||||
# flush_every_sample_chunks = 1
|
||||
|
||||
[scenario.remote_write.prom_store]
|
||||
pending_rows_flush_interval = "1s"
|
||||
max_batch_rows = 100000
|
||||
```
|
||||
|
||||
Cases can optionally add `[scenario.remote_write.value]` to control the generated
|
||||
sample values without changing label cardinality or the runner lifecycle:
|
||||
|
||||
```toml
|
||||
[scenario.remote_write.value]
|
||||
pattern = "quantized_signal" # linear, constant, modulo, unique, seeded_random,
|
||||
# run_length, quantized_signal,
|
||||
# signal_with_sporadic_stalls, mixed_signal_repeated
|
||||
base = 0.0
|
||||
step = 0.125
|
||||
cardinality = 4096 # buckets for modulo/seeded_random/quantized_signal/run_length
|
||||
seed = 12345 # deterministic seeded_random input
|
||||
run_length = 8 # adjacent samples per bucket for run_length/quantized_signal
|
||||
stall_every = 100 # interval for signal_with_sporadic_stalls
|
||||
stall_length = 16 # held samples inside each stall interval
|
||||
mixed_every = 5 # every Nth sample becomes the repeated base value
|
||||
```
|
||||
|
||||
The default `linear` pattern preserves the helper's historical formula. Use
|
||||
`constant` or low-cardinality `modulo`/`seeded_random` values for repeated-value
|
||||
data shapes, `run_length` for run-heavy low-cardinality series, `quantized_signal`
|
||||
for signal-like values collapsed into a finite bucket set, `signal_with_sporadic_stalls`
|
||||
for mostly continuous signals with periodic flat spots, and `mixed_signal_repeated`
|
||||
for signal-plus-periodic-default mixtures. `unique` or high-cardinality buckets
|
||||
still work for broad sample-value distributions. This is a generic sample-value
|
||||
control for query/ingestion cases; it does not inspect or assert storage
|
||||
encoding, Parquet footers, or storage policy choices. For chunked remote-write
|
||||
ingestion, the runner passes the sample offset and total sample count to the
|
||||
helper so non-linear value patterns use a stable global/per-series ordinal across
|
||||
chunks.
|
||||
|
||||
Case schema, value distribution defaults, storage defaults, and read-bench
|
||||
defaults are owned by Rust. The Python runner calls
|
||||
`query_perf_fixture plan --case <case.toml>` and orchestrates the normalized JSON.
|
||||
The same helper exposes `direct-sst`, `prom-remote-write`, and `inspect-footer`
|
||||
subcommands; the old direct invocation (`query_perf_fixture --case ... --out-dir
|
||||
...`) remains compatible.
|
||||
|
||||
Remote-write cases that need to validate storage output can add
|
||||
`[scenario.remote_write.storage]`. When present, the scenario body becomes:
|
||||
remote-write → flush → visibility check → query measurement → stop and await
|
||||
datanode → Parquet footer/size/encoding inspection → optional read-bench. Footer
|
||||
inspection is handled by `query_perf_fixture inspect-footer`, which reads local
|
||||
Parquet footers directly; it does not require Python `pyarrow`.
|
||||
|
||||
```toml
|
||||
[scenario.remote_write.storage]
|
||||
inspect = true
|
||||
column = "greptime_value"
|
||||
include_metadata_files = false
|
||||
# Optional: inspect below datanode data home instead of the whole fresh data dir.
|
||||
# root_suffix = "greptime/public/<table_id>"
|
||||
min_files = 1
|
||||
min_files_with_column = 1
|
||||
require_encodings = ["BYTE_STREAM_SPLIT"]
|
||||
forbid_encodings = ["PLAIN_DICTIONARY"]
|
||||
max_total_file_size_bytes = 104857600
|
||||
max_column_compressed_size_bytes = 52428800
|
||||
max_column_uncompressed_size_bytes = 209715200
|
||||
max_candidate_total_file_size_regression_pct = 10.0
|
||||
max_candidate_column_compressed_size_regression_pct = 10.0
|
||||
max_candidate_column_uncompressed_size_regression_pct = 10.0
|
||||
```
|
||||
|
||||
The `*_pct` storage thresholds are percent regression limits comparing candidate
|
||||
against base; `pct` means percent, not percentile. Per-target storage checks
|
||||
(`min_files`, `min_files_with_column`, required/forbidden encodings, and absolute
|
||||
`max_*_bytes`) run for both base and candidate. Comparative
|
||||
`max_candidate_*_regression_pct` checks remain base-vs-candidate. These checks
|
||||
are generic footer and byte-size assertions and do not encode product-specific
|
||||
heuristics. When storage inspection is enabled, `min_files` and
|
||||
`min_files_with_column` default to `1` even if omitted, so dry-runs show these
|
||||
planned checks and an empty flush or missing target column fails the scenario.
|
||||
By default the inspector root is the target datanode data home, which is suitable
|
||||
for a fresh single-case data directory. For reused or more complex data homes, set
|
||||
`root_suffix` to a path relative to the datanode data home to narrow inspection.
|
||||
|
||||
When storage inspection is enabled, `[scenario.remote_write.read_bench]` defaults
|
||||
to enabled with both datanode `parquetbench` and `scanbench`. Disable it with
|
||||
`enabled = false`. `parquetbench` measures per-SST reader cost, `scanbench`
|
||||
measures region scan cost, and query measurements still exercise the SQL/TQL
|
||||
frontend path. Treat all performance conclusions as release-only; debug builds
|
||||
are suitable only for command wiring and correctness checks.
|
||||
|
||||
The runner creates the configured database if needed, writes a per-target
|
||||
frontend config enabling `[prom_store]` with metric engine storage and a non-zero
|
||||
`pending_rows_flush_interval`, and validates that the logical metric table reaches
|
||||
`series_count * samples_per_series` rows before trusting the query measurements.
|
||||
Use `--remote-write-generator /path/to/prom_remote_write_fixture` to provide the
|
||||
Rust payload helper. `--fixture-only` is rejected for remote-write cases; use
|
||||
`--dry-run` for planning.
|
||||
Use `--fixture-generator /path/to/query_perf_fixture` to provide the Rust helper.
|
||||
Deprecated `--remote-write-generator` and `--storage-inspector` options are kept
|
||||
only for CLI compatibility. `--fixture-only` is rejected for remote-write cases;
|
||||
use `--dry-run` for planning.
|
||||
|
||||
Large manual remote-write cases can set `sample_chunk_size` to split ingestion by
|
||||
time. For each chunk, the runner invokes `prom_remote_write_fixture` with the
|
||||
time. For each chunk, the runner invokes `query_perf_fixture prom-remote-write` with the
|
||||
same series cardinality but a shorter `--samples-per-series` and an advanced
|
||||
`--start-unix-millis`. `flush_every_sample_chunks` controls periodic
|
||||
`ADMIN FLUSH_TABLE('<physical_table>')` calls; with `flush_every_sample_chunks = 1`,
|
||||
@@ -89,6 +183,25 @@ need multi-window file distribution should span multiple compaction windows. If
|
||||
`sample_chunk_size` is omitted, the runner keeps the older single-helper-invocation
|
||||
behavior and flushes once at the end.
|
||||
|
||||
The default remote-write coverage set contains four cases with the same 2048
|
||||
series × 14,400 samples (29,491,200 rows per target) shape. Each writes ten
|
||||
one-day chunks, flushes every chunk, requires at least two visible SSTs with the
|
||||
value column, runs seven read-bench iterations with `parquetbench` capped at
|
||||
four SSTs while `scanbench` covers the landed region, and measures one ten-day
|
||||
TQL selector with two warmups and 15 iterations:
|
||||
|
||||
- `prom_remote_write_seeded_random`: high-distinct seeded-random values with
|
||||
cardinality 29,491,200 and seed 8444.
|
||||
- `prom_remote_write_run_heavy`: low-cardinality values in exact 16-sample runs.
|
||||
- `prom_remote_write_mixed_every`: continuous signal values with every fifth
|
||||
sample repeated at the base value.
|
||||
- `prom_remote_write_integer_counter`: strictly increasing integer counter
|
||||
values, with disjoint ranges for each series.
|
||||
|
||||
These cases record normal storage/footer and read-bench results but do not gate
|
||||
specific value encodings or storage-size outcomes. Use the controlled encoding
|
||||
experiment matrix for those policy comparisons.
|
||||
|
||||
`tests/perf/query_cases/prom_remote_write_7913/case.toml` is a larger manual
|
||||
case for issue #7913. It writes 8192 series × 20160 samples through remote-write
|
||||
in 1440-sample daily time chunks, flushing after each chunk before running 1d/7d/14d
|
||||
@@ -246,7 +359,8 @@ as a fallback. Manifest files are reflinked or copied, not hardlinked.
|
||||
Fixture generator smoke test:
|
||||
|
||||
```bash
|
||||
cargo run -p cmd --bin query_perf_fixture -- \
|
||||
cargo run -p cmd --bin query_perf_fixture --features dev-tools -- \
|
||||
direct-sst \
|
||||
--case tests/perf/query_cases/smoke_direct_sst/case.toml \
|
||||
--out-dir /tmp/query-perf-smoke
|
||||
```
|
||||
@@ -271,10 +385,10 @@ Remote-write runner dry-run:
|
||||
|
||||
```bash
|
||||
uv run --no-project python tests/perf/query_regression_runner.py \
|
||||
--case tests/perf/query_cases/prom_remote_write_smoke/case.toml \
|
||||
--case tests/perf/query_cases/prom_remote_write_seeded_random/case.toml \
|
||||
--base-bin /path/to/base/greptime \
|
||||
--candidate-bin /path/to/candidate/greptime \
|
||||
--remote-write-generator /path/to/prom_remote_write_fixture \
|
||||
--fixture-generator /path/to/query_perf_fixture \
|
||||
--work-dir /tmp/query-perf-remote-write \
|
||||
--dry-run
|
||||
```
|
||||
@@ -289,14 +403,22 @@ query regression runs. It builds its own binaries for now:
|
||||
candidate checkout
|
||||
- runner and summary formatter from the candidate checkout
|
||||
|
||||
The workflow runs automatically only for non-draft PRs labeled
|
||||
`query-regression` (on label/ready-for-review/reopen events, not every push).
|
||||
The workflow builds base and candidate `greptime` as normal release-equivalent
|
||||
binaries. Candidate `query_perf_fixture` is the extra head-side helper binary;
|
||||
the runner uses candidate `greptime datanode parquetbench/scanbench` as the
|
||||
read-bench tool against each target's data directory.
|
||||
|
||||
The workflow runs automatically only when `query-regression` is added to a
|
||||
non-draft PR; it does not rerun on pushes, ready-for-review, or reopen events.
|
||||
PR runs build base/candidate once and then run the default case set with
|
||||
`--allow-large-fixture`. Manual `workflow_dispatch` runs can pass `all`, one case
|
||||
path, or a comma/whitespace-separated list of case paths, and can override refs.
|
||||
It always uploads `query-regression-work/**` and `query-regression-summary.md`,
|
||||
writes the Markdown summary to the workflow step summary, and updates a sticky PR
|
||||
comment through the trusted follow-up workflow.
|
||||
The main report artifact uploads only aggregate/per-target JSON reports,
|
||||
component logs, and `query-regression-summary.md` with seven-day retention;
|
||||
fixture data, SSTs, and cluster state are excluded. PR runs also upload a
|
||||
separate trusted-comment artifact containing PR metadata and aggregate reports.
|
||||
The workflow writes the Markdown summary to the workflow step summary and
|
||||
updates a sticky PR comment through the trusted follow-up workflow.
|
||||
|
||||
## Built-in cases
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
[case]
|
||||
name = "prom_remote_write_integer_counter"
|
||||
description = "End-to-end Prometheus remote-write regression with strictly increasing integer counter values"
|
||||
|
||||
[scenario]
|
||||
kind = "prom_remote_write_then_query"
|
||||
|
||||
[scenario.remote_write]
|
||||
database = "public"
|
||||
metric = "prom_remote_write_integer_counter"
|
||||
physical_table = "greptime_physical_table"
|
||||
series_count = 2048
|
||||
samples_per_series = 14400
|
||||
sample_chunk_size = 1440
|
||||
flush_every_sample_chunks = 1
|
||||
start_unix_millis = 1704067200000
|
||||
step_millis = 60000
|
||||
chunk_series_count = 256
|
||||
timeout_seconds = 600
|
||||
visibility_timeout_seconds = 300
|
||||
|
||||
[scenario.remote_write.value]
|
||||
pattern = "unique"
|
||||
base = 0
|
||||
step = 1
|
||||
|
||||
[scenario.remote_write.prom_store]
|
||||
pending_rows_flush_interval = "1s"
|
||||
max_batch_rows = 100000
|
||||
|
||||
[scenario.remote_write.storage]
|
||||
column = "greptime_value"
|
||||
min_files = 2
|
||||
min_files_with_column = 2
|
||||
|
||||
[scenario.remote_write.read_bench]
|
||||
iterations = 7
|
||||
max_files = 4
|
||||
|
||||
[[scenario.queries]]
|
||||
name = "selector_10d_integer_counter"
|
||||
kind = "tql"
|
||||
query = "TQL ANALYZE VERBOSE (1704067200, 1704931200, '1h') prom_remote_write_integer_counter{host=~'host.*'}"
|
||||
warmup = 2
|
||||
iterations = 15
|
||||
|
||||
[scenario.queries.thresholds]
|
||||
max_candidate_latency_regression_pct = 75
|
||||
@@ -0,0 +1,49 @@
|
||||
[case]
|
||||
name = "prom_remote_write_mixed_every"
|
||||
description = "End-to-end Prometheus remote-write regression with periodic repeated signal values"
|
||||
|
||||
[scenario]
|
||||
kind = "prom_remote_write_then_query"
|
||||
|
||||
[scenario.remote_write]
|
||||
database = "public"
|
||||
metric = "prom_remote_write_mixed_every"
|
||||
physical_table = "greptime_physical_table"
|
||||
series_count = 2048
|
||||
samples_per_series = 14400
|
||||
sample_chunk_size = 1440
|
||||
flush_every_sample_chunks = 1
|
||||
start_unix_millis = 1704067200000
|
||||
step_millis = 60000
|
||||
chunk_series_count = 256
|
||||
timeout_seconds = 600
|
||||
visibility_timeout_seconds = 300
|
||||
|
||||
[scenario.remote_write.value]
|
||||
pattern = "mixed_signal_repeated"
|
||||
base = 0
|
||||
step = 0.125
|
||||
mixed_every = 5
|
||||
|
||||
[scenario.remote_write.prom_store]
|
||||
pending_rows_flush_interval = "1s"
|
||||
max_batch_rows = 100000
|
||||
|
||||
[scenario.remote_write.storage]
|
||||
column = "greptime_value"
|
||||
min_files = 2
|
||||
min_files_with_column = 2
|
||||
|
||||
[scenario.remote_write.read_bench]
|
||||
iterations = 7
|
||||
max_files = 4
|
||||
|
||||
[[scenario.queries]]
|
||||
name = "selector_10d_mixed_every"
|
||||
kind = "tql"
|
||||
query = "TQL ANALYZE VERBOSE (1704067200, 1704931200, '1h') prom_remote_write_mixed_every{host=~'host.*'}"
|
||||
warmup = 2
|
||||
iterations = 15
|
||||
|
||||
[scenario.queries.thresholds]
|
||||
max_candidate_latency_regression_pct = 75
|
||||
@@ -0,0 +1,50 @@
|
||||
[case]
|
||||
name = "prom_remote_write_run_heavy"
|
||||
description = "End-to-end Prometheus remote-write regression with repeated 16-sample value runs"
|
||||
|
||||
[scenario]
|
||||
kind = "prom_remote_write_then_query"
|
||||
|
||||
[scenario.remote_write]
|
||||
database = "public"
|
||||
metric = "prom_remote_write_run_heavy"
|
||||
physical_table = "greptime_physical_table"
|
||||
series_count = 2048
|
||||
samples_per_series = 14400
|
||||
sample_chunk_size = 1440
|
||||
flush_every_sample_chunks = 1
|
||||
start_unix_millis = 1704067200000
|
||||
step_millis = 60000
|
||||
chunk_series_count = 256
|
||||
timeout_seconds = 600
|
||||
visibility_timeout_seconds = 300
|
||||
|
||||
[scenario.remote_write.value]
|
||||
pattern = "run_length"
|
||||
base = 0
|
||||
step = 1
|
||||
cardinality = 1024
|
||||
run_length = 16
|
||||
|
||||
[scenario.remote_write.prom_store]
|
||||
pending_rows_flush_interval = "1s"
|
||||
max_batch_rows = 100000
|
||||
|
||||
[scenario.remote_write.storage]
|
||||
column = "greptime_value"
|
||||
min_files = 2
|
||||
min_files_with_column = 2
|
||||
|
||||
[scenario.remote_write.read_bench]
|
||||
iterations = 7
|
||||
max_files = 4
|
||||
|
||||
[[scenario.queries]]
|
||||
name = "selector_10d_run_heavy"
|
||||
kind = "tql"
|
||||
query = "TQL ANALYZE VERBOSE (1704067200, 1704931200, '1h') prom_remote_write_run_heavy{host=~'host.*'}"
|
||||
warmup = 2
|
||||
iterations = 15
|
||||
|
||||
[scenario.queries.thresholds]
|
||||
max_candidate_latency_regression_pct = 75
|
||||
@@ -0,0 +1,48 @@
|
||||
[case]
|
||||
name = "prom_remote_write_seeded_random"
|
||||
description = "End-to-end Prometheus remote-write regression with high-distinct seeded-random values"
|
||||
|
||||
[scenario]
|
||||
kind = "prom_remote_write_then_query"
|
||||
|
||||
[scenario.remote_write]
|
||||
database = "public"
|
||||
metric = "prom_remote_write_seeded_random"
|
||||
physical_table = "greptime_physical_table"
|
||||
series_count = 2048
|
||||
samples_per_series = 14400
|
||||
sample_chunk_size = 1440
|
||||
flush_every_sample_chunks = 1
|
||||
start_unix_millis = 1704067200000
|
||||
step_millis = 60000
|
||||
chunk_series_count = 256
|
||||
timeout_seconds = 600
|
||||
visibility_timeout_seconds = 300
|
||||
|
||||
[scenario.remote_write.value]
|
||||
pattern = "seeded_random"
|
||||
cardinality = 29_491_200
|
||||
seed = 8444
|
||||
|
||||
[scenario.remote_write.prom_store]
|
||||
pending_rows_flush_interval = "1s"
|
||||
max_batch_rows = 100000
|
||||
|
||||
[scenario.remote_write.storage]
|
||||
column = "greptime_value"
|
||||
min_files = 2
|
||||
min_files_with_column = 2
|
||||
|
||||
[scenario.remote_write.read_bench]
|
||||
iterations = 7
|
||||
max_files = 4
|
||||
|
||||
[[scenario.queries]]
|
||||
name = "selector_10d_seeded_random"
|
||||
kind = "tql"
|
||||
query = "TQL ANALYZE VERBOSE (1704067200, 1704931200, '1h') prom_remote_write_seeded_random{host=~'host.*'}"
|
||||
warmup = 2
|
||||
iterations = 15
|
||||
|
||||
[scenario.queries.thresholds]
|
||||
max_candidate_latency_regression_pct = 75
|
||||
@@ -1,31 +0,0 @@
|
||||
[case]
|
||||
name = "prom_remote_write_smoke"
|
||||
description = "Tiny Prometheus remote-write ingestion then TQL query smoke case"
|
||||
|
||||
[scenario]
|
||||
kind = "prom_remote_write_then_query"
|
||||
|
||||
[scenario.remote_write]
|
||||
database = "public"
|
||||
metric = "prom_remote_write_smoke"
|
||||
physical_table = "greptime_physical_table"
|
||||
series_count = 8
|
||||
samples_per_series = 30
|
||||
start_unix_millis = 1_704_067_200_000
|
||||
step_millis = 15_000
|
||||
chunk_series_count = 4
|
||||
timeout_seconds = 30
|
||||
|
||||
[scenario.remote_write.prom_store]
|
||||
pending_rows_flush_interval = "1s"
|
||||
max_batch_rows = 10000
|
||||
|
||||
[[scenario.queries]]
|
||||
name = "selector_5m_smoke"
|
||||
kind = "tql"
|
||||
query = "TQL ANALYZE VERBOSE (1704067200, 1704067650, '15s') prom_remote_write_smoke{host=~'host.*'}"
|
||||
warmup = 0
|
||||
iterations = 2
|
||||
|
||||
[scenario.queries.thresholds]
|
||||
max_candidate_latency_regression_pct = 100
|
||||
@@ -66,7 +66,8 @@ def parse_args() -> argparse.Namespace:
|
||||
p.add_argument("--base-bin", required=True, type=Path)
|
||||
p.add_argument("--candidate-bin", required=True, type=Path)
|
||||
p.add_argument("--fixture-generator", type=Path)
|
||||
p.add_argument("--remote-write-generator", type=Path, help="prom_remote_write_fixture helper binary")
|
||||
p.add_argument("--remote-write-generator", type=Path, help="deprecated: use --fixture-generator query_perf_fixture")
|
||||
p.add_argument("--storage-inspector", type=Path, help="deprecated: use --fixture-generator query_perf_fixture")
|
||||
p.add_argument("--work-dir", required=True, type=Path)
|
||||
p.add_argument("--fixture-cache-dir", type=Path, help="persistent directory for generated fixtures, keyed by case content")
|
||||
p.add_argument("--reuse-fixture", action="store_true")
|
||||
@@ -83,6 +84,16 @@ def load_case(path: Path) -> dict[str, Any]:
|
||||
return tomllib.load(f)
|
||||
|
||||
|
||||
def load_normalized_case(case_path: Path, fixture_generator: Path | None) -> dict[str, Any]:
|
||||
if fixture_generator is None:
|
||||
raise ValueError("--fixture-generator query_perf_fixture is required for Rust-owned case planning")
|
||||
result = run_command([str(fixture_generator), "plan", "--case", str(case_path)])
|
||||
if result["returncode"] != 0:
|
||||
raise RuntimeError(f"query_perf_fixture plan failed: {result['stderr'][:2000]}")
|
||||
plan = json.loads(result["stdout"])
|
||||
return {"case": load_case(case_path).get("case", {}), "scenario": plan["scenario"], "schema_version": plan.get("schema_version")}
|
||||
|
||||
|
||||
def require_binary(path: Path, name: str, *, dry_run: bool) -> None:
|
||||
if dry_run:
|
||||
return
|
||||
@@ -157,11 +168,9 @@ def scenario(case: dict[str, Any]) -> dict[str, Any]:
|
||||
def case_tables(case: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
value = scenario(case)
|
||||
if value.get("kind") == "prom_remote_write_then_query":
|
||||
remote = value.get("remote_write") or {}
|
||||
metric = remote.get("metric") or remote.get("metric_name")
|
||||
database = remote.get("database", "public")
|
||||
if not metric:
|
||||
raise ValueError("remote-write scenario requires scenario.remote_write.metric")
|
||||
remote = value["remote_write"]
|
||||
metric = remote["metric"]
|
||||
database = remote["database"]
|
||||
return [{"database": database, "name": metric, "engine": "metric", "validate_show_create_engine": False}]
|
||||
tables = value.get("tables") or []
|
||||
if not tables or value.get("layout", {}).get("regions") != 1:
|
||||
@@ -548,7 +557,7 @@ def generate_fixture(generator: Path | None, case_path: Path, fixture_dir: Path,
|
||||
if fixture_dir.exists() and not reuse_fixture:
|
||||
shutil.rmtree(fixture_dir)
|
||||
fixture_dir.mkdir(parents=True, exist_ok=True)
|
||||
cmd = [str(generator), "--case", str(case_path), "--out-dir", str(fixture_dir)]
|
||||
cmd = [str(generator), "direct-sst", "--case", str(case_path), "--out-dir", str(fixture_dir)]
|
||||
if table_name is not None:
|
||||
cmd += ["--table", table_name]
|
||||
if region_id is not None:
|
||||
@@ -777,6 +786,243 @@ def enforce_thresholds(case: dict[str, Any], base: dict[str, Any], candidate: di
|
||||
return results
|
||||
|
||||
|
||||
def storage_config(remote: dict[str, Any]) -> dict[str, Any] | None:
|
||||
value = remote["storage"]
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
if value["inspect"] is False:
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def run_storage_inspection(helper: Path | None, target: RunTarget, storage: dict[str, Any], *, dry_run: bool) -> dict[str, Any]:
|
||||
if helper is None:
|
||||
if not dry_run:
|
||||
raise ValueError("--fixture-generator query_perf_fixture is required when storage inspection is enabled")
|
||||
helper = Path("query_perf_fixture")
|
||||
root = target.datanode_data_dir
|
||||
if storage.get("root_suffix"):
|
||||
root = root / str(storage["root_suffix"])
|
||||
cmd = [
|
||||
str(helper),
|
||||
"inspect-footer",
|
||||
"--root", str(root),
|
||||
"--column", str(storage["column"]),
|
||||
]
|
||||
if storage["include_metadata_files"]:
|
||||
cmd.append("--include-metadata-files")
|
||||
if dry_run:
|
||||
return {"status": "dry-run", "cmd": cmd, "root": str(root)}
|
||||
result = run_command(cmd)
|
||||
result["status"] = "ok" if result["returncode"] == 0 else "failed"
|
||||
if result["returncode"] != 0:
|
||||
raise RuntimeError(f"storage inspector failed for {target.name}: {result['stderr'][:2000]}")
|
||||
try:
|
||||
result["summary"] = json.loads(result["stdout"])
|
||||
except json.JSONDecodeError:
|
||||
result["summary_parse_error"] = result["stdout"]
|
||||
result["status"] = "failed"
|
||||
return result
|
||||
|
||||
|
||||
def storage_summary(inspection: dict[str, Any]) -> dict[str, Any]:
|
||||
summary = inspection.get("summary")
|
||||
if isinstance(summary, dict):
|
||||
nested = summary.get("summary")
|
||||
if isinstance(nested, dict):
|
||||
return nested
|
||||
return {}
|
||||
|
||||
|
||||
def enforce_storage_thresholds(storage: dict[str, Any] | None, base: dict[str, Any] | None, candidate: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||
if not storage:
|
||||
return []
|
||||
results: list[dict[str, Any]] = []
|
||||
target_summaries = [("base", storage_summary(base or {})), ("candidate", storage_summary(candidate or {}))]
|
||||
cand = target_summaries[1][1]
|
||||
base_summary = storage_summary(base or {})
|
||||
|
||||
def add_abs(name: str, field: str) -> None:
|
||||
limit = storage.get(name)
|
||||
if limit is None:
|
||||
return
|
||||
for target_name, summary in target_summaries:
|
||||
actual = summary.get(field)
|
||||
ok = actual is not None and float(actual) <= float(limit)
|
||||
results.append({"target": target_name, "threshold": name, "status": "passed" if ok else "failed", "actual": actual, "limit": limit})
|
||||
|
||||
def add_min(name: str, field: str) -> None:
|
||||
limit = storage[name]
|
||||
if limit is None:
|
||||
return
|
||||
for target_name, summary in target_summaries:
|
||||
actual = summary.get(field)
|
||||
ok = actual is not None and int(actual) >= int(limit)
|
||||
results.append({"target": target_name, "threshold": name, "status": "passed" if ok else "failed", "actual": actual, "limit": limit})
|
||||
|
||||
def add_cmp(name: str, field: str) -> None:
|
||||
limit = storage.get(name)
|
||||
if limit is None:
|
||||
return
|
||||
base_value = base_summary.get(field)
|
||||
candidate_value = cand.get(field)
|
||||
if base_value in (None, 0) or candidate_value is None:
|
||||
results.append({"threshold": name, "status": "failed", "reason": "missing or zero base/candidate value", "base": base_value, "candidate": candidate_value})
|
||||
return
|
||||
actual = (float(candidate_value) - float(base_value)) / float(base_value) * 100.0
|
||||
results.append({"threshold": name, "status": "passed" if actual <= float(limit) else "failed", "actual_pct": actual, "limit_pct": limit, "base": base_value, "candidate": candidate_value})
|
||||
|
||||
add_min("min_files", "file_count")
|
||||
add_min("min_files_with_column", "files_with_column")
|
||||
add_abs("max_total_file_size_bytes", "total_file_size")
|
||||
add_abs("max_column_compressed_size_bytes", "column_compressed_size")
|
||||
add_abs("max_column_uncompressed_size_bytes", "column_uncompressed_size")
|
||||
for target_name, summary in target_summaries:
|
||||
encodings = set(summary.get("unique_encodings") or [])
|
||||
for required in storage["require_encodings"]:
|
||||
results.append({"target": target_name, "threshold": "require_encodings", "encoding": required, "status": "passed" if required in encodings else "failed"})
|
||||
for forbidden in storage["forbid_encodings"]:
|
||||
results.append({"target": target_name, "threshold": "forbid_encodings", "encoding": forbidden, "status": "passed" if forbidden not in encodings else "failed"})
|
||||
add_cmp("max_candidate_total_file_size_regression_pct", "total_file_size")
|
||||
add_cmp("max_candidate_column_compressed_size_regression_pct", "column_compressed_size")
|
||||
add_cmp("max_candidate_column_uncompressed_size_regression_pct", "column_uncompressed_size")
|
||||
return results
|
||||
|
||||
|
||||
def planned_storage_thresholds(storage: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||
if not storage:
|
||||
return []
|
||||
return list(storage["planned_thresholds"])
|
||||
|
||||
|
||||
REGION_DIR_RE = re.compile(r"^(\d+)_(\d{10})$")
|
||||
|
||||
|
||||
def inspection_report(inspection: dict[str, Any] | None) -> dict[str, Any]:
|
||||
if not inspection:
|
||||
return {}
|
||||
report = inspection.get("summary")
|
||||
return report if isinstance(report, dict) else {}
|
||||
|
||||
|
||||
def inspected_relative_path(target: RunTarget, inspection: dict[str, Any], relative_path: str) -> Path:
|
||||
report = inspection_report(inspection)
|
||||
root_text = report.get("root") or inspection.get("root")
|
||||
if not root_text:
|
||||
return Path(relative_path)
|
||||
root = Path(root_text).resolve(strict=False)
|
||||
data_home = target.datanode_data_dir.resolve(strict=False)
|
||||
try:
|
||||
root_suffix = root.relative_to(data_home)
|
||||
except ValueError as e:
|
||||
raise RuntimeError(f"storage inspection root {root} is not under datanode data home {data_home}") from e
|
||||
return root_suffix / relative_path
|
||||
|
||||
|
||||
def parse_sst_bench_target(target: RunTarget, inspection: dict[str, Any], file: dict[str, Any], *, dry_run: bool) -> dict[str, str] | None:
|
||||
relative_path = str(file.get("relative_path") or "")
|
||||
if dry_run and (not relative_path or "<" in relative_path):
|
||||
return {
|
||||
"relative_path": relative_path or "<table-dir>/<table-id>_<region-seq>/data/<file-id>.parquet",
|
||||
"table_dir": "<table-dir>/",
|
||||
"region_id": "<table-id>:<region-seq>",
|
||||
"path_type": "data",
|
||||
"file_id": "<file-id>",
|
||||
}
|
||||
if not relative_path:
|
||||
return None
|
||||
full_relative = inspected_relative_path(target, inspection, relative_path)
|
||||
parts = full_relative.parts
|
||||
if not parts or not parts[-1].endswith(".parquet"):
|
||||
return None
|
||||
region_index = next((idx for idx, part in enumerate(parts) if REGION_DIR_RE.match(part)), None)
|
||||
if region_index is None or region_index == 0:
|
||||
if dry_run:
|
||||
return {
|
||||
"relative_path": relative_path,
|
||||
"table_dir": "<table-dir>/",
|
||||
"region_id": "<table-id>:<region-seq>",
|
||||
"path_type": "data",
|
||||
"file_id": Path(parts[-1]).stem,
|
||||
}
|
||||
return None
|
||||
region_match = REGION_DIR_RE.match(parts[region_index])
|
||||
assert region_match is not None
|
||||
file_index = len(parts) - 1
|
||||
path_type = "bare"
|
||||
if region_index + 1 < file_index and parts[region_index + 1] in ("data", "metadata"):
|
||||
path_type = parts[region_index + 1]
|
||||
if path_type == "metadata":
|
||||
return None
|
||||
table_dir = "/".join(parts[:region_index]).rstrip("/") + "/"
|
||||
return {
|
||||
"relative_path": str(full_relative),
|
||||
"table_dir": table_dir,
|
||||
"region_id": f"{int(region_match.group(1))}:{int(region_match.group(2))}",
|
||||
"path_type": path_type,
|
||||
"file_id": Path(parts[-1]).stem,
|
||||
}
|
||||
|
||||
|
||||
def run_read_bench(bench_binary: Path, target: RunTarget, read_bench: dict[str, Any] | None, storage_inspection: dict[str, Any] | None, *, dry_run: bool) -> dict[str, Any]:
|
||||
if not read_bench or read_bench["enabled"] is False:
|
||||
return {"status": "skipped", "reason": "read_bench disabled"}
|
||||
run_parquetbench = read_bench["parquetbench"]
|
||||
run_scanbench = read_bench["scanbench"]
|
||||
if not run_parquetbench and not run_scanbench:
|
||||
return {"status": "skipped", "reason": "parquetbench and scanbench disabled"}
|
||||
if storage_inspection is None:
|
||||
raise ValueError("read_bench requires storage inspection")
|
||||
bench_dir = target.work_dir / "read_bench"
|
||||
config_toml = bench_dir / "bench.toml"
|
||||
scan_json = bench_dir / "scan.json"
|
||||
files = inspection_report(storage_inspection).get("files") or []
|
||||
ssts = [f for f in files if f.get("relative_path", "").endswith(".parquet") and f.get("columns")]
|
||||
if dry_run and not ssts:
|
||||
ssts = [{"relative_path": "<landed-sst>.parquet"}]
|
||||
if read_bench["max_files"] is not None:
|
||||
ssts = ssts[: int(read_bench["max_files"])]
|
||||
bench_targets = [parsed for f in ssts if (parsed := parse_sst_bench_target(target, storage_inspection, f, dry_run=dry_run)) is not None]
|
||||
if not bench_targets:
|
||||
return {"status": "failed", "reason": "no inspected data SST files available for read_bench"}
|
||||
config_text = f'[storage]\ndata_home = "{target.datanode_data_dir}"\ntype = "File"\n\n[[region_engine]]\n[region_engine.mito]\n'
|
||||
commands = []
|
||||
parquet_runs = []
|
||||
scan_runs = []
|
||||
iterations = str(int(read_bench["iterations"]))
|
||||
if run_parquetbench:
|
||||
for bench_target in bench_targets:
|
||||
cmd = [str(bench_binary), "datanode", "parquetbench", "--config", str(config_toml), "--region-id", bench_target["region_id"], "--table-dir", bench_target["table_dir"], "--file-id", bench_target["file_id"], "--scan-config", str(scan_json), "--path-type", bench_target["path_type"], "--iterations", iterations, "--reader", str(read_bench["parquet_reader"])]
|
||||
commands.append(cmd)
|
||||
parquet_runs.append({**bench_target, "cmd": cmd, "status": "dry-run" if dry_run else "planned"})
|
||||
regions: dict[tuple[str, str, str], list[str]] = {}
|
||||
if run_scanbench:
|
||||
for bench_target in bench_targets:
|
||||
regions.setdefault((bench_target["table_dir"], bench_target["region_id"], bench_target["path_type"]), []).append(bench_target["relative_path"])
|
||||
for (table_dir, region_id, path_type), region_files in regions.items():
|
||||
cmd = [str(bench_binary), "datanode", "scanbench", "--config", str(config_toml), "--region-id", region_id, "--table-dir", table_dir, "--scan-config", str(scan_json), "--path-type", path_type, "--scanner", str(read_bench["scan_scanner"]), "--parallelism", str(int(read_bench["parallelism"])), "--iterations", iterations]
|
||||
commands.append(cmd)
|
||||
scan_runs.append({"table_dir": table_dir, "region_id": region_id, "path_type": path_type, "files": region_files, "cmd": cmd, "status": "dry-run" if dry_run else "planned"})
|
||||
if dry_run:
|
||||
return {"status": "dry-run", "config_path": str(config_toml), "scan_config_path": str(scan_json), "commands": commands, "parquetbench": parquet_runs, "scanbench": scan_runs}
|
||||
bench_dir.mkdir(parents=True, exist_ok=True)
|
||||
config_toml.write_text(config_text)
|
||||
scan_json.write_text(json.dumps({"projection_names": read_bench["projection"]}, indent=2) + "\n")
|
||||
avg_re = re.compile(r"Average duration[^0-9]*([0-9.]+)\s*ms", re.I)
|
||||
for runs in (parquet_runs, scan_runs):
|
||||
for run in runs:
|
||||
result = run_command(run["cmd"])
|
||||
run.update(result)
|
||||
run["status"] = "ok" if result["returncode"] == 0 else "failed"
|
||||
m = avg_re.search(result.get("stdout", ""))
|
||||
if m:
|
||||
run["average_ms"] = float(m.group(1))
|
||||
def median(runs: list[dict[str, Any]]) -> float | None:
|
||||
vals = [r["average_ms"] for r in runs if "average_ms" in r]
|
||||
return statistics.median(vals) if vals else None
|
||||
return {"status": "ok" if all(r.get("status") == "ok" for r in parquet_runs + scan_runs) else "failed", "config_path": str(config_toml), "scan_config_path": str(scan_json), "parquetbench": parquet_runs, "scanbench": scan_runs, "aggregate": {"parquetbench_median_average_ms": median(parquet_runs), "scanbench_median_average_ms": median(scan_runs)}}
|
||||
|
||||
|
||||
def write_json(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n")
|
||||
@@ -791,18 +1037,18 @@ def require_fresh_work_dirs(targets: list[RunTarget], *, reuse_work_dir: bool, d
|
||||
|
||||
|
||||
def write_frontend_prom_config(target: RunTarget, remote: dict[str, Any]) -> Path:
|
||||
prom = remote.get("prom_store") or {}
|
||||
prom = remote["prom_store"]
|
||||
path = target.work_dir / "frontend-prom-store.toml"
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(
|
||||
"[prom_store]\n"
|
||||
"enable = true\n"
|
||||
"with_metric_engine = true\n"
|
||||
f"pending_rows_flush_interval = {json.dumps(str(prom.get('pending_rows_flush_interval', '1s')))}\n"
|
||||
f"max_batch_rows = {int(prom.get('max_batch_rows', 100000))}\n"
|
||||
f"max_concurrent_flushes = {int(prom.get('max_concurrent_flushes', 256))}\n"
|
||||
f"worker_channel_capacity = {int(prom.get('worker_channel_capacity', 65526))}\n"
|
||||
f"max_inflight_requests = {int(prom.get('max_inflight_requests', 3000))}\n"
|
||||
f"pending_rows_flush_interval = {json.dumps(str(prom['pending_rows_flush_interval']))}\n"
|
||||
f"max_batch_rows = {int(prom['max_batch_rows'])}\n"
|
||||
f"max_concurrent_flushes = {int(prom['max_concurrent_flushes'])}\n"
|
||||
f"worker_channel_capacity = {int(prom['worker_channel_capacity'])}\n"
|
||||
f"max_inflight_requests = {int(prom['max_inflight_requests'])}\n"
|
||||
)
|
||||
return path
|
||||
|
||||
@@ -810,23 +1056,38 @@ def write_frontend_prom_config(target: RunTarget, remote: dict[str, Any]) -> Pat
|
||||
def run_remote_write(generator: Path | None, target: RunTarget, remote: dict[str, Any], *, dry_run: bool) -> dict[str, Any]:
|
||||
if generator is None:
|
||||
if not dry_run:
|
||||
raise ValueError("--remote-write-generator is required for prom_remote_write_then_query unless --dry-run is set")
|
||||
generator = Path("prom_remote_write_fixture")
|
||||
metric = remote.get("metric") or remote.get("metric_name")
|
||||
physical_table = remote.get("physical_table", "greptime_physical_table")
|
||||
raise ValueError("--fixture-generator query_perf_fixture is required for prom_remote_write_then_query")
|
||||
generator = Path("query_perf_fixture")
|
||||
metric = remote["metric"]
|
||||
physical_table = remote["physical_table"]
|
||||
cmd = [
|
||||
str(generator),
|
||||
"prom-remote-write",
|
||||
"--endpoint", f"http://127.0.0.1:{target.http_port}/v1/prometheus/write",
|
||||
"--database", remote.get("database", "public"),
|
||||
"--database", remote["database"],
|
||||
"--metric", metric,
|
||||
"--physical-table", physical_table,
|
||||
"--series-count", str(int(remote.get("series_count", 8))),
|
||||
"--samples-per-series", str(int(remote.get("samples_per_series", 30))),
|
||||
"--start-unix-millis", str(int(remote.get("start_unix_millis", 1_704_067_200_000))),
|
||||
"--step-millis", str(int(remote.get("step_millis", 15_000))),
|
||||
"--chunk-series-count", str(int(remote.get("chunk_series_count", remote.get("batch_size", 8)))),
|
||||
"--timeout-seconds", str(int(remote.get("timeout_seconds", 60))),
|
||||
"--series-count", str(int(remote["series_count"])),
|
||||
"--samples-per-series", str(int(remote["samples_per_series"])),
|
||||
"--start-unix-millis", str(int(remote["start_unix_millis"])),
|
||||
"--step-millis", str(int(remote["step_millis"])),
|
||||
"--chunk-series-count", str(int(remote["chunk_series_count"])),
|
||||
"--timeout-seconds", str(int(remote["timeout_seconds"])),
|
||||
]
|
||||
value = remote["value"]
|
||||
cmd.extend(["--value-pattern", str(value["pattern"])])
|
||||
cmd.extend(["--value-base", str(value["base"])])
|
||||
cmd.extend(["--value-step", str(value["step"])])
|
||||
cmd.extend(["--value-cardinality", str(int(value["cardinality"]))])
|
||||
cmd.extend(["--value-seed", str(int(value["seed"]))])
|
||||
cmd.extend(["--value-run-length", str(int(value["run_length"]))])
|
||||
cmd.extend(["--value-stall-every", str(int(value["stall_every"]))])
|
||||
cmd.extend(["--value-stall-length", str(int(value["stall_length"]))])
|
||||
cmd.extend(["--value-mixed-every", str(int(value["mixed_every"]))])
|
||||
if "sample_offset" in remote:
|
||||
cmd.extend(["--value-sample-offset", str(int(remote["sample_offset"]))])
|
||||
if "total_samples_per_series" in remote:
|
||||
cmd.extend(["--value-total-samples-per-series", str(int(remote["total_samples_per_series"]))])
|
||||
if dry_run:
|
||||
return {"status": "dry-run", "cmd": cmd}
|
||||
result = run_command(cmd)
|
||||
@@ -862,23 +1123,23 @@ def flush_remote_physical_table(target: RunTarget, db: str, physical_table: str,
|
||||
|
||||
|
||||
def run_remote_write_ingestion(generator: Path | None, target: RunTarget, remote: dict[str, Any], args: argparse.Namespace, *, dry_run: bool) -> tuple[dict[str, Any], list[dict[str, Any]]]:
|
||||
sample_chunk_size = remote.get("sample_chunk_size")
|
||||
db = remote.get("database", "public")
|
||||
physical_table = remote.get("physical_table", "greptime_physical_table")
|
||||
sample_chunk_size = remote["sample_chunk_size"]
|
||||
db = remote["database"]
|
||||
physical_table = remote["physical_table"]
|
||||
if sample_chunk_size is None:
|
||||
rw = run_remote_write(generator, target, remote, dry_run=dry_run)
|
||||
flush = flush_remote_physical_table(target, db, physical_table, args, dry_run=dry_run, reason="final")
|
||||
return rw, [flush]
|
||||
|
||||
total_samples = int(remote.get("samples_per_series", 30))
|
||||
total_samples = int(remote["samples_per_series"])
|
||||
chunk_samples = int(sample_chunk_size)
|
||||
if chunk_samples <= 0:
|
||||
raise ValueError("scenario.remote_write.sample_chunk_size must be positive")
|
||||
flush_every = int(remote.get("flush_every_sample_chunks", 1))
|
||||
flush_every = int(remote["flush_every_sample_chunks"])
|
||||
if flush_every <= 0:
|
||||
raise ValueError("scenario.remote_write.flush_every_sample_chunks must be positive")
|
||||
start = int(remote.get("start_unix_millis", 1_704_067_200_000))
|
||||
step = int(remote.get("step_millis", 15_000))
|
||||
start = int(remote["start_unix_millis"])
|
||||
step = int(remote["step_millis"])
|
||||
chunks: list[dict[str, Any]] = []
|
||||
flushes: list[dict[str, Any]] = []
|
||||
chunk_index = 0
|
||||
@@ -889,6 +1150,8 @@ def run_remote_write_ingestion(generator: Path | None, target: RunTarget, remote
|
||||
chunk_remote = dict(remote)
|
||||
chunk_remote["samples_per_series"] = current
|
||||
chunk_remote["start_unix_millis"] = start + offset * step
|
||||
chunk_remote["sample_offset"] = offset
|
||||
chunk_remote["total_samples_per_series"] = total_samples
|
||||
result = run_remote_write(generator, target, chunk_remote, dry_run=dry_run)
|
||||
result["sample_offset"] = offset
|
||||
result["samples_per_series"] = current
|
||||
@@ -901,8 +1164,8 @@ def run_remote_write_ingestion(generator: Path | None, target: RunTarget, remote
|
||||
flushes.append(flush_remote_physical_table(target, db, physical_table, args, dry_run=dry_run, reason="final", chunk_index=chunk_index))
|
||||
aggregate = summarize_remote_write_chunks(chunks)
|
||||
if dry_run:
|
||||
series_count = int(remote.get("series_count", 8))
|
||||
chunk_series_count = int(remote.get("chunk_series_count", remote.get("batch_size", 8)))
|
||||
series_count = int(remote["series_count"])
|
||||
chunk_series_count = int(remote["chunk_series_count"])
|
||||
aggregate["rows"] = series_count * total_samples
|
||||
aggregate["samples_written"] = aggregate["rows"]
|
||||
aggregate["batches"] = chunk_index * ((series_count + chunk_series_count - 1) // chunk_series_count)
|
||||
@@ -917,7 +1180,7 @@ def run_remote_write_ingestion(generator: Path | None, target: RunTarget, remote
|
||||
|
||||
|
||||
def expected_remote_write_rows(remote: dict[str, Any]) -> int:
|
||||
return int(remote.get("series_count", 8)) * int(remote.get("samples_per_series", 30))
|
||||
return int(remote["series_count"]) * int(remote["samples_per_series"])
|
||||
|
||||
|
||||
def extract_count_value(result: dict[str, Any]) -> int | None:
|
||||
@@ -966,16 +1229,21 @@ def poll_expected_count(target: RunTarget, table_name: str, db: str, expected_ro
|
||||
|
||||
|
||||
def run_remote_write_scenario(args: argparse.Namespace, case: dict[str, Any], case_path: Path, targets: list[RunTarget], report: dict[str, Any]) -> None:
|
||||
remote = scenario(case).get("remote_write") or {}
|
||||
remote = scenario(case)["remote_write"]
|
||||
tables = case_tables(case)
|
||||
if args.fixture_only:
|
||||
raise ValueError("--fixture-only is not supported for prom_remote_write_then_query; use --dry-run for planning")
|
||||
if args.remote_write_generator is not None:
|
||||
require_binary(args.remote_write_generator, "remote-write generator", dry_run=args.dry_run)
|
||||
helper = args.fixture_generator or args.remote_write_generator
|
||||
if helper is not None:
|
||||
require_binary(helper, "query_perf_fixture", dry_run=args.dry_run)
|
||||
elif not args.dry_run:
|
||||
raise ValueError("--remote-write-generator is required for prom_remote_write_then_query")
|
||||
raise ValueError("--fixture-generator is required for prom_remote_write_then_query")
|
||||
storage = storage_config(remote)
|
||||
if storage and helper is None and not args.dry_run:
|
||||
raise ValueError("--fixture-generator is required when scenario.remote_write.storage.inspect is enabled")
|
||||
clusters: list[DistributedCluster] = []
|
||||
target_results = []
|
||||
storage_results = []
|
||||
try:
|
||||
for target in targets:
|
||||
target.work_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -984,28 +1252,40 @@ def run_remote_write_scenario(args: argparse.Namespace, case: dict[str, Any], ca
|
||||
clusters.append(cluster)
|
||||
if not args.dry_run:
|
||||
cluster.start_all(config_path)
|
||||
db = remote.get("database", "public")
|
||||
db = remote["database"]
|
||||
create_database = {"status": "dry-run", "database": db}
|
||||
if not args.dry_run:
|
||||
create_database = http_post_sql(target.http_port, f"CREATE DATABASE IF NOT EXISTS {sql_ident(db)}", "public", args.http_timeout)
|
||||
rw, flushes = run_remote_write_ingestion(args.remote_write_generator, target, remote, args, dry_run=args.dry_run)
|
||||
physical = remote.get("physical_table", "greptime_physical_table")
|
||||
rw, flushes = run_remote_write_ingestion(helper, target, remote, args, dry_run=args.dry_run)
|
||||
visibility = {"status": "dry-run"}
|
||||
query_result = {"validation": [], "validation_errors": [], "measurements": [], "status": "planned"}
|
||||
if not args.dry_run:
|
||||
visibility = poll_expected_count(target, tables[0]["name"], db, expected_remote_write_rows(remote), float(remote.get("visibility_timeout_seconds", 30)), args.http_timeout)
|
||||
visibility = poll_expected_count(target, tables[0]["name"], db, expected_remote_write_rows(remote), float(remote["visibility_timeout_seconds"]), args.http_timeout)
|
||||
query_result = run_queries(target, case, tables, args.http_timeout)
|
||||
cluster.stop_component("datanode")
|
||||
storage_inspection = None
|
||||
if storage:
|
||||
storage_inspection = run_storage_inspection(helper or args.storage_inspector, target, storage, dry_run=args.dry_run)
|
||||
storage_results.append(storage_inspection)
|
||||
read_bench_result = run_read_bench(args.candidate_bin, target, remote["read_bench"], storage_inspection, dry_run=args.dry_run)
|
||||
tr = {"name": target.name, "binary": str(target.binary), "work_dir": str(target.work_dir), "components": cluster.component_report(), "frontend_config": str(config_path), "create_database": create_database, "remote_write": rw, "flushes": flushes, "flush": flushes[-1] if flushes else None, "visibility": visibility, **query_result}
|
||||
if storage_inspection is not None:
|
||||
tr["storage_inspection"] = storage_inspection
|
||||
tr["read_bench"] = read_bench_result
|
||||
flushes_ok = all(flush.get("ok") for flush in flushes)
|
||||
remote_checks_ok = args.dry_run or (create_database.get("ok") and flushes_ok and visibility.get("ok") and visibility.get("row_count_ok"))
|
||||
storage_ok = storage_inspection is None or args.dry_run or storage_inspection.get("status") == "ok"
|
||||
read_bench_ok = args.dry_run or read_bench_result.get("status") in ("ok", "skipped")
|
||||
remote_checks_ok = args.dry_run or (create_database.get("ok") and flushes_ok and visibility.get("ok") and visibility.get("row_count_ok") and storage_ok and read_bench_ok)
|
||||
if not remote_checks_ok:
|
||||
tr.setdefault("validation_errors", []).append({"phase": "remote_write_visibility", "create_database_ok": create_database.get("ok"), "flushes_ok": flushes_ok, "visibility_ok": visibility.get("ok"), "row_count_ok": visibility.get("row_count_ok"), "create_database": create_database, "flushes": flushes, "visibility": visibility})
|
||||
tr.setdefault("validation_errors", []).append({"phase": "remote_write_visibility_storage", "create_database_ok": create_database.get("ok"), "flushes_ok": flushes_ok, "visibility_ok": visibility.get("ok"), "row_count_ok": visibility.get("row_count_ok"), "storage_ok": storage_ok, "read_bench_ok": read_bench_ok, "create_database": create_database, "flushes": flushes, "visibility": visibility, "read_bench": read_bench_result})
|
||||
tr["status"] = "planned" if args.dry_run else ("measured" if remote_checks_ok and query_result["status"] == "ok" else "failed")
|
||||
write_json(target.report_path, tr)
|
||||
report["targets"].append(tr)
|
||||
target_results.append(query_result)
|
||||
if not args.dry_run:
|
||||
report["thresholds"] = enforce_thresholds(case, target_results[0], target_results[1])
|
||||
report["thresholds"] = enforce_thresholds(case, target_results[0], target_results[1]) + enforce_storage_thresholds(storage, storage_results[0] if storage_results else None, storage_results[1] if len(storage_results) > 1 else None)
|
||||
elif storage:
|
||||
report["thresholds"] = planned_storage_thresholds(storage)
|
||||
report["status"] = "planned" if args.dry_run else ("failed" if any(t["status"] == "failed" for t in report["thresholds"]) or any(t.get("status") == "failed" for t in report["targets"]) else "ok")
|
||||
finally:
|
||||
for cluster in reversed(clusters):
|
||||
@@ -1015,7 +1295,9 @@ def run_remote_write_scenario(args: argparse.Namespace, case: dict[str, Any], ca
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
case_path = args.case.resolve()
|
||||
case = load_case(case_path)
|
||||
if args.fixture_generator is not None:
|
||||
require_binary(args.fixture_generator, "fixture generator", dry_run=False)
|
||||
case = load_normalized_case(case_path, args.fixture_generator)
|
||||
scenario_config = scenario(case)
|
||||
scenario_kind = scenario_config.get("kind")
|
||||
tables = case_tables(case)
|
||||
@@ -1025,9 +1307,6 @@ def main() -> int:
|
||||
require_binary(args.candidate_bin, "candidate", dry_run=args.dry_run or args.fixture_only)
|
||||
if scenario_kind == "direct_readable_sst" and not args.dry_run and args.fixture_generator is None:
|
||||
raise ValueError("--fixture-generator is required unless --dry-run is set")
|
||||
if args.fixture_generator is not None:
|
||||
require_binary(args.fixture_generator, "fixture generator", dry_run=args.dry_run)
|
||||
|
||||
ports = allocate_ports(16)
|
||||
targets = [make_target("base", args.base_bin.resolve(), work_root, ports[:8]), make_target("candidate", args.candidate_bin.resolve(), work_root, ports[8:])]
|
||||
require_fresh_work_dirs(targets, reuse_work_dir=args.reuse_work_dir, dry_run=args.dry_run, fixture_only=args.fixture_only)
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright 2023 Greptime Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Regression coverage for quiescing remote-write storage before read benches."""
|
||||
|
||||
import argparse
|
||||
import importlib.util
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
|
||||
RUNNER_PATH = Path(__file__).with_name("query_regression_runner.py")
|
||||
SPEC = importlib.util.spec_from_file_location("query_regression_runner_under_test", RUNNER_PATH)
|
||||
assert SPEC is not None and SPEC.loader is not None
|
||||
runner = importlib.util.module_from_spec(SPEC)
|
||||
sys.modules[SPEC.name] = runner
|
||||
SPEC.loader.exec_module(runner)
|
||||
|
||||
|
||||
class RemoteWriteCompactionToctouTest(unittest.TestCase):
|
||||
def test_quiesces_each_target_before_storage_inspection_and_read_bench(self) -> None:
|
||||
lifecycle: dict[str, list[str]] = {"base": [], "candidate": []}
|
||||
clusters = []
|
||||
|
||||
class RecordingCluster:
|
||||
def __init__(self, target):
|
||||
self.target = target
|
||||
self.active = set()
|
||||
self.stop_all_calls = 0
|
||||
clusters.append(self)
|
||||
|
||||
def start_all(self, _config_path):
|
||||
self.active.update(("metasrv", "datanode", "frontend"))
|
||||
|
||||
def stop_component(self, name):
|
||||
if name in self.active:
|
||||
lifecycle[self.target.name].append(f'cluster.stop_component("{name}")')
|
||||
self.active.remove(name)
|
||||
|
||||
def stop_all(self):
|
||||
self.stop_all_calls += 1
|
||||
for name in ("frontend", "datanode", "metasrv"):
|
||||
self.stop_component(name)
|
||||
|
||||
def component_report(self):
|
||||
return {}
|
||||
|
||||
def run_queries(target, _case, _tables, _http_timeout):
|
||||
lifecycle[target.name].append("query measurement completes")
|
||||
return {"validation": [], "validation_errors": [], "measurements": [], "status": "ok"}
|
||||
|
||||
def inspect_storage(_helper, target, _storage, *, dry_run):
|
||||
self.assertFalse(dry_run)
|
||||
lifecycle[target.name].append("storage inspection")
|
||||
return {"status": "ok", "summary": {"summary": {}}}
|
||||
|
||||
def read_bench(_binary, target, _read_bench, _inspection, *, dry_run):
|
||||
self.assertFalse(dry_run)
|
||||
lifecycle[target.name].append("read bench")
|
||||
return {"status": "ok"}
|
||||
|
||||
remote_write = {
|
||||
"database": "public",
|
||||
"metric": "cpu_usage",
|
||||
"physical_table": "cpu_usage_physical",
|
||||
"series_count": 1,
|
||||
"samples_per_series": 1,
|
||||
"visibility_timeout_seconds": 1,
|
||||
"storage": {
|
||||
"inspect": True,
|
||||
"min_files": None,
|
||||
"min_files_with_column": None,
|
||||
"max_total_file_size_bytes": None,
|
||||
"max_column_compressed_size_bytes": None,
|
||||
"max_column_uncompressed_size_bytes": None,
|
||||
"require_encodings": [],
|
||||
"forbid_encodings": [],
|
||||
},
|
||||
"read_bench": {"enabled": True},
|
||||
"prom_store": {
|
||||
"pending_rows_flush_interval": "1s",
|
||||
"max_batch_rows": 1,
|
||||
"max_concurrent_flushes": 1,
|
||||
"worker_channel_capacity": 1,
|
||||
"max_inflight_requests": 1,
|
||||
},
|
||||
}
|
||||
case = {
|
||||
"scenario": {
|
||||
"kind": "prom_remote_write_then_query",
|
||||
"remote_write": remote_write,
|
||||
"queries": [],
|
||||
}
|
||||
}
|
||||
args = argparse.Namespace(
|
||||
fixture_only=False,
|
||||
fixture_generator=Path("/bin/true"),
|
||||
remote_write_generator=None,
|
||||
storage_inspector=None,
|
||||
candidate_bin=Path("/bin/true"),
|
||||
dry_run=False,
|
||||
http_timeout=1.0,
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
root = Path(tmpdir)
|
||||
targets = [
|
||||
runner.make_target("base", Path("/bin/true"), root, list(range(10000, 10008))),
|
||||
runner.make_target("candidate", Path("/bin/true"), root, list(range(10008, 10016))),
|
||||
]
|
||||
report = {"targets": [], "thresholds": []}
|
||||
with (
|
||||
patch.object(runner, "DistributedCluster", RecordingCluster),
|
||||
patch.object(runner, "run_remote_write_ingestion", return_value=({"status": "ok"}, [{"ok": True}])),
|
||||
patch.object(runner, "http_post_sql", return_value={"ok": True}),
|
||||
patch.object(runner, "poll_expected_count", return_value={"ok": True, "row_count_ok": True}),
|
||||
patch.object(runner, "run_queries", side_effect=run_queries),
|
||||
patch.object(runner, "run_storage_inspection", side_effect=inspect_storage),
|
||||
patch.object(runner, "run_read_bench", side_effect=read_bench),
|
||||
):
|
||||
runner.run_remote_write_scenario(args, case, root / "case.toml", targets, report)
|
||||
|
||||
for cluster in clusters:
|
||||
self.assertEqual(cluster.stop_all_calls, 1)
|
||||
self.assertEqual(cluster.active, set())
|
||||
cluster.stop_all()
|
||||
self.assertEqual(cluster.stop_all_calls, 2)
|
||||
self.assertEqual(cluster.active, set())
|
||||
|
||||
expected = [
|
||||
"query measurement completes",
|
||||
'cluster.stop_component("datanode")',
|
||||
"storage inspection",
|
||||
"read bench",
|
||||
]
|
||||
relevant = {name: [event for event in events if event in expected] for name, events in lifecycle.items()}
|
||||
for target_name in ("base", "candidate"):
|
||||
with self.subTest(target=target_name):
|
||||
self.assertEqual(relevant[target_name], expected)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user