feat(mito2): add opt-in byte-stream-split encoding for float SST fields (#9069)

* feat(mito2): add opt-in byte stream split encoding

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* fix(mito2): correct float encoding checks

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(compat): cover float SST encoding

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* fix(mito2): compile float encoding tests

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* fix(mito2): release parquet test writer

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* fix(mito2): register float test primary key

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(mito2): verify BSS write lifecycles

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(metric-engine): verify BSS physical SST

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(mito2): verify bulk BSS lifecycle

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* fix(mito2): compile bulk BSS test

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* refactor(mito2): narrow bulk encoding constructors

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(compat): accept generated float upgrade output

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(compat): accept generated float downgrade output

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* refactor(mito2): narrow bulk encoding builder

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(perf): add default versus BSS storage comparison

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(perf): align BSS reader benchmarks with prior study

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* fix(perf): parse current read benchmark averages

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* fix(perf): retain default float encoding in direct SST fixtures

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(perf): isolate BSS user SSTs and benchmark every file

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* docs(perf): record measured BSS storage and reader tradeoffs

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* docs(perf): expose warm scan variability and evidence limits

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* docs(perf): clarify BSS baseline and storage measurement scope

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(perf): model bounded mixed integer and fractional metric series

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* docs(perf): report bounded mixed BSS measurements and query regressions

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* docs(perf): qualify timings affected by concurrent host builds

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* test(perf): include float BSS comparison in default regression cases

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

* fix(perf): omit unsupported float encoding option from baseline setup

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>

---------

Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
This commit is contained in:
discord9
2026-09-14 12:05:42 +00:00
committed by GitHub
parent 03d4e62a22
commit ba0f7acd93
35 changed files with 1521 additions and 120 deletions
@@ -108,7 +108,7 @@ agent, which Aliyun public Ubuntu images include.
An allowlisted repository admin commenting `/query-regression` on the PR is
**trust admission for that exact PR revision**. `/query-regression` runs the
six routine default cases; `/query-regression heavy` runs only the
nine routine default cases; `/query-regression heavy` runs only the
high-cardinality `prom_remote_write_7913` remote-write case.
`slash-command-dispatch.yml` (`issue_comment` on the default branch) parses
the command and `repository_dispatch`es; `query-regression-slash.yml` admits
+1
View File
@@ -33,6 +33,7 @@ from typing import Any
DEFAULT_CASES = [
"tests/perf/query_cases/smoke_direct_sst/case.toml",
"tests/perf/query_cases/sst_float_bss/case.toml",
"tests/perf/query_cases/prom_remote_write_seeded_random/case.toml",
"tests/perf/query_cases/prom_remote_write_run_heavy/case.toml",
"tests/perf/query_cases/prom_remote_write_mixed_every/case.toml",
@@ -105,6 +105,10 @@ pub(super) struct PromRemoteWritePlan {
#[serde(default = "default_visibility_timeout_seconds")]
pub(super) visibility_timeout_seconds: u64,
#[serde(default)]
pub(super) base_setup_sql: Vec<String>,
#[serde(default)]
pub(super) candidate_setup_sql: Vec<String>,
#[serde(default)]
pub(super) prom_store: PromStoreConfig,
#[serde(default)]
pub(super) value: ValueConfig,
@@ -200,6 +204,7 @@ pub(super) enum ValuePattern {
QuantizedSignal,
SignalWithSporadicStalls,
MixedSignalRepeated,
BoundedMixed,
}
impl std::fmt::Display for ValuePattern {
@@ -436,6 +441,63 @@ pub(super) fn default_parallelism() -> u64 {
1
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn remote_write_setup_sql_defaults_to_empty() {
let case: CaseFile = toml::from_str(
r#"
[scenario]
kind = "prom_remote_write_then_query"
[scenario.remote_write]
metric = "metric"
"#,
)
.unwrap();
let Scenario::PromRemoteWriteThenQuery(scenario) = case.scenario else {
panic!("expected prom_remote_write_then_query scenario");
};
assert!(scenario.remote_write.base_setup_sql.is_empty());
assert!(scenario.remote_write.candidate_setup_sql.is_empty());
}
#[test]
fn remote_write_setup_sql_roundtrips() {
let case: CaseFile = toml::from_str(
r#"
[scenario]
kind = "prom_remote_write_then_query"
[scenario.remote_write]
metric = "metric"
base_setup_sql = ["CREATE TABLE base_table"]
candidate_setup_sql = ["CREATE TABLE candidate_table", "ALTER TABLE candidate_table SET 'x'='y'"]
"#,
)
.unwrap();
let roundtrip: CaseFile =
serde_json::from_str(&serde_json::to_string(&case).unwrap()).unwrap();
let Scenario::PromRemoteWriteThenQuery(scenario) = roundtrip.scenario else {
panic!("expected prom_remote_write_then_query scenario");
};
assert_eq!(
scenario.remote_write.base_setup_sql,
["CREATE TABLE base_table"]
);
assert_eq!(
scenario.remote_write.candidate_setup_sql,
[
"CREATE TABLE candidate_table",
"ALTER TABLE candidate_table SET 'x'='y'"
]
);
}
}
impl Scenario {
pub(super) fn kind(&self) -> &'static str {
match self {
@@ -469,6 +469,7 @@ pub(super) async fn run_direct_sst(args: DirectArgs) {
write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
row_group_size: scenario.layout.row_group_size,
max_file_size: None,
float_field_encoding: Default::default(),
};
let infos = match format {
FormatType::Flat => writer.write_all_flat(source, Some(sequence), &opts).await,
@@ -197,8 +197,35 @@ fn deterministic_prom_value(args: &PromRemoteWriteArgs, series_idx: u64, sample_
args.value_base + (series_idx % 97) as f64 + local as f64 * args.value_step
}
}
ValuePattern::BoundedMixed => bounded_mixed_value(args, series_idx, local),
}
}
fn bounded_mixed_value(args: &PromRemoteWriteArgs, series_idx: u64, local: u64) -> f64 {
const SPANS: [i64; 4] = [10, 1_000, 100_000, 10_000_000];
let series = splitmix64(series_idx ^ args.value_seed);
let span = SPANS[(series >> 62) as usize] * (1 + ((series >> 32) % 10) as i64);
let baseline = (series % args.value_cardinality.max(1)) as f64 * span as f64 / 10.0;
let run_length = args.value_run_length.max(1);
let group = local / run_length;
let phase = local % run_length;
let anchor = bounded_mixed_anchor(series, group, span);
let next_anchor = bounded_mixed_anchor(series, group + 1, span);
let signal = baseline + anchor + (next_anchor - anchor) * phase as f64 / run_length as f64;
let value = (args.value_base + args.value_step * signal).round();
if series_idx % args.value_mixed_every.max(1) == args.value_mixed_every.max(1) - 1 {
let numerator = splitmix64(series ^ local) % 999 + 1;
value + numerator as f64 / 1_000.0
} else {
value
}
}
fn bounded_mixed_anchor(series: u64, group: u64, span: i64) -> f64 {
let limit = span - 1;
(splitmix64(series ^ group) % (2 * limit as u64 + 1)) as i64 as f64 - limit as f64
}
fn splitmix64(mut value: u64) -> u64 {
value = value.wrapping_add(0x9e3779b97f4a7c15);
let mut mixed = value;
@@ -307,7 +334,7 @@ mod tests {
for chunk_idx in 0..10 {
let mut chunk = args(ValuePattern::SeededRandom);
chunk.samples_per_series = 1_440;
chunk.value_sample_offset = chunk_idx * 1_440;
chunk.value_sample_offset = chunk_idx * 1_439;
chunk.value_total_samples_per_series = Some(14_400);
for sample_idx in 0..chunk.samples_per_series {
assert_eq!(
@@ -319,6 +346,118 @@ mod tests {
}
}
#[test]
fn bounded_mixed_has_bounded_95_5_per_series_values() {
let mut args = args(ValuePattern::BoundedMixed);
args.series_count = 1_000;
args.samples_per_series = 4_320;
args.value_cardinality = 1_000;
args.value_seed = 42;
args.value_run_length = 60;
args.value_mixed_every = 20;
let mut integral_series = 0;
let mut fractional_series = 0;
let mut spans = HashSet::new();
let mut ranges = HashSet::new();
for series_idx in 0..args.series_count {
let series = splitmix64(series_idx ^ args.value_seed);
let span = [10_i64, 1_000, 100_000, 10_000_000][(series >> 62) as usize]
* (1 + ((series >> 32) % 10) as i64);
let baseline = (series % args.value_cardinality.max(1)) as f64 * span as f64 / 10.0;
let values = (0..args.samples_per_series)
.map(|sample_idx| deterministic_prom_value(&args, series_idx, sample_idx))
.collect::<Vec<_>>();
assert!(values.iter().all(|value| value.is_finite()));
assert!(
values
.iter()
.all(|value| *value >= baseline - span as f64 - 1.0
&& *value <= baseline + span as f64 + 1.0)
);
assert!(
values.windows(2).any(|pair| pair[1] > pair[0]),
"series {series_idx} never rises"
);
assert!(
values.windows(2).any(|pair| pair[1] < pair[0]),
"series {series_idx} never falls"
);
ranges.insert((
values
.iter()
.copied()
.fold(f64::INFINITY, f64::min)
.to_bits(),
values
.iter()
.copied()
.fold(f64::NEG_INFINITY, f64::max)
.to_bits(),
));
spans.insert(span);
if series_idx % args.value_mixed_every == args.value_mixed_every - 1 {
fractional_series += 1;
assert!(values.iter().all(|value| value.fract() != 0.0));
} else {
integral_series += 1;
assert!(values.iter().all(|value| value.fract() == 0.0));
}
}
assert_eq!((integral_series, fractional_series), (950, 50));
assert!(spans.len() > 4);
assert!(ranges.len() > 1);
}
#[test]
fn bounded_mixed_is_seeded_and_chunk_bit_identical() {
let mut monolithic = args(ValuePattern::BoundedMixed);
monolithic.samples_per_series = 14_400;
monolithic.value_total_samples_per_series = Some(14_400);
monolithic.value_cardinality = 1_000;
monolithic.value_seed = 42;
monolithic.value_run_length = 60;
monolithic.value_mixed_every = 20;
let original = (0..600)
.map(|sample_idx| deterministic_prom_value(&monolithic, 19, sample_idx))
.collect::<Vec<_>>();
monolithic.value_seed = 43;
assert_ne!(
original,
(0..600)
.map(|sample_idx| deterministic_prom_value(&monolithic, 19, sample_idx))
.collect::<Vec<_>>()
);
monolithic.value_seed = 42;
for series_idx in [0, 19] {
for chunk_idx in 0..10 {
let mut chunk = args(ValuePattern::BoundedMixed);
chunk.samples_per_series = 1_440;
chunk.value_sample_offset = chunk_idx * 1_439;
chunk.value_total_samples_per_series = Some(14_400);
chunk.value_cardinality = 1_000;
chunk.value_seed = 42;
chunk.value_run_length = 60;
chunk.value_mixed_every = 20;
for sample_idx in 0..chunk.samples_per_series {
assert_eq!(
deterministic_prom_value(&chunk, series_idx, sample_idx).to_bits(),
deterministic_prom_value(
&monolithic,
series_idx,
chunk.value_sample_offset + sample_idx
)
.to_bits()
);
}
}
}
}
#[test]
fn values_continue_across_series_boundaries() {
let mut args = args(ValuePattern::Unique);
@@ -116,6 +116,10 @@ pub(super) struct RemoteWrite {
pub(super) sample_chunk_size: Option<u64>,
pub(super) flush_every_sample_chunks: u64,
pub(super) visibility_timeout_seconds: u64,
#[serde(default)]
pub(super) base_setup_sql: Vec<String>,
#[serde(default)]
pub(super) candidate_setup_sql: Vec<String>,
pub(super) prom_store: PromStore,
pub(super) value: RemoteValue,
pub(super) storage: Option<StorageConfig>,
@@ -57,6 +57,7 @@ pub(super) async fn run_prepare_remote(args: PrepareRemoteArgs) -> Result<()> {
args.base_http_port,
&args.fixture_generator,
&remote,
&remote.base_setup_sql,
&client,
)
.await?;
@@ -65,6 +66,7 @@ pub(super) async fn run_prepare_remote(args: PrepareRemoteArgs) -> Result<()> {
args.candidate_http_port,
&args.fixture_generator,
&remote,
&remote.candidate_setup_sql,
&client,
)
.await?;
@@ -88,6 +90,7 @@ async fn prepare_remote_target(
port: u16,
generator: &Path,
remote: &RemoteWrite,
setup_sql: &[String],
client: &Client,
) -> Result<Value> {
let create_database = http_post_sql(
@@ -107,6 +110,7 @@ async fn prepare_remote_target(
)
.into());
}
let setup = run_setup_sql(client, port, &remote.database, name, setup_sql).await?;
let (remote_write, flushes) = ingest_remote_write(generator, port, remote, client).await?;
let expected_rows = remote
.series_count
@@ -124,6 +128,7 @@ async fn prepare_remote_target(
Ok(json!({
"name": name,
"create_database": create_database,
"setup_sql": setup,
"remote_write": remote_write,
"flushes": flushes,
"visibility": visibility,
@@ -131,6 +136,28 @@ async fn prepare_remote_target(
}))
}
async fn run_setup_sql(
client: &Client,
port: u16,
database: &str,
target: &str,
setup_sql: &[String],
) -> Result<Vec<Value>> {
let mut results = Vec::with_capacity(setup_sql.len());
for (index, statement) in setup_sql.iter().enumerate() {
let result = http_post_sql(client, port, statement, database).await;
if !result["ok"].as_bool().unwrap_or(false) {
return Err(format!(
"setup SQL statement {} failed for {target}: {result}",
index + 1
)
.into());
}
results.push(result);
}
Ok(results)
}
#[derive(Clone, Debug, PartialEq, Eq)]
struct SampleChunk {
index: u64,
@@ -434,9 +461,110 @@ fn json_number(value: f64) -> String {
#[cfg(test)]
mod tests {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::{TcpListener, TcpStream};
use super::*;
use crate::query_regression_runner::model::{PromStore, RemoteValue};
async fn read_http_request(stream: &mut TcpStream) -> Vec<u8> {
let mut request = Vec::new();
let mut chunk = [0; 1024];
loop {
let read = stream.read(&mut chunk).await.unwrap();
assert_ne!(
read, 0,
"HTTP client closed request before sending its body"
);
request.extend_from_slice(&chunk[..read]);
let Some(header_end) = request.windows(4).position(|window| window == b"\r\n\r\n")
else {
continue;
};
let headers = std::str::from_utf8(&request[..header_end]).unwrap();
let content_length = headers
.lines()
.find_map(|line| line.strip_prefix("content-length: "))
.or_else(|| {
headers
.lines()
.find_map(|line| line.strip_prefix("Content-Length: "))
})
.unwrap()
.parse::<usize>()
.unwrap();
if request.len() >= header_end + 4 + content_length {
return request;
}
}
}
async fn spawn_sql_server(
responses: Vec<String>,
) -> (u16, tokio::task::JoinHandle<Vec<String>>) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
let server = tokio::spawn(async move {
let mut requests = Vec::with_capacity(responses.len());
for body in responses {
let (mut stream, _) = listener.accept().await.unwrap();
let request = read_http_request(&mut stream).await;
requests.push(String::from_utf8(request).unwrap());
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n\
Connection: close\r\nContent-Length: {}\r\n\r\n{body}",
body.len()
);
stream.write_all(response.as_bytes()).await.unwrap();
}
requests
});
(port, server)
}
#[tokio::test]
async fn setup_sql_runs_in_order_with_configured_database() {
let (port, server) = spawn_sql_server(vec!["{\"code\":0}".to_string(); 2]).await;
let client = Client::new();
let setup = vec![
"CREATE TABLE physical (ts TIMESTAMP TIME INDEX) ENGINE=metric".to_string(),
"ALTER TABLE physical SET 'compaction.twcs.trigger_file_num'='100'".to_string(),
];
let results = run_setup_sql(&client, port, "perf", "candidate", &setup)
.await
.unwrap();
assert_eq!(results.len(), 2);
let requests = server.await.unwrap();
assert_eq!(requests.len(), 2);
assert!(requests[0].contains("db=perf"));
assert!(requests[0].contains("sql=CREATE"));
assert!(requests[1].contains("db=perf"));
assert!(requests[1].contains("sql=ALTER"));
}
#[tokio::test]
async fn setup_sql_stops_at_the_failing_statement() {
let (port, server) =
spawn_sql_server(vec!["{\"code\":0}".to_string(), "{\"code\":1}".to_string()]).await;
let client = Client::new();
let setup = vec![
"CREATE TABLE physical (ts TIMESTAMP TIME INDEX) ENGINE=metric".to_string(),
"invalid SQL".to_string(),
"ALTER TABLE physical SET 'compaction.twcs.trigger_file_num'='100'".to_string(),
];
let error = run_setup_sql(&client, port, "perf", "base", &setup)
.await
.unwrap_err();
assert!(error.to_string().contains("statement 2 failed for base"));
let requests = server.await.unwrap();
assert_eq!(requests.len(), 2);
assert!(requests[1].contains("sql=invalid+SQL"));
}
#[test]
fn schedules_remote_sample_chunks_and_flushes() {
let remote = RemoteWrite {
@@ -452,6 +580,8 @@ mod tests {
sample_chunk_size: Some(2),
flush_every_sample_chunks: 2,
visibility_timeout_seconds: 30,
base_setup_sql: Vec::new(),
candidate_setup_sql: Vec::new(),
prom_store: PromStore {
pending_rows_flush_interval: "1s".to_string(),
max_batch_rows: 1,
@@ -306,13 +306,25 @@ fn run_bench_command(command: Vec<String>, mut run: Value) -> Result<Value> {
}
fn parse_average_duration(stdout: &str) -> Option<f64> {
Regex::new(r"(?i)Average duration[^0-9]*([0-9.]+)\s*ms")
.ok()?
.captures(stdout)?
.get(1)?
.as_str()
.parse()
.ok()
let captures = Regex::new(
r"(?i)Average:\s+(?:\x1b\[[0-9;]*m)*\d+(?:\x1b\[[0-9;]*m)*\s+rows(?:,\s+(?:\x1b\[[0-9;]*m)*\d+(?:\x1b\[[0-9;]*m)*\s+record batches)?\s+in\s+([0-9]+(?:\.[0-9]+)?)\s*(ns|µs|us|ms|s)\s+over\s+\d+\s+iterations|Average duration[^0-9]*([0-9]+(?:\.[0-9]+)?)\s*ms",
)
.ok()?
.captures(stdout)?;
let (value, unit) = match (captures.get(1), captures.get(2), captures.get(3)) {
(Some(value), Some(unit), _) => (value.as_str(), unit.as_str()),
(_, _, Some(value)) => (value.as_str(), "ms"),
_ => return None,
};
let value = value.parse::<f64>().ok()?;
let milliseconds = match unit.to_ascii_lowercase().as_str() {
"ns" => value / 1_000_000.0,
"µs" | "us" => value / 1_000.0,
"ms" => value,
"s" => value * 1_000.0,
_ => return None,
};
milliseconds.is_finite().then_some(milliseconds)
}
fn bench_median(runs: &[Value]) -> Option<f64> {
@@ -328,12 +340,51 @@ mod tests {
use super::*;
#[test]
fn parses_average_duration_and_groups_scan_paths() {
assert_eq!(
parse_average_duration("work\nAverage duration: 12.5 ms\n"),
Some(12.5)
);
assert_eq!(parse_average_duration("completed"), None);
fn parses_average_durations() {
let cases = [
(
"parquetbench nanoseconds",
" Average: 42 rows, 2 record batches in 500ns over 4 iterations",
Some(0.0005),
),
(
"scanbench colored microseconds",
"\x1b[1;32mSummary\x1b[0m Average: \x1b[36m12\x1b[0m rows in 250µs over 2 iterations",
Some(0.25),
),
(
"scanbench seconds",
"Summary Average: 12 rows in 2.5s over 2 iterations",
Some(2500.0),
),
(
"legacy milliseconds",
"Average duration: 12.5 ms",
Some(12.5),
),
("absent", "Benchmark completed!", None),
(
"malformed",
"Summary Average: 12 rows in unknown over 2 iterations",
None,
),
];
for (name, stdout, expected) in cases {
let actual = parse_average_duration(stdout);
match (actual, expected) {
(Some(actual), Some(expected)) => assert!(
(actual - expected).abs() < f64::EPSILON,
"{name}: expected {expected}, got {actual}"
),
(None, None) => {}
(actual, expected) => panic!("{name}: expected {expected:?}, got {actual:?}"),
}
}
}
#[test]
fn groups_scan_paths() {
let targets = vec![
BenchTarget {
relative_path: "data/a.parquet".to_string(),
+62 -6
View File
@@ -242,18 +242,22 @@ mod tests {
use common_error::ext::ErrorExt;
use common_query::prelude::{greptime_timestamp, greptime_value};
use common_recordbatch::RecordBatches;
use datafusion::parquet::basic::Encoding;
use datatypes::arrow::array::{Float64Array, StringArray, TimestampMillisecondArray};
use datatypes::arrow::datatypes::{DataType, Field, Schema as ArrowSchema, TimeUnit};
use datatypes::arrow::record_batch::RecordBatch;
use mito2::config::MitoConfig;
use mito2::sst::parquet::metadata::MetadataLoader;
use mito2::sst::parquet::reader::MetadataCacheMetrics;
use store_api::metric_engine_consts::{
METRIC_ENGINE_NAME, PHYSICAL_TABLE_METADATA_KEY, PRIMARY_KEY_ENCODING,
};
use store_api::mito_engine_options::EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING;
use store_api::path_utils::table_dir;
use store_api::region_engine::RegionEngine;
use store_api::region_request::{
PathType, RegionBulkInsertsRequest, RegionCloseRequest, RegionOpenRequest,
RegionPutRequest, RegionRequest,
PathType, RegionBulkInsertsRequest, RegionCloseRequest, RegionFlushRequest,
RegionOpenRequest, RegionPutRequest, RegionRequest,
};
use store_api::storage::{RegionId, ScanRequest};
@@ -611,16 +615,68 @@ mod tests {
#[tokio::test]
async fn test_bulk_insert_sparse_encoding() {
let env = TestEnv::new().await;
env.init_metric_region().await;
let physical_region_id = env.default_physical_region_id();
env.create_physical_region(
physical_region_id,
&TestEnv::default_table_dir(),
vec![(
EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING.to_string(),
"byte_stream_split".to_string(),
)],
)
.await;
let logical_region_id = env.default_logical_region_id();
env.create_logical_region(physical_region_id, logical_region_id)
.await;
let request = build_bulk_request(logical_region_id, build_logical_batch(0, 4), false);
let rows = 4;
let request = build_bulk_request(logical_region_id, build_logical_batch(0, rows), false);
let response = env
.metric()
.handle_request(logical_region_id, request)
.await
.unwrap();
assert_eq!(response.affected_rows, 4);
assert_eq!(response.affected_rows, rows);
let data_region_id = crate::utils::to_data_region_id(physical_region_id);
env.mito()
.handle_request(
data_region_id,
RegionRequest::Flush(RegionFlushRequest::default()),
)
.await
.unwrap();
let region = env.mito().find_region(data_region_id).unwrap();
let entry = region
.manifest_sst_entries()
.await
.into_iter()
.find(|entry| entry.visible && entry.file_path.ends_with(".parquet"))
.unwrap();
let mut cache_metrics = MetadataCacheMetrics::default();
let footer = MetadataLoader::new(
region.access_layer().object_store().clone(),
&entry.file_path,
entry.file_size,
)
.load(&mut cache_metrics)
.await
.unwrap();
let field_column_index = footer
.file_metadata()
.schema_descr()
.columns()
.iter()
.position(|column| column.name() == greptime_value())
.unwrap();
assert!(!footer.row_groups().is_empty());
assert!(footer.row_groups().iter().all(|row_group| {
row_group
.column(field_column_index)
.encodings()
.any(|encoding| encoding == Encoding::BYTE_STREAM_SPLIT)
}));
let stream = env
.metric()
@@ -628,7 +684,7 @@ mod tests {
.await
.unwrap();
let batches = RecordBatches::try_collect(stream).await.unwrap();
assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), 4);
assert_eq!(batches.iter().map(|b| b.num_rows()).sum::<usize>(), rows);
}
#[tokio::test]
+1
View File
@@ -655,6 +655,7 @@ where
write_buffer_size: compaction_region.engine_config.sst_write_buffer_size,
max_file_size: picker_output.max_file_size,
row_group_size: compaction_region.region_options.row_group_size(),
float_field_encoding: compaction_region.region_options.float_field_encoding,
};
let merger = self.merger.clone();
let compaction_region = compaction_region.clone();
+1
View File
@@ -337,6 +337,7 @@ mod tests {
primary_key_encoding: None,
write_buffer_size: None,
preserve_row_sequence: false,
float_field_encoding: Default::default(),
},
compaction_time_window: None,
}
+36 -1
View File
@@ -29,6 +29,7 @@ use datatypes::prelude::ConcreteDataType;
use datatypes::schema::ColumnSchema;
use futures::TryStreamExt;
use itertools::Itertools;
use parquet::basic::{Encoding, Type as PhysicalType};
use rstest::rstest;
use rstest_reuse::{self, apply};
use store_api::metadata::ColumnMetadata;
@@ -826,7 +827,9 @@ async fn test_engine_with_write_cache_with_format(flat_format: bool) {
let engine = env.create_engine(mito_config).await;
let region_id = RegionId::new(1, 1);
let request = CreateRequestBuilder::new().build();
let request = CreateRequestBuilder::new()
.insert_option("experimental_sst_float_field_encoding", "byte_stream_split")
.build();
let column_schemas = rows_schema(&request);
engine
@@ -842,6 +845,38 @@ async fn test_engine_with_write_cache_with_format(flat_format: bool) {
flush_region(&engine, region_id, None).await;
let region = engine.get_region(region_id).unwrap();
let file = region
.version()
.ssts
.levels()
.iter()
.flat_map(|level| level.files.values())
.next()
.expect("write-cache flushed SST")
.clone();
let reader = region
.access_layer
.read_sst(file)
.build()
.await
.unwrap()
.expect("write-cache flushed SST reader");
assert!(
reader
.parquet_metadata()
.row_groups()
.iter()
.flat_map(|row_group| row_group.columns())
.any(|column| {
column.column_path().string() == "field_0"
&& column.column_type() == PhysicalType::DOUBLE
&& column
.encodings()
.any(|encoding| encoding == Encoding::BYTE_STREAM_SPLIT)
})
);
let request = ScanRequest::default();
let scanner = engine.scanner(region_id, request).await.unwrap();
+31
View File
@@ -26,6 +26,7 @@ use common_recordbatch::{RecordBatches, SendableRecordBatchStream};
use common_time::Timestamp;
use datatypes::arrow::array::AsArray;
use datatypes::arrow::datatypes::TimestampMillisecondType;
use parquet::basic::{Encoding, Type as PhysicalType};
use store_api::region_engine::{RegionEngine, RegionRole};
use store_api::region_request::AlterKind::SetRegionOptions;
use store_api::region_request::{
@@ -954,6 +955,7 @@ async fn test_compaction_region_with_format(flat_format: bool) {
let request = CreateRequestBuilder::new()
.insert_option("compaction.type", "twcs")
.insert_option("experimental_sst_float_field_encoding", "byte_stream_split")
.build();
let column_schemas = request
@@ -974,6 +976,35 @@ async fn test_compaction_region_with_format(flat_format: bool) {
compact(&engine, region_id).await;
let region = engine.get_region(region_id).unwrap();
let file = region.version().ssts.levels()[1]
.files
.values()
.next()
.expect("compaction output SST")
.clone();
let reader = region
.access_layer
.read_sst(file)
.build()
.await
.unwrap()
.expect("compaction output SST reader");
assert!(
reader
.parquet_metadata()
.row_groups()
.iter()
.flat_map(|row_group| row_group.columns())
.any(|column| {
column.column_path().string() == "field_0"
&& column.column_type() == PhysicalType::DOUBLE
&& column
.encodings()
.any(|encoding| encoding == Encoding::BYTE_STREAM_SPLIT)
})
);
let scanner = engine
.scanner(region_id, ScanRequest::default())
.await
+36 -1
View File
@@ -27,6 +27,7 @@ use common_base::readable_size::ReadableSize;
use common_recordbatch::RecordBatches;
use common_time::util::current_time_millis;
use common_wal::options::{KafkaWalOptions, WAL_OPTIONS_KEY, WalOptions};
use parquet::basic::{Encoding, Type as PhysicalType};
use rstest::rstest;
use rstest_reuse::{self, apply};
use store_api::ManifestVersion;
@@ -279,7 +280,9 @@ async fn test_manual_flush_with_format(flat_format: bool) {
)
.await;
let request = CreateRequestBuilder::new().build();
let request = CreateRequestBuilder::new()
.insert_option("experimental_sst_float_field_encoding", "byte_stream_split")
.build();
let column_schemas = rows_schema(&request);
engine
@@ -295,6 +298,38 @@ async fn test_manual_flush_with_format(flat_format: bool) {
flush_region(&engine, region_id, None).await;
let region = engine.get_region(region_id).unwrap();
let file = region
.version()
.ssts
.levels()
.iter()
.flat_map(|level| level.files.values())
.next()
.expect("flushed SST")
.clone();
let reader = region
.access_layer
.read_sst(file)
.build()
.await
.unwrap()
.expect("flushed SST reader");
assert!(
reader
.parquet_metadata()
.row_groups()
.iter()
.flat_map(|row_group| row_group.columns())
.any(|column| {
column.column_path().string() == "field_0"
&& column.column_type() == PhysicalType::DOUBLE
&& column
.encodings()
.any(|encoding| encoding == Encoding::BYTE_STREAM_SPLIT)
})
);
let request = ScanRequest::default();
let scanner = engine.scanner(region_id, request).await.unwrap();
assert_eq!(0, scanner.num_memtables());
+1
View File
@@ -451,6 +451,7 @@ impl RegionFlushTask {
let mut write_opts = WriteOptions {
write_buffer_size: self.engine_config.sst_write_buffer_size,
float_field_encoding: version.options.float_field_encoding,
..Default::default()
};
if let Some(row_group_size) = self.row_group_size {
+2
View File
@@ -491,6 +491,7 @@ impl MemtableBuilderProvider {
BulkMemtableBuilder::new(self.write_buffer_manager.clone(), !dedup, merge_mode)
.with_config(config.clone())
.with_row_group_size(options.row_group_size())
.with_float_field_encoding(options.float_field_encoding)
.with_compact_dispatcher(self.compact_dispatcher.clone()),
),
Some(MemtableOptions::TimeSeries) => Arc::new(TimeSeriesMemtableBuilder::new(
@@ -515,6 +516,7 @@ impl MemtableBuilderProvider {
)
.with_config(self.default_bulk_memtable_config.clone())
.with_row_group_size(options.row_group_size())
.with_float_field_encoding(options.float_field_encoding)
.with_compact_dispatcher(self.compact_dispatcher.clone());
if let Some(MemtableOptions::Bulk(config)) = &options.memtable {
+135 -25
View File
@@ -40,6 +40,7 @@ use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use serde_with::{DisplayFromStr, serde_as};
use store_api::metadata::RegionMetadataRef;
use store_api::mito_engine_options::FloatFieldEncoding;
use store_api::storage::{ColumnId, FileId, RegionId, SequenceRange};
use tokio::sync::Semaphore;
@@ -435,6 +436,7 @@ pub struct BulkMemtable {
merge_mode: MergeMode,
/// Max number of rows in a parquet row group for encoded parts.
row_group_size: usize,
float_field_encoding: FloatFieldEncoding,
}
impl std::fmt::Debug for BulkMemtable {
@@ -665,27 +667,7 @@ impl Memtable for BulkMemtable {
}
fn fork(&self, id: MemtableId, metadata: &RegionMetadataRef) -> MemtableRef {
Arc::new(Self {
id,
config: self.config.clone(),
parts: Arc::new(RwLock::new(BulkParts::default())),
metadata: metadata.clone(),
alloc_tracker: AllocTracker::new(self.alloc_tracker.write_buffer_manager()),
max_timestamp: AtomicI64::new(i64::MIN),
min_timestamp: AtomicI64::new(i64::MAX),
max_sequence: AtomicU64::new(0),
num_rows: AtomicUsize::new(0),
compactor: Arc::new(Mutex::new(MemtableCompactor::new(
metadata.region_id,
id,
self.config.clone(),
self.row_group_size,
))),
compact_dispatcher: self.compact_dispatcher.clone(),
append_mode: self.append_mode,
merge_mode: self.merge_mode,
row_group_size: self.row_group_size,
})
self.fork_inner(id, metadata)
}
fn compact(&self, for_flush: bool) -> Result<()> {
@@ -714,6 +696,32 @@ impl Memtable for BulkMemtable {
}
impl BulkMemtable {
fn fork_inner(&self, id: MemtableId, metadata: &RegionMetadataRef) -> Arc<BulkMemtable> {
Arc::new(Self {
id,
config: self.config.clone(),
parts: Arc::new(RwLock::new(BulkParts::default())),
metadata: metadata.clone(),
alloc_tracker: AllocTracker::new(self.alloc_tracker.write_buffer_manager()),
max_timestamp: AtomicI64::new(i64::MIN),
min_timestamp: AtomicI64::new(i64::MAX),
max_sequence: AtomicU64::new(0),
num_rows: AtomicUsize::new(0),
compactor: Arc::new(Mutex::new(MemtableCompactor::new(
metadata.region_id,
id,
self.config.clone(),
self.row_group_size,
self.float_field_encoding,
))),
compact_dispatcher: self.compact_dispatcher.clone(),
append_mode: self.append_mode,
merge_mode: self.merge_mode,
row_group_size: self.row_group_size,
float_field_encoding: self.float_field_encoding,
})
}
/// Creates a new BulkMemtable with the default row group size.
pub fn new(
id: MemtableId,
@@ -747,6 +755,32 @@ impl BulkMemtable {
append_mode: bool,
merge_mode: MergeMode,
row_group_size: usize,
) -> Self {
Self::new_with_row_group_size_and_encoding(
id,
config,
metadata,
write_buffer_manager,
compact_dispatcher,
append_mode,
merge_mode,
row_group_size,
FloatFieldEncoding::default(),
)
}
/// Creates a new BulkMemtable with the given row group size and float encoding.
#[allow(clippy::too_many_arguments)]
fn new_with_row_group_size_and_encoding(
id: MemtableId,
config: BulkMemtableConfig,
metadata: RegionMetadataRef,
write_buffer_manager: Option<WriteBufferManagerRef>,
compact_dispatcher: Option<Arc<CompactDispatcher>>,
append_mode: bool,
merge_mode: MergeMode,
row_group_size: usize,
float_field_encoding: FloatFieldEncoding,
) -> Self {
let config = config.sanitize();
let region_id = metadata.region_id;
@@ -765,11 +799,13 @@ impl BulkMemtable {
id,
config,
row_group_size,
float_field_encoding,
))),
compact_dispatcher,
append_mode,
merge_mode,
row_group_size,
float_field_encoding,
}
}
@@ -1179,6 +1215,7 @@ struct MemtableCompactor {
config: BulkMemtableConfig,
/// Max number of rows in a parquet row group for encoded parts.
row_group_size: usize,
float_field_encoding: FloatFieldEncoding,
}
impl MemtableCompactor {
@@ -1188,12 +1225,14 @@ impl MemtableCompactor {
memtable_id: MemtableId,
config: BulkMemtableConfig,
row_group_size: usize,
float_field_encoding: FloatFieldEncoding,
) -> Self {
Self {
region_id,
memtable_id,
config,
row_group_size,
float_field_encoding,
}
}
@@ -1233,6 +1272,7 @@ impl MemtableCompactor {
let encode_row_threshold = self.config.encode_row_threshold;
let encode_bytes_threshold = self.config.encode_bytes_threshold;
let row_group_size = self.row_group_size;
let float_field_encoding = self.float_field_encoding;
// Merge all groups in parallel
let merged_parts = collected
@@ -1247,6 +1287,7 @@ impl MemtableCompactor {
encode_row_threshold,
encode_bytes_threshold,
row_group_size,
float_field_encoding,
)
})
.collect::<Result<Vec<Option<MergedPart>>>>()?;
@@ -1282,6 +1323,7 @@ impl MemtableCompactor {
encode_row_threshold: usize,
encode_bytes_threshold: usize,
row_group_size: usize,
float_field_encoding: FloatFieldEncoding,
) -> Result<Option<MergedPart>> {
if parts_to_merge.is_empty() {
return Ok(None);
@@ -1382,7 +1424,11 @@ impl MemtableCompactor {
if estimated_total_rows > encode_row_threshold
|| estimated_total_bytes > encode_bytes_threshold
{
let encoder = BulkPartEncoder::new(metadata.clone(), row_group_size)?;
let encoder = BulkPartEncoder::new_with_float_field_encoding(
metadata.clone(),
row_group_size,
float_field_encoding,
)?;
let mut metrics = BulkPartEncodeMetrics::default();
let encoded_part = encoder.encode_record_batch_iter(
boxed_iter,
@@ -1508,6 +1554,7 @@ pub struct BulkMemtableBuilder {
merge_mode: MergeMode,
/// Max number of rows in a parquet row group for encoded parts.
row_group_size: usize,
float_field_encoding: FloatFieldEncoding,
}
impl Default for BulkMemtableBuilder {
@@ -1519,6 +1566,7 @@ impl Default for BulkMemtableBuilder {
append_mode: false,
merge_mode: MergeMode::default(),
row_group_size: DEFAULT_ROW_GROUP_SIZE,
float_field_encoding: FloatFieldEncoding::default(),
}
}
}
@@ -1550,6 +1598,11 @@ impl BulkMemtableBuilder {
self
}
pub(super) fn with_float_field_encoding(mut self, encoding: FloatFieldEncoding) -> Self {
self.float_field_encoding = encoding;
self
}
/// Sets the compact dispatcher.
pub fn with_compact_dispatcher(mut self, compact_dispatcher: Arc<CompactDispatcher>) -> Self {
self.compact_dispatcher = Some(compact_dispatcher);
@@ -1564,7 +1617,7 @@ impl BulkMemtableBuilder {
impl MemtableBuilder for BulkMemtableBuilder {
fn build(&self, id: MemtableId, metadata: &RegionMetadataRef) -> MemtableRef {
Arc::new(BulkMemtable::new_with_row_group_size(
Arc::new(BulkMemtable::new_with_row_group_size_and_encoding(
id,
self.config.clone(),
metadata.clone(),
@@ -1573,6 +1626,7 @@ impl MemtableBuilder for BulkMemtableBuilder {
self.append_mode,
self.merge_mode,
self.row_group_size,
self.float_field_encoding,
))
}
@@ -1597,6 +1651,7 @@ mod tests {
use datatypes::schema::ColumnSchema;
use datatypes::types::json_type::{JsonNativeType, JsonObjectType};
use mito_codec::row_converter::build_primary_key_codec;
use parquet::basic::{Encoding, Type as PhysicalType};
use serde_json::json;
use store_api::metadata::{ColumnMetadata, RegionMetadataBuilder, RegionMetadataRef};
@@ -1829,6 +1884,7 @@ mod tests {
usize::MAX,
usize::MAX,
DEFAULT_ROW_GROUP_SIZE,
FloatFieldEncoding::default(),
)?
.unwrap();
let MergedPart::Multi(part) = merged else {
@@ -2143,7 +2199,7 @@ mod tests {
#[test]
fn test_bulk_memtable_fork() {
let metadata = metadata_for_test();
let original_memtable = BulkMemtable::new(
let original_memtable = BulkMemtable::new_with_row_group_size_and_encoding(
333,
BulkMemtableConfig::default(),
metadata.clone(),
@@ -2151,6 +2207,8 @@ mod tests {
None,
false,
MergeMode::LastRow,
DEFAULT_ROW_GROUP_SIZE,
FloatFieldEncoding::ByteStreamSplit,
);
let bulk_part =
@@ -2159,9 +2217,25 @@ mod tests {
original_memtable.write_bulk(bulk_part).unwrap();
let forked_memtable = original_memtable.fork(444, &metadata);
let forked_memtable = original_memtable.fork_inner(444, &metadata);
assert_eq!(forked_memtable.id(), 444);
assert_eq!(
FloatFieldEncoding::ByteStreamSplit,
original_memtable.float_field_encoding
);
assert_eq!(
original_memtable.float_field_encoding,
forked_memtable.float_field_encoding
);
assert_eq!(
original_memtable.float_field_encoding,
forked_memtable
.compactor
.lock()
.unwrap()
.float_field_encoding
);
assert!(forked_memtable.is_empty());
assert_eq!(0, forked_memtable.stats().num_rows);
@@ -2780,6 +2854,42 @@ mod tests {
assert!(!BulkParts::is_merge_candidate_by_size(true, 9, 8));
}
#[test]
fn test_bulk_part_encoder_preserves_byte_stream_split() -> WhateverResult<()> {
let metadata = metadata_for_test();
let bulk_part = create_bulk_part_with_converter(
"byte_stream_split",
0,
vec![1000, 2000],
vec![Some(10.0), Some(20.0)],
100,
)?;
let encoder = BulkPartEncoder::new_with_float_field_encoding(
metadata,
DEFAULT_ROW_GROUP_SIZE,
FloatFieldEncoding::ByteStreamSplit,
)?;
let encoded_part = encoder.encode_part(&bulk_part)?.unwrap();
let field_column = encoded_part
.metadata()
.parquet_metadata
.row_groups()
.iter()
.flat_map(|row_group| row_group.columns())
.find(|column| {
column.column_path().string() == "v1"
&& column.column_type() == PhysicalType::DOUBLE
})
.expect("Float64 field should be present in encoded bulk part");
assert!(
field_column
.encodings()
.any(|encoding| encoding == Encoding::BYTE_STREAM_SPLIT)
);
Ok(())
}
#[test]
fn test_encoded_part_batch_size_uses_largest_uncompressed_row_group() {
const NUM_ROWS: usize = 14;
+19 -11
View File
@@ -53,6 +53,7 @@ use smallvec::SmallVec;
use snafu::{OptionExt, ResultExt};
use store_api::codec::PrimaryKeyEncoding;
use store_api::metadata::{RegionMetadata, RegionMetadataRef};
use store_api::mito_engine_options::FloatFieldEncoding;
use store_api::storage::consts::PRIMARY_KEY_COLUMN_NAME;
use store_api::storage::{ColumnId, FileId, SequenceNumber, SequenceRange};
@@ -70,7 +71,7 @@ use crate::sst::SeriesEstimator;
use crate::sst::index::IndexOutput;
use crate::sst::parquet::flat_format::primary_key_column_index;
use crate::sst::parquet::format::{PrimaryKeyArray, PrimaryKeyArrayBuilder};
use crate::sst::parquet::{PARQUET_METADATA_KEY, SstInfo};
use crate::sst::parquet::{PARQUET_METADATA_KEY, SstInfo, apply_float_field_encoding};
const INIT_DICT_VALUE_CAPACITY: usize = 8;
@@ -1306,22 +1307,29 @@ pub struct BulkPartEncoder {
impl BulkPartEncoder {
pub fn new(metadata: RegionMetadataRef, row_group_size: usize) -> Result<BulkPartEncoder> {
Self::new_with_float_field_encoding(metadata, row_group_size, FloatFieldEncoding::default())
}
pub(super) fn new_with_float_field_encoding(
metadata: RegionMetadataRef,
row_group_size: usize,
float_field_encoding: FloatFieldEncoding,
) -> Result<BulkPartEncoder> {
// TODO(yingwen): Skip arrow schema if needed.
let json = metadata.to_json().context(InvalidMetadataSnafu)?;
let key_value_meta =
parquet::file::metadata::KeyValue::new(PARQUET_METADATA_KEY.to_string(), json);
// TODO(yingwen): Do we need compression?
let writer_props = Some(
WriterProperties::builder()
.set_key_value_metadata(Some(vec![key_value_meta]))
.set_write_batch_size(row_group_size)
.set_max_row_group_row_count(Some(row_group_size))
.set_compression(Compression::ZSTD(ZstdLevel::default()))
.set_column_index_truncate_length(None)
.set_statistics_truncate_length(None)
.build(),
);
let mut props = WriterProperties::builder()
.set_key_value_metadata(Some(vec![key_value_meta]))
.set_write_batch_size(row_group_size)
.set_max_row_group_row_count(Some(row_group_size))
.set_compression(Compression::ZSTD(ZstdLevel::default()))
.set_column_index_truncate_length(None)
.set_statistics_truncate_length(None);
props = apply_float_field_encoding(props, &metadata, float_field_encoding);
let writer_props = Some(props.build());
Ok(Self {
metadata,
+40 -2
View File
@@ -32,7 +32,9 @@ use store_api::codec::PrimaryKeyEncoding;
use store_api::metric_engine_consts::{
MEMTABLE_PARTITION_TREE_PRIMARY_KEY_ENCODING, PRIMARY_KEY_ENCODING,
};
use store_api::mito_engine_options::{COMPACTION_OVERRIDE, MAX_ROW_GROUP_ROW_COUNT_LIMIT};
use store_api::mito_engine_options::{
COMPACTION_OVERRIDE, FloatFieldEncoding, MAX_ROW_GROUP_ROW_COUNT_LIMIT,
};
use store_api::storage::{ColumnId, RegionId};
use strum::EnumString;
@@ -128,6 +130,9 @@ pub struct RegionOptions {
/// sequence metadata.
#[serde(default, skip_serializing_if = "is_false")]
pub preserve_row_sequence: bool,
/// Encoding for direct floating-point field columns in Parquet SSTs.
#[serde(default)]
pub float_field_encoding: FloatFieldEncoding,
}
fn is_false(value: &bool) -> bool {
@@ -287,6 +292,8 @@ impl RegionOptions {
sst_format = Some(FormatType::Flat);
}
let float_field_encoding = options.float_field_encoding.unwrap_or_default();
let compaction_override_flag = options_map
.get(COMPACTION_OVERRIDE)
.map(|v| matches!(v.to_lowercase().as_str(), "true" | "1"))
@@ -322,6 +329,7 @@ impl RegionOptions {
primary_key_encoding,
write_buffer_size: options.write_buffer_size,
preserve_row_sequence: options.preserve_row_sequence,
float_field_encoding,
};
opts.validate()?;
@@ -469,6 +477,9 @@ struct RegionOptionsWithoutEnum {
max_row_group_row_count: Option<usize>,
#[serde_as(as = "DisplayFromStr")]
preserve_row_sequence: bool,
#[serde(rename = "experimental_sst_float_field_encoding")]
#[serde_as(as = "NoneAsEmptyString")]
float_field_encoding: Option<FloatFieldEncoding>,
}
impl Default for RegionOptionsWithoutEnum {
@@ -485,6 +496,7 @@ impl Default for RegionOptionsWithoutEnum {
sst_format: options.sst_format,
max_row_group_row_count: options.max_row_group_row_count,
preserve_row_sequence: options.preserve_row_sequence,
float_field_encoding: Some(options.float_field_encoding),
}
}
}
@@ -628,7 +640,9 @@ mod tests {
use common_error::ext::ErrorExt;
use common_error::status_code::StatusCode;
use common_wal::options::KafkaWalOptions;
use store_api::mito_engine_options::{SKIP_WAL_KEY, WRITE_BUFFER_SIZE_KEY};
use store_api::mito_engine_options::{
EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, SKIP_WAL_KEY, WRITE_BUFFER_SIZE_KEY,
};
use super::*;
@@ -646,6 +660,26 @@ mod tests {
assert_eq!(RegionOptions::default(), options);
}
#[test]
fn test_float_field_encoding_defaults_and_parses() {
let options = RegionOptions::try_from_options(RegionId::new(0, 0), &make_map(&[])).unwrap();
assert_eq!(FloatFieldEncoding::Default, options.float_field_encoding);
let map = make_map(&[(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, "default")]);
let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
assert_eq!(FloatFieldEncoding::Default, options.float_field_encoding);
let map = make_map(&[(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, "byte_stream_split")]);
let options = RegionOptions::try_from_options(RegionId::new(0, 0), &map).unwrap();
assert_eq!(
FloatFieldEncoding::ByteStreamSplit,
options.float_field_encoding
);
let map = make_map(&[(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, "invalid")]);
assert!(RegionOptions::try_from_options(RegionId::new(0, 0), &map).is_err());
}
#[test]
fn test_with_ttl() {
let map = make_map(&[("ttl", "7d")]);
@@ -1103,6 +1137,7 @@ mod tests {
primary_key_encoding: None,
write_buffer_size: None,
preserve_row_sequence: false,
float_field_encoding: FloatFieldEncoding::default(),
};
assert_eq!(expect, options);
}
@@ -1161,6 +1196,7 @@ mod tests {
primary_key_encoding: None,
write_buffer_size: Some(ReadableSize::mb(128)),
preserve_row_sequence: true,
float_field_encoding: FloatFieldEncoding::default(),
};
let region_options_json_str = serde_json::to_string(&options).unwrap();
assert!(region_options_json_str.contains("preserve_row_sequence"));
@@ -1172,6 +1208,7 @@ mod tests {
let got: RegionOptions = serde_json::from_str(old_region_options_json_str).unwrap();
assert_eq!(None, got.write_buffer_size);
assert!(!got.preserve_row_sequence);
assert_eq!(FloatFieldEncoding::Default, got.float_field_encoding);
let CompactionOptions::Twcs(twcs) = got.compaction;
assert_eq!(16, twcs.active_window_l1_merge_trigger);
assert_eq!(8, twcs.inactive_window_l1_merge_trigger);
@@ -1238,6 +1275,7 @@ mod tests {
primary_key_encoding: None,
write_buffer_size: None,
preserve_row_sequence: false,
float_field_encoding: FloatFieldEncoding::default(),
};
assert_eq!(options, got);
}
+208 -4
View File
@@ -17,9 +17,14 @@
use std::collections::BTreeMap;
use std::sync::Arc;
use api::v1::SemanticType;
use common_base::readable_size::ReadableSize;
use datatypes::json::JsonSettings;
use parquet::file::metadata::ParquetMetaData;
use parquet::file::properties::WriterPropertiesBuilder;
use parquet::schema::types::ColumnPath;
use store_api::metadata::RegionMetadataRef;
use store_api::mito_engine_options::FloatFieldEncoding;
use store_api::storage::{ColumnId, FileId};
use crate::sst::DEFAULT_WRITE_BUFFER_SIZE;
@@ -72,6 +77,27 @@ pub(crate) struct Json2TargetLayout {
/// batching without changing the row group layout of newly written SSTs.
pub const DEFAULT_ROW_GROUP_SIZE: usize = 100 * 1024;
/// Applies the configured encoding to direct floating-point field columns.
pub(crate) fn apply_float_field_encoding(
mut builder: WriterPropertiesBuilder,
metadata: &RegionMetadataRef,
encoding: FloatFieldEncoding,
) -> WriterPropertiesBuilder {
if encoding == FloatFieldEncoding::ByteStreamSplit {
for column in &metadata.column_metadatas {
if column.semantic_type == SemanticType::Field
&& column.column_schema.data_type.is_float()
{
let path = ColumnPath::new(vec![column.column_schema.name.clone()]);
builder = builder
.set_column_encoding(path.clone(), parquet::basic::Encoding::BYTE_STREAM_SPLIT)
.set_column_dictionary_enabled(path, false);
}
}
}
builder
}
/// Parquet write options.
#[derive(Debug, Clone)]
pub struct WriteOptions {
@@ -83,6 +109,8 @@ pub struct WriteOptions {
/// Note: This is not a hard limit as we can only observe the file size when
/// ArrowWrite writes to underlying writers.
pub max_file_size: Option<usize>,
/// Encoding policy for direct floating-point field columns.
pub float_field_encoding: FloatFieldEncoding,
}
impl Default for WriteOptions {
@@ -91,6 +119,7 @@ impl Default for WriteOptions {
write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
row_group_size: DEFAULT_ROW_GROUP_SIZE,
max_file_size: None,
float_field_encoding: FloatFieldEncoding::default(),
}
}
}
@@ -125,6 +154,7 @@ mod tests {
use std::sync::Arc;
use api::v1::{OpType, SemanticType};
use bytes::Bytes;
use common_function::function::FunctionRef;
use common_function::function_factory::ScalarFunctionFactory;
use common_function::scalars::matches::MatchesFunction;
@@ -135,20 +165,24 @@ mod tests {
use datafusion_expr::{BinaryExpr, Expr, Literal, Operator, col, lit};
use datatypes::arrow;
use datatypes::arrow::array::{
ArrayRef, AsArray, BinaryDictionaryBuilder, RecordBatch, StringArray,
StringDictionaryBuilder, TimestampMillisecondArray, UInt8Array, UInt64Array,
Array, ArrayRef, AsArray, BinaryDictionaryBuilder, Float32Array, Float64Array, Int32Array,
RecordBatch, StringArray, StringDictionaryBuilder, TimestampMillisecondArray, UInt8Array,
UInt64Array,
};
use datatypes::arrow::datatypes::{DataType, Field, Schema, UInt32Type};
use datatypes::arrow::datatypes::{DataType, Field, Schema, TimeUnit, UInt32Type};
use datatypes::arrow::util::pretty::pretty_format_batches;
use datatypes::prelude::ConcreteDataType;
use datatypes::schema::{FulltextAnalyzer, FulltextBackend, FulltextOptions};
use object_store::ObjectStore;
use parquet::arrow::AsyncArrowWriter;
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use parquet::arrow::{ArrowWriter, AsyncArrowWriter};
use parquet::basic::{Compression, Encoding, ZstdLevel};
use parquet::file::metadata::{KeyValue, PageIndexPolicy};
use parquet::file::properties::WriterProperties;
use parquet::schema::types::ColumnPath;
use store_api::codec::PrimaryKeyEncoding;
use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataBuilder};
use store_api::mito_engine_options::FloatFieldEncoding;
use store_api::region_request::PathType;
use store_api::storage::{ColumnSchema, RegionId};
use table::predicate::Predicate;
@@ -187,6 +221,176 @@ mod tests {
const FILE_DIR: &str = "/";
const REGION_ID: RegionId = RegionId::new(0, 0);
#[test]
fn test_float_field_encoding_properties_and_roundtrip() {
let mut metadata_builder = RegionMetadataBuilder::new(REGION_ID);
metadata_builder
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new("f32", ConcreteDataType::float32_datatype(), true),
semantic_type: SemanticType::Field,
column_id: 0,
})
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new("f64", ConcreteDataType::float64_datatype(), true),
semantic_type: SemanticType::Field,
column_id: 1,
})
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new("tag", ConcreteDataType::float32_datatype(), true),
semantic_type: SemanticType::Tag,
column_id: 2,
})
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new("i32", ConcreteDataType::int32_datatype(), true),
semantic_type: SemanticType::Field,
column_id: 3,
})
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new(
"ts",
ConcreteDataType::timestamp_millisecond_datatype(),
false,
),
semantic_type: SemanticType::Timestamp,
column_id: 4,
});
metadata_builder.primary_key(vec![2]);
let metadata = Arc::new(metadata_builder.build().unwrap());
let f32_values = [
Some(1.5_f32),
Some(2.0),
Some(0.0),
Some(-0.0),
Some(f32::INFINITY),
Some(f32::from_bits(0x7fc0_1234)),
None,
];
let f64_values = [
Some(1.5_f64),
Some(2.0),
Some(0.0),
Some(-0.0),
Some(f64::NEG_INFINITY),
Some(f64::from_bits(0x7ff8_0000_0000_1234)),
None,
];
let schema = Arc::new(Schema::new(vec![
Field::new("f32", DataType::Float32, true),
Field::new("f64", DataType::Float64, true),
Field::new("tag", DataType::Float32, true),
Field::new("i32", DataType::Int32, true),
Field::new(
"ts",
DataType::Timestamp(TimeUnit::Millisecond, None),
false,
),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Float32Array::from(f32_values.to_vec())) as ArrayRef,
Arc::new(Float64Array::from(f64_values.to_vec())) as ArrayRef,
Arc::new(Float32Array::from(vec![
Some(1.0),
None,
Some(-0.0),
Some(2.0),
Some(3.0),
Some(4.0),
None,
])),
Arc::new(Int32Array::from(vec![
Some(1),
None,
Some(3),
Some(4),
Some(5),
Some(6),
None,
])),
Arc::new(TimestampMillisecondArray::from_iter_values(0..7)),
],
)
.unwrap();
let path = |name: &str| ColumnPath::new(vec![name.to_string()]);
let bss = apply_float_field_encoding(
WriterProperties::builder(),
&metadata,
FloatFieldEncoding::ByteStreamSplit,
);
assert_eq!(
Some(Encoding::BYTE_STREAM_SPLIT),
bss.clone().build().encoding(&path("f32"))
);
assert_eq!(
Some(Encoding::BYTE_STREAM_SPLIT),
bss.clone().build().encoding(&path("f64"))
);
assert!(!bss.clone().build().dictionary_enabled(&path("f32")));
assert!(!bss.clone().build().dictionary_enabled(&path("f64")));
assert_eq!(None, bss.clone().build().encoding(&path("i32")));
assert!(bss.clone().build().dictionary_enabled(&path("tag")));
let default = apply_float_field_encoding(
WriterProperties::builder().set_encoding(Encoding::PLAIN),
&metadata,
FloatFieldEncoding::Default,
)
.build();
assert_eq!(Some(Encoding::PLAIN), default.encoding(&path("f32")));
assert!(default.dictionary_enabled(&path("f32")));
assert!(default.dictionary_enabled(&path("f64")));
let mut bytes = Vec::new();
let mut writer = ArrowWriter::try_new(&mut bytes, schema, Some(bss.build())).unwrap();
writer.write(&batch).unwrap();
let footer = writer.finish().unwrap();
drop(writer);
for name in ["f32", "f64"] {
let column = footer.row_groups()[0]
.columns()
.iter()
.find(|column| column.column_path().string() == name)
.unwrap();
assert!(
column
.encodings()
.any(|encoding| encoding == Encoding::BYTE_STREAM_SPLIT)
);
}
let mut reader = ParquetRecordBatchReaderBuilder::try_new(Bytes::from(bytes))
.unwrap()
.build()
.unwrap();
let actual = reader.next().unwrap().unwrap();
let actual_f32 = actual
.column(0)
.as_any()
.downcast_ref::<Float32Array>()
.unwrap();
let actual_f64 = actual
.column(1)
.as_any()
.downcast_ref::<Float64Array>()
.unwrap();
for (index, value) in f32_values.into_iter().enumerate() {
assert_eq!(value.is_none(), actual_f32.is_null(index));
if let Some(value) = value {
assert_eq!(value.to_bits(), actual_f32.value(index).to_bits());
}
}
for (index, value) in f64_values.into_iter().enumerate() {
assert_eq!(value.is_none(), actual_f64.is_null(index));
if let Some(value) = value {
assert_eq!(value.to_bits(), actual_f64.value(index).to_bits());
}
}
assert!(actual.column(2).is_null(1));
assert!(actual.column(3).is_null(1));
}
#[derive(Clone)]
struct FixedPathProvider {
region_file_id: RegionFileId,
+20 -27
View File
@@ -38,7 +38,7 @@ use object_store::{FuturesAsyncWriter, ObjectStore};
use parquet::arrow::AsyncArrowWriter;
use parquet::basic::{Compression, Encoding, ZstdLevel};
use parquet::file::metadata::KeyValue;
use parquet::file::properties::{WriterProperties, WriterPropertiesBuilder};
use parquet::file::properties::WriterProperties;
use parquet::schema::types::ColumnPath;
use smallvec::smallvec;
use snafu::{OptionExt, ResultExt};
@@ -61,7 +61,9 @@ use crate::sst::parquet::flat_format::{
FlatWriteFormat, primary_key_column_index, time_index_column_index,
};
use crate::sst::parquet::format::{PrimaryKeyArray, PrimaryKeyWriteFormat};
use crate::sst::parquet::{PARQUET_METADATA_KEY, SstInfo, WriteOptions};
use crate::sst::parquet::{
PARQUET_METADATA_KEY, SstInfo, WriteOptions, apply_float_field_encoding,
};
use crate::sst::{
DEFAULT_WRITE_BUFFER_SIZE, DEFAULT_WRITE_CONCURRENCY, FlatSchemaOptions, SeriesEstimator,
maybe_wrap_schema,
@@ -473,29 +475,6 @@ where
Ok(results)
}
/// Customizes per-column config according to schema and maybe column cardinality.
fn customize_column_config(
builder: WriterPropertiesBuilder,
region_metadata: &RegionMetadataRef,
) -> WriterPropertiesBuilder {
let ts_col = ColumnPath::new(vec![
region_metadata
.time_index_column()
.column_schema
.name
.clone(),
]);
let seq_col = ColumnPath::new(vec![SEQUENCE_COLUMN_NAME.to_string()]);
let op_type_col = ColumnPath::new(vec![OP_TYPE_COLUMN_NAME.to_string()]);
builder
.set_column_encoding(seq_col.clone(), Encoding::DELTA_BINARY_PACKED)
.set_column_dictionary_enabled(seq_col, false)
.set_column_encoding(ts_col.clone(), Encoding::DELTA_BINARY_PACKED)
.set_column_dictionary_enabled(ts_col, false)
.set_column_compression(op_type_col, Compression::UNCOMPRESSED)
}
async fn append_flat_batch(
&mut self,
batch: &RecordBatch,
@@ -564,8 +543,22 @@ where
.set_max_row_group_row_count(Some(opts.row_group_size))
.set_column_index_truncate_length(None)
.set_statistics_truncate_length(None);
let props_builder = Self::customize_column_config(props_builder, &self.metadata);
let ts_col = ColumnPath::new(vec![
self.metadata.time_index_column().column_schema.name.clone(),
]);
let seq_col = ColumnPath::new(vec![SEQUENCE_COLUMN_NAME.to_string()]);
let op_type_col = ColumnPath::new(vec![OP_TYPE_COLUMN_NAME.to_string()]);
let props_builder = props_builder
.set_column_encoding(seq_col.clone(), Encoding::DELTA_BINARY_PACKED)
.set_column_dictionary_enabled(seq_col, false)
.set_column_encoding(ts_col.clone(), Encoding::DELTA_BINARY_PACKED)
.set_column_dictionary_enabled(ts_col, false)
.set_column_compression(op_type_col, Compression::UNCOMPRESSED);
let props_builder = apply_float_field_encoding(
props_builder,
&self.metadata,
opts.float_field_encoding,
);
let writer_props = props_builder.build();
let sst_file_path = self.path_provider.build_sst_file_path(RegionFileId::new(
+46
View File
@@ -88,6 +88,31 @@ pub const MAX_ROW_GROUP_ROW_COUNT: &str = "max_row_group_row_count";
pub const MAX_ROW_GROUP_ROW_COUNT_LIMIT: usize = 10 * 1024 * 1024;
/// Option key for preserving per-row sequence numbers through flush and compaction.
pub const PRESERVE_ROW_SEQUENCE: &str = "preserve_row_sequence";
/// Option key for experimental Parquet float field encoding.
pub const EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING: &str = "experimental_sst_float_field_encoding";
/// Encoding policy for direct floating-point field columns in Parquet SSTs.
#[derive(
Debug,
Default,
Clone,
Copy,
PartialEq,
Eq,
Hash,
serde::Serialize,
serde::Deserialize,
strum::EnumString,
)]
#[serde(rename_all = "snake_case")]
#[strum(serialize_all = "snake_case")]
pub enum FloatFieldEncoding {
/// The existing Parquet writer behavior.
#[default]
Default,
/// Parquet byte-stream-split encoding.
ByteStreamSplit,
}
// Note: Adding new options here should also check if this option should be removed in [metric_engine::engine::create::region_options_for_metadata_region].
/// Conflicting values supplied through the legacy and canonical TWCS trigger options.
@@ -155,6 +180,7 @@ pub fn is_mito_engine_option_key(key: &str) -> bool {
SST_FORMAT_KEY,
MAX_ROW_GROUP_ROW_COUNT,
PRESERVE_ROW_SEQUENCE,
EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING,
]
.contains(&key)
}
@@ -217,9 +243,29 @@ mod tests {
assert!(is_mito_engine_option_key("append_mode"));
assert!(is_mito_engine_option_key("max_row_group_row_count"));
assert!(is_mito_engine_option_key("preserve_row_sequence"));
assert!(is_mito_engine_option_key(
EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING
));
assert!(!is_mito_engine_option_key("foo"));
}
#[test]
fn test_float_field_encoding_values() {
assert_eq!(
"default".parse::<FloatFieldEncoding>(),
Ok(FloatFieldEncoding::Default)
);
assert_eq!(
"byte_stream_split".parse::<FloatFieldEncoding>(),
Ok(FloatFieldEncoding::ByteStreamSplit)
);
assert_eq!(
serde_json::from_str::<FloatFieldEncoding>("\"byte_stream_split\"").unwrap(),
FloatFieldEncoding::ByteStreamSplit
);
assert!(serde_json::from_str::<FloatFieldEncoding>("\"unknown\"").is_err());
}
#[test]
fn test_normalize_twcs_trigger_aliases_to_legacy_key() {
let expected = HashMap::from([(TWCS_TRIGGER_FILE_NUM.to_string(), "4".to_string())]);
+34 -4
View File
@@ -36,10 +36,10 @@ use store_api::metric_engine_consts::{
LOGICAL_TABLE_METADATA_KEY, PHYSICAL_TABLE_METADATA_KEY, is_metric_engine_option_key,
};
use store_api::mito_engine_options::{
APPEND_MODE_KEY, COMPACTION_TYPE, MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD,
MEMTABLE_BULK_ENCODE_ROW_THRESHOLD, MEMTABLE_BULK_MAX_MERGE_GROUPS,
MEMTABLE_BULK_MERGE_THRESHOLD, MEMTABLE_TYPE, MERGE_MODE_KEY, SST_FORMAT_KEY,
TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER, TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM,
APPEND_MODE_KEY, COMPACTION_TYPE, EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, FloatFieldEncoding,
MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD, MEMTABLE_BULK_ENCODE_ROW_THRESHOLD,
MEMTABLE_BULK_MAX_MERGE_GROUPS, MEMTABLE_BULK_MERGE_THRESHOLD, MEMTABLE_TYPE, MERGE_MODE_KEY,
SST_FORMAT_KEY, TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER, TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM,
TWCS_FALLBACK_TO_LOCAL, TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER,
TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM, TWCS_MAX_OUTPUT_FILE_SIZE, TWCS_TIME_WINDOW,
TWCS_TRIGGER_FILE_NUM, is_mito_engine_option_key, normalize_twcs_trigger_options,
@@ -256,6 +256,16 @@ impl TableOptions {
options.ttl = Some(ttl_value);
}
if let Some(encoding) = kvs.get(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING) {
encoding.parse::<FloatFieldEncoding>().map_err(|_| {
ParseTableOptionSnafu {
key: EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING,
value: encoding,
}
.build()
})?;
}
if let Some(skip_wal) = kvs.get(SKIP_WAL_KEY) {
options.skip_wal = skip_wal.parse().map_err(|_| {
ParseTableOptionSnafu {
@@ -885,6 +895,7 @@ mod tests {
assert!(validate_table_option(WRITE_BUFFER_SIZE_KEY));
assert!(validate_table_option(STORAGE_KEY));
assert!(validate_table_option(MEMTABLE_BULK_MERGE_THRESHOLD));
assert!(validate_table_option(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING));
assert!(validate_table_option(REPARTITION_COLUMN_HINT_KEY));
assert!(validate_table_option(REPARTITION_PARTITION_NUM_HINT_KEY));
assert_eq!(AnnotationFamily::of_key("repartition.unknown.hint"), None);
@@ -923,6 +934,25 @@ mod tests {
assert!(!validate_database_option("foo"));
}
#[test]
fn test_parse_float_field_encoding_option() {
let options = TableOptions::try_from_iter([(
EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING,
"byte_stream_split",
)])
.unwrap();
assert_eq!(
options
.extra_options
.get(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING),
Some(&"byte_stream_split".to_string())
);
assert!(
TableOptions::try_from_iter([(EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING, "invalid",)])
.is_err()
);
}
#[test]
fn test_database_trigger_value_boundaries() {
let maximum = usize::MAX.to_string();
@@ -1,8 +1,8 @@
name = "downgrade_compatibility"
reason = "Verify v1.1.4 can reopen a table whose region WAL options were written by the current binary, and can read all rows of an append-only table flushed and compacted with preserve_row_sequence enabled."
reason = "Verify v1.1.4 can reopen tables whose region WAL options or byte-stream-split (BSS) float SSTs were written by the current binary, and can read all rows of an append-only table flushed and compacted with preserve_row_sequence enabled."
introduced_by = "fix: preserve legacy region WAL options format; preserve_row_sequence"
topologies = ["distributed", "standalone"]
from_range = [">=v1.2.0"]
to_range = ["=v1.1.4"]
features = ["table", "wal", "downgrade", "append", "preserve_row_sequence"]
features = ["table", "wal", "downgrade", "append", "preserve_row_sequence", "sst", "float", "byte_stream_split"]
owner = "metasrv"
@@ -32,3 +32,18 @@ INSERT INTO t_preserve_sequence_downgrade VALUES
ADMIN FLUSH_TABLE('t_preserve_sequence_downgrade');
ADMIN COMPACT_TABLE('t_preserve_sequence_downgrade');
CREATE TABLE t_sst_float_field_encoding_downgrade(
ts TIMESTAMP TIME INDEX,
host STRING PRIMARY KEY,
f FLOAT,
d DOUBLE
) ENGINE=mito
WITH('experimental_sst_float_field_encoding'='byte_stream_split');
INSERT INTO t_sst_float_field_encoding_downgrade VALUES
('2024-02-10 00:00:00+0000', 'host_a', 1.25, 10.5),
('2024-02-10 00:01:00+0000', 'host_b', -2.5, 20.25),
('2024-02-10 00:02:00+0000', 'host_c', NULL, NULL);
ADMIN FLUSH_TABLE('t_sst_float_field_encoding_downgrade');
@@ -17,3 +17,15 @@ SELECT ts, host, val FROM t_preserve_sequence_downgrade ORDER BY ts, host;
| 2024-02-09T00:02:00 | host_a | 3 |
| 2024-02-09T00:03:00 | host_c | 4 |
+---------------------+--------+-----+
SELECT ts, host, f, d
FROM t_sst_float_field_encoding_downgrade
ORDER BY ts, host;
+---------------------+--------+------+-------+
| ts | host | f | d |
+---------------------+--------+------+-------+
| 2024-02-10T00:00:00 | host_a | 1.25 | 10.5 |
| 2024-02-10T00:01:00 | host_b | -2.5 | 20.25 |
| 2024-02-10T00:02:00 | host_c | | |
+---------------------+--------+------+-------+
@@ -1,3 +1,7 @@
SELECT ts, host, val FROM t_downgrade_compatibility ORDER BY ts, host;
SELECT ts, host, val FROM t_preserve_sequence_downgrade ORDER BY ts, host;
SELECT ts, host, f, d
FROM t_sst_float_field_encoding_downgrade
ORDER BY ts, host;
@@ -0,0 +1,8 @@
name = "sst_float_field_encoding_upgrade"
reason = "Verify newer binaries can read default-encoded Mito SSTs containing FLOAT and DOUBLE fields written by older binaries."
introduced_by = "experimental_sst_float_field_encoding"
topologies = ["distributed", "standalone"]
from_range = ["*"]
to_range = ["*"]
features = ["table", "sst", "float", "upgrade"]
owner = "storage"
@@ -0,0 +1,13 @@
CREATE TABLE t_sst_float_field_encoding_upgrade(
ts TIMESTAMP TIME INDEX,
host STRING PRIMARY KEY,
f FLOAT,
d DOUBLE
) ENGINE=mito;
INSERT INTO t_sst_float_field_encoding_upgrade VALUES
('2024-02-10 00:00:00+0000', 'host_a', 1.25, 10.5),
('2024-02-10 00:01:00+0000', 'host_b', -2.5, 20.25),
('2024-02-10 00:02:00+0000', 'host_c', NULL, NULL);
ADMIN FLUSH_TABLE('t_sst_float_field_encoding_upgrade');
@@ -0,0 +1,11 @@
SELECT ts, host, f, d
FROM t_sst_float_field_encoding_upgrade
ORDER BY ts, host;
+---------------------+--------+------+-------+
| ts | host | f | d |
+---------------------+--------+------+-------+
| 2024-02-10T00:00:00 | host_a | 1.25 | 10.5 |
| 2024-02-10T00:01:00 | host_b | -2.5 | 20.25 |
| 2024-02-10T00:02:00 | host_c | | |
+---------------------+--------+------+-------+
@@ -0,0 +1,3 @@
SELECT ts, host, f, d
FROM t_sst_float_field_encoding_upgrade
ORDER BY ts, host;
+129 -20
View File
@@ -86,15 +86,19 @@ sample values without changing label cardinality or the runner lifecycle:
[scenario.remote_write.value]
pattern = "quantized_signal" # linear, constant, modulo, unique, seeded_random,
# run_length, quantized_signal,
# signal_with_sporadic_stalls, mixed_signal_repeated
# signal_with_sporadic_stalls, mixed_signal_repeated,
# bounded_mixed
base = 0.0
step = 0.125
cardinality = 4096 # buckets for modulo/seeded_random/quantized_signal/run_length
seed = 12345 # deterministic seeded_random input
run_length = 8 # adjacent samples per bucket for run_length/quantized_signal
cardinality = 4096 # buckets for modulo/seeded_random/quantized_signal/run_length;
# baseline buckets for bounded_mixed
seed = 12345 # deterministic seeded_random input and bounded_mixed series hash
run_length = 8 # adjacent samples per bucket for run_length/quantized_signal;
# interpolation interval for bounded_mixed
stall_every = 100 # interval for signal_with_sporadic_stalls
stall_length = 16 # held samples inside each stall interval
mixed_every = 5 # every Nth sample becomes the repeated base value
mixed_every = 5 # every Nth sample becomes the repeated base value;
# one fractional bounded_mixed series per N series
```
The default `linear` pattern preserves the helper's historical formula. Use
@@ -103,12 +107,29 @@ data shapes, `run_length` for run-heavy low-cardinality series, `quantized_signa
for signal-like values collapsed into a finite bucket set, `signal_with_sporadic_stalls`
for mostly continuous signals with periodic flat spots, and `mixed_signal_repeated`
for signal-plus-periodic-default mixtures. `unique` or high-cardinality buckets
still work for broad sample-value distributions. This is a generic sample-value
control for query/ingestion cases; it does not inspect or assert storage
encoding, Parquet footers, or storage policy choices. For chunked remote-write
ingestion, the runner passes the sample offset and total sample count to the
helper so non-linear value patterns use a stable global/per-series ordinal across
chunks.
still work for broad sample-value distributions.
`bounded_mixed` is synthetic rather than an empirical workload. For series `s`,
`h = splitmix64(s ^ seed)` selects a span from `[10, 1_000, 100_000, 10_000_000]`
using its top two bits, multiplied by `1 + ((h >> 32) % 10)`. Its baseline is
`(h % max(cardinality, 1)) * span / 10`. Ranges can overlap. At local sample
`l = sample_offset + sample_idx`, it linearly interpolates hash-derived anchors
in `[-span + 1, span - 1]`, changing anchors every `max(run_length, 1)` samples.
It applies `base + step * signal`, then rounds to an integer. Every
`max(mixed_every, 1)`-th series adds a hash-derived fraction from `1/1000` to
`999/1000`. The 95/5 finite, nonintegral guarantee applies to the case's bounded
parameters, not arbitrary huge or nonfinite base/step inputs. With `base = 0`
and `step = 1`, output lies within `baseline ± span`, allowing one unit for
rounding and the fractional addition. The explicit case uses 1,000 equal-length
series and `mixed_every = 20`, giving exactly 95% integer-valued and 5%
fractional Float64 samples. Its ranges and temporal shape are synthetic design
parameters, not measured properties from the survey.
This is a generic sample-value control for query/ingestion cases; it does not
inspect or assert storage encoding, Parquet footers, or storage policy choices.
For chunked remote-write ingestion, the runner passes the sample offset and total
sample count to the helper so non-linear value patterns use a stable global/per-series
ordinal across chunks.
Case schema, value distribution defaults, storage defaults, and read-bench
defaults are owned by Rust. The outer CI driver calls
@@ -162,12 +183,15 @@ measures region scan cost, and query measurements still exercise the SQL/TQL
frontend path. Treat all performance conclusions as release-only; debug builds
are suitable only for command wiring and correctness checks.
`prepare-remote` creates the configured database if needed. The outer driver writes a per-target
frontend config enabling `[prom_store]` with metric engine storage and a non-zero
`pending_rows_flush_interval`, and validates that the logical metric table reaches
`series_count * samples_per_series` rows before trusting the query measurements.
Use `--fixture-generator /path/to/query_perf_fixture` to provide the Rust helper
to the outer driver.
`prepare-remote` creates the configured database if needed. A remote-write case
can provide `base_setup_sql` and `candidate_setup_sql` lists; each target runs its
own complete statements in order after database creation and before ingestion.
The outer driver writes a per-target frontend config enabling `[prom_store]` with
metric engine storage and a non-zero `pending_rows_flush_interval`, and validates
that the logical metric table reaches `series_count * samples_per_series` rows
before trusting the query measurements. Use
`--fixture-generator /path/to/query_perf_fixture` to provide the Rust helper to
the outer driver.
Large manual remote-write cases can set `sample_chunk_size` to split ingestion by
time. For each chunk, `prepare-remote` invokes `query_perf_fixture prom-remote-write` with the
@@ -204,7 +228,9 @@ case for issue #7913. It writes 8192 series × 20160 samples through remote-writ
in 1440-sample daily time chunks, flushing after each chunk before running 1d/7d/14d
TQL selectors. It is not included in the default `all` case set because ingestion
cost dominates routine CI validation. Commenting `/query-regression heavy` runs
only this case; `/query-regression` runs the eight routine default cases. Manual
only this case; `/query-regression` runs the nine routine default cases, including
`sst_float_bss`, `promql_instant_last_row_9034`, and
`mito_prefilter_all_match`. Manual
workflow dispatch accepts the `heavy` token to select this case.
The routine default set also includes
@@ -371,6 +397,89 @@ uv run --no-project python .github/scripts/query-regression-run.py \
--work-dir /tmp/query-regression-work
```
### SST float BSS comparison
`tests/perf/query_cases/sst_float_bss/case.toml` is included in the routine
`all` default case group and compares a default empty physical metric table with
a candidate byte-stream-split (BSS) physical table. It writes 1,000 series × 4,320 samples
(4,320,000 rows) using synthetic `bounded_mixed` values: 95% integral series and
5% nonintegral series, with bounded per-series fluctuation rather than globally
unique values. This deliberately matches a 95/5 design; it is not an empirical
claim about any production population. A separate survey sample found 94.79%
integral values, but that observation does not make its values globally unique.
Current mixed-data evidence, including SST inspection, exact-bit row verification,
warmed endpoint SQL, and both warm-reader projections, is recorded in
[`query_cases/sst_float_bss/RESULTS.md`](query_cases/sst_float_bss/RESULTS.md).
Timing observations were collected while other builds saturated the shared host;
they are not performance acceptance evidence. Before rerunning timings, ensure
the host is idle (not merely this agent), record load throughout, and avoid
concurrent builds. CPU affinity alone does not isolate memory or I/O contention.
The historical unique-integer workload is preserved only in Git history and does
not apply to this case.
Both targets must use the exact same release `greptime` binary; only the
per-target table setup SQL differs. Run the existing driver from a checkout
containing that binary, with absolute paths and fresh data directories for every
run:
```bash
REPO="$(pwd -P)"
RELEASE_GREPTIME="/absolute/path/to/release/greptime"
FIXTURE_GENERATOR="/absolute/path/to/release/query_perf_fixture"
RUNNER="/absolute/path/to/release/query_regression_runner"
WORK_DIR="/absolute/path/to/fresh/sst-float-bss-run-1"
cd "$REPO"
uv run --no-project python "$REPO/.github/scripts/query-regression-run.py" \
--cases "$REPO/tests/perf/query_cases/sst_float_bss/case.toml" \
--base-src "$REPO" \
--candidate-src . \
--base-bin "$RELEASE_GREPTIME" \
--candidate-bin "$RELEASE_GREPTIME" \
--fixture-generator "$FIXTURE_GENERATOR" \
--runner "$RUNNER" \
--work-dir "$WORK_DIR" \
--summary-script "$REPO/.github/scripts/query-regression-summary.py"
```
Repeat the command three times with a different fresh absolute `WORK_DIR` each
run. In each `query-regression-report.json`, compare base and candidate
`targets[].storage_inspection.summary.summary.total_file_size`, and each query's
`targets[].measurements[].latency_ms_median`. Storage percentage is
`(candidate_total_file_size - base_total_file_size) / base_total_file_size * 100`;
query latency percentage is
`(candidate_latency_ms_median - base_latency_ms_median) / base_latency_ms_median * 100`.
The configured three warmups occur after the initial query validation and before
that query's 15 measured endpoint requests, so query latency is a warmed
frontend/cache measurement. The case restricts storage inspection to
`data/greptime/public`, excluding `greptime_private` SSTs. After the datanodes
stop, it also runs seven iterations of value-only `parquetbench` for all
inspected SST files and sequential `scanbench` over the corresponding regions,
with parallelism one. The three explicit flushes yielded six SSTs in the observed
mixed runs—three 1,105,920-row files and three 334,080-row files—because storage split each
flush; this is an observation, not a guaranteed file layout. Their per-run
output and aggregate `parquetbench_median_average_ms` and
`scanbench_median_average_ms` are under
`targets[].read_bench`; they are quiescent local-file read/scan diagnostics, not
warmed frontend-query latency measurements. Bench averages include iteration one;
no OS cache is dropped, and the driver runs base before candidate.
Historical [PR #8548](https://github.com/GreptimeTeam/greptimedb/pull/8548)
reported storage savings alongside warm-read slowdowns for a different mixed
counter/gauge study. It used value-only and all-column projections, discarded the
first iteration, and alternated target order. This case reuses that reader
measurement approach, not its dataset or results. For the supplementary warm
comparison, reuse recorded commands against stopped data directories with eight
iterations, discard iteration one, and alternate target order for eight rounds.
For parquetbench, sum all per-file warm medians before taking the outer median;
for scanbench, take the outer median of whole-region warm medians. Keep
post-flush and post-compaction measurements separate.
The case's `-5.0` storage target and `25` query-latency guardrail are experimental
acceptance targets, not observed-benefit claims; do not relax them if a run
fails. Its three periodic flushes and shared high TWCS trigger avoid the normal
four-file compaction trigger from confounding the layout. Check footer encodings
(BSS on candidate and no BSS on base) and data equality from the artifacts rather
than through a new harness framework.
For a focused manual reproduction of the Mito prefilter all-match optimization, use the existing lifecycle command above with `--cases tests/perf/query_cases/mito_prefilter_all_match/case.toml`. Its four count probes require explicit result inspection rather than automatic validation: expect `262144`, `131072`, `16896`, and `16896` in query order. The default lifecycle retains caches, so the optimization remains exercised but its timing includes warm-cache effects; use an explicitly configured cold environment when a cold comparison is required.
The Rust runner subcommands are also useful for focused diagnostics:
@@ -400,8 +509,8 @@ parquetbench/scanbench` as the read-bench tool against each target's data direct
The workflow runs when an allowlisted repository admin comments
`/query-regression` on a non-draft PR. It does not rerun on pushes,
ready-for-review, or reopen events. `/query-regression` runs the eight routine
default cases, including `promql_instant_last_row_9034` and
ready-for-review, or reopen events. `/query-regression` runs the nine routine
default cases, including `sst_float_bss`, `promql_instant_last_row_9034`, and
`mito_prefilter_all_match`; `/query-regression heavy` runs only the
high-cardinality remote-write #7913 case. PR runs build base/candidate once and
use `--allow-large-fixture`. Manual `workflow_dispatch` runs can pass `all`,
@@ -0,0 +1,145 @@
# SST float BSS: mixed-data post-flush evidence
This report replaces the earlier unique-integer experiment as the primary evidence
for `sst_float_bss`. The detailed historical numbers remain in Git history; they
apply only to that superseded workload and must not be used for the current mixed
case.
> **Timing measurements are confounded.** After these runs, the user reported
> that other agents' builds saturated all 20 logical CPUs during measurement.
> No host-wide idle/load gate was enforced. The latency tables below are retained
> as raw observations only: neither improvements nor threshold failures establish
> BSS performance. SST byte counts and cross-target stored-row checks remain
> valid for the inspected dataset. Stable-host timing verification is pending.
## Case and data shape
- Generator/case revision: `149b49de5f3`.
- Base and candidate used the same release `greptime` binary. The only physical
table difference was `experimental_sst_float_field_encoding`: `default` versus
`byte_stream_split`; base used dictionary encoding and candidate used BSS
without dictionaries. This is not a dictionary-policy-held-constant comparison.
- Each of three fresh post-flush runs wrote 1,000 series × 4,320 samples at
60-second intervals: 4,320,000 rows per target. Three explicit flushes produced
six SSTs per target: three × 1,105,920 rows and three × 334,080 rows, with 45
`greptime_value` row groups total. This is observed layout, not a layout
guarantee.
- `bounded_mixed` is a synthetic, uncalibrated design: a series hash selects its
baseline and span band, and 60-sample anchor interpolation produces bounded
fluctuation. It does not represent a measured production distribution.
- Exactly 950 equal-length series are integer-only and 50 are fractional-only,
yielding 4,104,000 integer rows and 216,000 fractional rows per target. The
equal-length 1,000-series design deliberately makes both the series and row
shares 95%/5%.
- The 95% design is not an empirical claim. A [local preview survey project](http://192.168.50.85:8765/view/prometheus-metrics-186-project-preliminary)
observed 94.79% integral **samples**; it does not establish a global rate,
global uniqueness, or a 95% series distribution.
## Initial SST inspection and value verification
Inspection is limited to `data/greptime/public`; bytes exclude WAL, indexes,
metadata, and internal/private-table storage. Each of the three runs had the
same totals:
| Measure | Default | BSS | BSS change |
| --- | ---: | ---: | ---: |
| Selected SST bytes | 20,443,131 | 8,884,743 | -56.5392% |
| `greptime_value` compressed bytes | 20,201,188 | 8,643,964 | -57.2106% |
| SST files / rows / value row groups | 6 / 4,320,000 / 45 | 6 / 4,320,000 / 45 | — |
Footer inspection found `PLAIN`/`RLE`/`RLE_DICTIONARY` for the default value
column and `RLE`/`BYTE_STREAM_SPLIT` for BSS (both ZSTD level 1). The configured
storage threshold remains a 5% reduction; this repeated SST-size result passes
that threshold. It is post-flush evidence only, not a compaction result or a
general float-compression claim.
For every base/candidate group in all three runs, rows were compared by the same
primary key and timestamp and their Float64 bits were exact matches: 950
integer-only and 50 fractional-only series, 4,104,000 and 216,000 rows
respectively. The six canonical sorted-row SHA-256 results are identical:
```
48f87d7d01bc8a0ce9a9cdb3033f322e787784fcae61f1417ac25b05c4a40375
```
This verifies exact cross-target stored rows, not independent reconstruction of
the generator output. All 1,000 series had both rises and falls, and every one
of the 36 inspected SSTs contained both integer-valued and fractional values;
these were not stored as two disjoint file sets.
## Warmed endpoint SQL
Each query used three warmups followed by 15 measured endpoint requests. Values
are median/p95 milliseconds; delta is `(BSS median - default median) / default
median × 100%`. The guardrail remains a maximum 25% candidate median regression and
was not changed for these runs.
| Run | Query | Default median / p95 | BSS median / p95 | Delta | Guardrail |
| --- | --- | ---: | ---: | ---: | --- |
| mixed-1 | `sum_all_values` | 27.13 / 37.38 | 25.45 / 38.99 | -6.21% | pass |
| mixed-1 | `hourly_sum_values` | 118.78 / 136.63 | 130.93 / 185.26 | +10.23% | pass |
| mixed-2 | `sum_all_values` | 24.02 / 35.02 | 36.34 / 56.01 | +51.31% | **fail** |
| mixed-2 | `hourly_sum_values` | 95.58 / 120.40 | 85.45 / 148.03 | -10.60% | pass |
| mixed-3 | `sum_all_values` | 38.25 / 55.41 | 19.68 / 29.49 | -48.55% | pass |
| mixed-3 | `hourly_sum_values` | 56.77 / 148.29 | 121.87 / 143.16 | +114.66% | **fail** |
The two recorded threshold failures remain in the artifacts, but the saturated
shared host prevents attributing them to BSS. Passing entries likewise do not
establish performance acceptance. No CPU isolation or cache flush was used.
## Stopped-directory warm readers
For each of the three datasets, eight outer rounds alternate base/candidate order.
Each command runs eight iterations, discards the first, and takes the median of
iterations 28. Parquetbench reads every SST: sum the six per-file medians, then
take the median of the eight round totals. Scanbench scans the whole region and
reports the outer median of its eight warm medians. These are distinct metrics,
not interchangeable whole-query latencies. Deltas are ratios of separately
aggregated medians. Value projection selects `greptime_value`; all-column
parquetbench returns five physical columns, while scanbench uses `{}` (all
region columns). Projection order is fixed; target order alternates.
| Dataset | Projection | Reader | Base ms | BSS ms | Change |
| --- | --- | --- | ---: | ---: | ---: |
| mixed-1 | value | parquetbench | 95.957 | 68.442 | -28.67% |
| mixed-1 | value | scanbench | 144.293 | 115.421 | -20.01% |
| mixed-1 | allcolumns | parquetbench | 176.620 | 103.761 | -41.25% |
| mixed-1 | allcolumns | scanbench | 200.881 | 202.660 | +0.89% |
| mixed-2 | value | parquetbench | 112.151 | 74.275 | -33.77% |
| mixed-2 | value | scanbench | 109.824 | 78.189 | -28.80% |
| mixed-2 | allcolumns | parquetbench | 141.944 | 102.681 | -27.66% |
| mixed-2 | allcolumns | scanbench | 211.414 | 163.261 | -22.78% |
| mixed-3 | value | parquetbench | 121.827 | 67.565 | -44.54% |
| mixed-3 | value | scanbench | 100.968 | 79.878 | -20.89% |
| mixed-3 | allcolumns | parquetbench | 113.984 | 94.361 | -17.22% |
| mixed-3 | allcolumns | scanbench | 193.101 | 164.175 | -14.98% |
All 672 commands / 5,376 raw iterations returned the expected rows; timing,
projection configurations, parquet schemas and order metadata were checked.
Run 1 all-column scanbench differed by **+0.89%**, negligible relative to the
observed variation, not evidence of a regression. Its round medians ranged from
167.197663.465 ms (base) and 133.339557.352 ms (BSS). There is substantial
variability; these measurements do not establish a stable gain on every read
path. The reported concurrent CPU saturation affects apparent improvements as
well as slowdowns; no causal performance conclusion is drawn from these runs.
## Scope and recorded checks
This case compares default and BSS post-flush data only. It adds no benchmark
framework and no public artifact upload. Historical unique-integer results and
other mixed counter/gauge studies have different data and methods; they are
context only and do not apply to this workload.
Recorded checks for this change set: 37 targeted Rust tests, 20 Python tooling
tests, and 15 plans passed. Full-workspace tests were not run.
Local artifacts are under
`/mnt/nvme_rust/rust-targets/metric-bss-perf/experiments/mixed-{1,2,3}/`,
including `value-verification.json`, `verification.txt`, and each run's
`sst_float_bss/query-regression-report.json`.
Warm artifacts are in sibling `warm-mixed-{1,2,3}/` directories;
`mixed-warm-comparison.json` contains the table and per-round ranges.
`mixed-file-composition.json` records integer/fractional counts for each SST.
The shared release `greptime` SHA-256 is
`2ce0ab1670cd4bdb0f60c87af3e1f494ee66c3c6ab83bce943a073638514eb2b`.
@@ -0,0 +1,83 @@
# Default all-group default-versus-BSS storage and warmed-query comparison.
[case]
name = "sst_float_bss"
description = "Compare default and byte-stream-split floating-point SST bytes and warmed value-scanning query latency with bounded 95/5 per-series DOUBLE values"
[scenario]
kind = "prom_remote_write_then_query"
[scenario.remote_write]
database = "public"
metric = "sst_float_bss"
physical_table = "sst_float_bss_physical"
series_count = 1000
samples_per_series = 4320
sample_chunk_size = 1440
flush_every_sample_chunks = 1
start_unix_millis = 1_704_067_200_000 # 2024-01-01T00:00:00Z
step_millis = 60000
chunk_series_count = 128
timeout_seconds = 600
visibility_timeout_seconds = 300
# Omit the default encoding from base setup because older binaries do not recognize it.
# The high shared TWCS trigger avoids compaction changing the three-flush layout.
base_setup_sql = [
"CREATE TABLE sst_float_bss_physical (greptime_timestamp TIMESTAMP TIME INDEX, greptime_value DOUBLE) ENGINE=metric WITH ('physical_metric_table'='', 'compaction.twcs.trigger_file_num'='100')",
]
candidate_setup_sql = [
"CREATE TABLE sst_float_bss_physical (greptime_timestamp TIMESTAMP TIME INDEX, greptime_value DOUBLE) ENGINE=metric WITH ('physical_metric_table'='', 'experimental_sst_float_field_encoding'='byte_stream_split', 'compaction.twcs.trigger_file_num'='100')",
]
[scenario.remote_write.value]
# Synthetic, not empirical: 95% integral and 5% nonintegral series vary
# smoothly within deterministic per-series ranges; series are not a global monotonic counter.
pattern = "bounded_mixed"
base = 0
step = 1
cardinality = 1000
seed = 42
mixed_every = 20
run_length = 60
[scenario.remote_write.prom_store]
pending_rows_flush_interval = "1s"
max_batch_rows = 100000
[scenario.remote_write.storage]
root_suffix = "data/greptime/public"
column = "greptime_value"
min_files = 1
min_files_with_column = 1
max_candidate_total_file_size_regression_pct = -5.0
[scenario.remote_write.read_bench]
enabled = true
parquetbench = true
scanbench = true
iterations = 7
projection = ["greptime_value"]
parquet_reader = "direct"
scan_scanner = "seq"
parallelism = 1
[[scenario.queries]]
name = "sum_all_values"
kind = "sql"
query = "SELECT sum(greptime_value) FROM sst_float_bss"
warmup = 3
iterations = 15
[scenario.queries.thresholds]
max_candidate_latency_regression_pct = 25
[[scenario.queries]]
name = "hourly_sum_values"
kind = "sql"
query = "SELECT date_bin(INTERVAL '1 hour', greptime_timestamp) AS time_window, sum(greptime_value) FROM sst_float_bss WHERE greptime_timestamp >= TIMESTAMP '2024-01-01 00:00:00' AND greptime_timestamp < TIMESTAMP '2024-01-04 00:00:00' GROUP BY time_window ORDER BY time_window"
warmup = 3
iterations = 15
[scenario.queries.thresholds]
max_candidate_latency_regression_pct = 25
@@ -16,6 +16,7 @@
"""Coverage for query-regression case group selection."""
import importlib.util
import re
import sys
import unittest
from pathlib import Path
@@ -35,6 +36,7 @@ class QueryRegressionCaseSelectionTest(unittest.TestCase):
runner.split_cases(["all"]),
[
"tests/perf/query_cases/smoke_direct_sst/case.toml",
"tests/perf/query_cases/sst_float_bss/case.toml",
"tests/perf/query_cases/prom_remote_write_seeded_random/case.toml",
"tests/perf/query_cases/prom_remote_write_run_heavy/case.toml",
"tests/perf/query_cases/prom_remote_write_mixed_every/case.toml",
@@ -45,8 +47,8 @@ class QueryRegressionCaseSelectionTest(unittest.TestCase):
],
)
def test_implicit_selection_has_eight_routine_cases(self) -> None:
self.assertEqual(len(runner.DEFAULT_CASES), 8)
def test_implicit_selection_has_nine_routine_cases(self) -> None:
self.assertEqual(len(runner.DEFAULT_CASES), 9)
self.assertEqual(runner.split_cases([]), runner.DEFAULT_CASES)
def test_heavy_selects_only_remote_write_7913(self) -> None:
@@ -59,6 +61,23 @@ class QueryRegressionCaseSelectionTest(unittest.TestCase):
case = "tests/perf/query_cases/sql_topk_order_by/case.toml"
self.assertEqual(runner.split_cases([case]), [case])
def test_sst_float_bss_omits_base_encoding_for_older_binaries(self) -> None:
case_path = Path(__file__).parent / "query_cases/sst_float_bss/case.toml"
setup_sql = dict(re.findall(
r'(base|candidate)_setup_sql = \[\s*"([^"\n]+)"',
case_path.read_text(),
))
base_setup = setup_sql["base"]
candidate_setup = setup_sql["candidate"]
candidate_option = "'experimental_sst_float_field_encoding'='byte_stream_split'"
self.assertNotIn("experimental_sst_float_field_encoding", base_setup)
self.assertIn(candidate_option, candidate_setup)
self.assertEqual(
base_setup,
candidate_setup.replace(f", {candidate_option}", ""),
)
if __name__ == "__main__":
unittest.main()