1#![allow(clippy::print_stderr, clippy::print_stdout)]
16
17use std::collections::HashMap;
30use std::fs;
31use std::io::{BufWriter, Write};
32use std::num::{NonZeroU64, NonZeroUsize};
33use std::path::{Path, PathBuf};
34use std::sync::Arc;
35
36use api::v1::{OpType, SemanticType};
37use async_trait::async_trait;
38use clap::Parser;
39use datatypes::arrow::array::{
40 ArrayRef, BinaryDictionaryBuilder, Float64Array, RecordBatch, StringDictionaryBuilder,
41 TimestampMillisecondArray, TimestampNanosecondArray, UInt8Array, UInt64Array,
42};
43use datatypes::arrow::datatypes::UInt32Type;
44use datatypes::prelude::ConcreteDataType;
45use datatypes::schema::{ColumnSchema, SkippingIndexOptions};
46use mito_codec::row_converter::{DensePrimaryKeyCodec, PrimaryKeyCodecExt, SortField};
47use mito2::access_layer::{FilePathProvider, Metrics, WriteType};
48use mito2::config::IndexConfig;
49use mito2::manifest::action::{RegionCheckpoint, RegionManifest, RemovedFilesRecord};
50use mito2::read::FlatSource;
51use mito2::sst::file::{FileMeta, RegionFileId};
52use mito2::sst::index::{Indexer, IndexerBuilder};
53use mito2::sst::parquet::writer::ParquetWriter;
54use mito2::sst::parquet::{SstInfo, WriteOptions};
55use mito2::sst::{
56 DEFAULT_WRITE_BUFFER_SIZE, FlatSchemaOptions, FormatType, to_flat_sst_arrow_schema,
57};
58use object_store::ObjectStore;
59use object_store::services::Fs as FsBuilder;
60use serde::Deserialize;
61use store_api::codec::PrimaryKeyEncoding;
62use store_api::metadata::{ColumnMetadata, RegionMetadata, RegionMetadataRef};
63use store_api::path_utils::region_name;
64use store_api::region_request::PathType;
65use store_api::storage::{FileId, RegionId};
66
67#[derive(Parser, Debug)]
68#[command(name = "query_perf_fixture")]
69#[command(about = "Generate direct-readable SST fixtures for query performance cases")]
70struct Args {
71 #[arg(long, value_name = "PATH")]
73 case: PathBuf,
74
75 #[arg(long, value_name = "DIR")]
77 out_dir: PathBuf,
78
79 #[arg(long, default_value = "4398046511104")]
81 region_id: u64,
82
83 #[arg(long, value_name = "DIR")]
85 table_dir: Option<String>,
86
87 #[arg(long, value_name = "NAME")]
89 table: Option<String>,
90
91 #[arg(long, default_value = "1000000")]
93 checkpoint_version: u64,
94
95 #[arg(long)]
97 allow_large: bool,
98
99 #[arg(long)]
101 dry_run: bool,
102}
103
104#[derive(Debug, Deserialize)]
105struct CaseFile {
106 scenario: Scenario,
107}
108
109#[derive(Debug, Deserialize)]
110#[serde(tag = "kind")]
111enum Scenario {
112 #[serde(rename = "direct_readable_sst")]
113 DirectReadableSst(DirectReadableSstScenario),
114}
115
116#[derive(Debug, Deserialize)]
117struct DirectReadableSstScenario {
118 #[serde(default)]
119 seed: Option<u64>,
120 tables: Vec<TableConfig>,
121 layout: LayoutConfig,
122}
123
124impl Scenario {
125 fn kind(&self) -> &'static str {
126 match self {
127 Scenario::DirectReadableSst(_) => "direct_readable_sst",
128 }
129 }
130
131 fn direct_readable_sst(&self) -> &DirectReadableSstScenario {
132 match self {
133 Scenario::DirectReadableSst(scenario) => scenario,
134 }
135 }
136}
137
138#[derive(Debug, Deserialize)]
139struct TableConfig {
140 database: String,
141 name: String,
142 engine: String,
143 #[serde(default)]
144 append_mode: Option<bool>,
145 #[serde(default)]
146 sst_format: Option<String>,
147 primary_key: Vec<String>,
148 time_index: String,
149 columns: Vec<ColumnConfig>,
150}
151
152#[derive(Debug, Deserialize)]
153struct ColumnConfig {
154 name: String,
155 #[serde(rename = "type")]
156 ty: String,
157 semantic: String,
158 distribution: Option<Distribution>,
159}
160
161#[derive(Debug, Deserialize)]
162#[serde(tag = "kind")]
163enum Distribution {
164 #[serde(rename = "cardinality")]
165 Cardinality {
166 values: NonZeroUsize,
167 prefix: String,
168 },
169 #[serde(rename = "deterministic_wave")]
170 DeterministicWave { min: f64, max: f64 },
171}
172
173#[derive(Debug, Deserialize)]
174struct LayoutConfig {
175 regions: usize,
176 sst_count: usize,
177 rows_per_sst: usize,
178 row_group_size: usize,
179 series_count: NonZeroUsize,
180 start_unix_nanos: i64,
181 step_nanos: i64,
182 time_range_layout: String,
183 series_layout: String,
184}
185
186struct NoopIndexBuilder;
187
188#[async_trait]
189impl IndexerBuilder for NoopIndexBuilder {
190 async fn build(&self, _file_id: FileId, _index_version: u64) -> Indexer {
191 Indexer::default()
192 }
193}
194
195#[derive(Clone)]
196struct FixedPathProvider {
197 table_dir: String,
198}
199
200impl FilePathProvider for FixedPathProvider {
201 fn build_index_file_path(&self, file_id: RegionFileId) -> String {
202 mito2::sst::location::index_file_path_legacy(&self.table_dir, file_id, PathType::Bare)
203 }
204 fn build_index_file_path_with_version(
205 &self,
206 index_id: mito2::sst::file::RegionIndexId,
207 ) -> String {
208 mito2::sst::location::index_file_path(&self.table_dir, index_id, PathType::Bare)
209 }
210 fn build_sst_file_path(&self, file_id: RegionFileId) -> String {
211 mito2::sst::location::sst_file_path(&self.table_dir, file_id, PathType::Bare)
212 }
213}
214
215fn semantic_type(s: &str) -> SemanticType {
216 match s.to_ascii_lowercase().as_str() {
217 "tag" => SemanticType::Tag,
218 "field" => SemanticType::Field,
219 "timestamp" => SemanticType::Timestamp,
220 other => panic!("unsupported semantic {other}"),
221 }
222}
223
224fn concrete_type(ty: &str) -> ConcreteDataType {
225 match ty.to_ascii_uppercase().as_str() {
226 "STRING" => ConcreteDataType::string_datatype(),
227 "DOUBLE" => ConcreteDataType::float64_datatype(),
228 "UINT64" => ConcreteDataType::uint64_datatype(),
229 "TIMESTAMP(9)" => ConcreteDataType::timestamp_nanosecond_datatype(),
230 "TIMESTAMP(3)" => ConcreteDataType::timestamp_millisecond_datatype(),
231 other => panic!("unsupported column type {other}"),
232 }
233}
234
235fn build_region_metadata(table: &TableConfig, region_id: RegionId) -> RegionMetadata {
236 let mut builder = store_api::metadata::RegionMetadataBuilder::new(region_id);
237 for (idx, col) in table.columns.iter().enumerate() {
238 let mut schema = ColumnSchema::new(
239 &col.name,
240 concrete_type(&col.ty),
241 col.semantic != "timestamp",
242 );
243 if col.semantic == "tag" {
244 schema = schema.with_inverted_index(true);
245 }
246 if col.semantic == "tag" || col.semantic == "field" {
247 schema = schema
248 .with_skipping_options(SkippingIndexOptions {
249 granularity: 1,
250 ..Default::default()
251 })
252 .expect("valid skipping index options for query perf fixture");
253 }
254 if col.name == table.time_index {
255 schema = schema.with_time_index(true);
256 }
257 builder.push_column_metadata(ColumnMetadata {
258 column_schema: schema,
259 semantic_type: semantic_type(&col.semantic),
260 column_id: idx as u32,
261 });
262 }
263 let pk_ids = table
264 .primary_key
265 .iter()
266 .map(|name| {
267 table
268 .columns
269 .iter()
270 .position(|c| c.name == *name)
271 .unwrap_or_else(|| panic!("primary key column {name} not found")) as u32
272 })
273 .collect();
274 builder.primary_key(pk_ids);
275 builder.primary_key_encoding(PrimaryKeyEncoding::Dense);
276 builder
277 .build()
278 .expect("region metadata should be valid for fixture table config")
279}
280
281fn encode_dense_primary_key(table: &TableConfig, values: &HashMap<String, String>) -> Vec<u8> {
282 let fields = table
283 .primary_key
284 .iter()
285 .enumerate()
286 .map(|(idx, _)| {
287 (
288 idx as u32,
289 SortField::new(ConcreteDataType::string_datatype()),
290 )
291 })
292 .collect();
293 let converter = DensePrimaryKeyCodec::with_fields(fields);
294 converter
295 .encode(
296 table
297 .primary_key
298 .iter()
299 .map(|name| datatypes::value::ValueRef::String(values[name].as_str())),
300 )
301 .expect("dense primary key encoding should match fixture primary key fields")
302}
303
304fn tag_value(col: &ColumnConfig, series: usize) -> String {
305 match col
306 .distribution
307 .as_ref()
308 .expect("tag distribution is required")
309 {
310 Distribution::Cardinality { values, prefix } => {
311 format!("{}{}", prefix, series % values.get())
312 }
313 _ => panic!("tag column {} requires cardinality distribution", col.name),
314 }
315}
316
317fn wave_value(min: f64, max: f64, row: usize) -> f64 {
318 let span = max - min;
319 let phase = (row % 1024) as f64 / 1023.0;
320 min + span * (0.5 - 0.5 * (std::f64::consts::TAU * phase).cos())
321}
322
323fn generate_record_batch(
324 table: &TableConfig,
325 metadata: &RegionMetadataRef,
326 layout: &LayoutConfig,
327 sst_idx: usize,
328 sequence: u64,
329) -> RecordBatch {
330 let flat_schema = to_flat_sst_arrow_schema(metadata, &FlatSchemaOptions::default());
331 let rows = layout.rows_per_sst;
332 let mut columns = Vec::new();
333 let base_row = sst_idx * rows;
334
335 for col in &table.columns {
336 match (col.semantic.as_str(), col.ty.to_ascii_uppercase().as_str()) {
337 ("tag", "STRING") => {
338 let mut b = StringDictionaryBuilder::<UInt32Type>::new();
339 for row in 0..rows {
340 let series = if layout.series_layout == "round_robin" {
341 (base_row + row) % layout.series_count.get()
342 } else {
343 sst_idx % layout.series_count.get()
344 };
345 let value = tag_value(col, series);
346 b.append_value(value);
347 }
348 columns.push(Arc::new(b.finish()) as ArrayRef);
349 }
350 ("field", "DOUBLE") => {
351 let (min, max) = match col.distribution.as_ref() {
352 Some(Distribution::DeterministicWave { min, max }) => (*min, *max),
353 _ => (0.0, 1.0),
354 };
355 columns.push(Arc::new(Float64Array::from(
356 (0..rows)
357 .map(|r| wave_value(min, max, base_row + r))
358 .collect::<Vec<_>>(),
359 )) as ArrayRef);
360 }
361 ("field", "UINT64") => columns.push(Arc::new(UInt64Array::from(
362 (0..rows).map(|r| (base_row + r) as u64).collect::<Vec<_>>(),
363 )) as ArrayRef),
364 ("timestamp", "TIMESTAMP(9)") => columns.push(Arc::new(TimestampNanosecondArray::from(
365 (0..rows)
366 .map(|r| layout.start_unix_nanos + ((base_row + r) as i64 * layout.step_nanos))
367 .collect::<Vec<_>>(),
368 )) as ArrayRef),
369 ("timestamp", "TIMESTAMP(3)") => {
370 columns.push(Arc::new(TimestampMillisecondArray::from(
371 (0..rows)
372 .map(|r| {
373 (layout.start_unix_nanos + ((base_row + r) as i64 * layout.step_nanos))
374 / 1_000_000
375 })
376 .collect::<Vec<_>>(),
377 )) as ArrayRef)
378 }
379 other => panic!("unsupported column combination {other:?}"),
380 }
381 }
382
383 let mut pk_builder = BinaryDictionaryBuilder::<UInt32Type>::new();
384 for row in 0..rows {
385 let series = if layout.series_layout == "round_robin" {
386 (base_row + row) % layout.series_count.get()
387 } else {
388 sst_idx % layout.series_count.get()
389 };
390 let values = table
391 .columns
392 .iter()
393 .filter(|c| c.semantic == "tag")
394 .map(|c| (c.name.clone(), tag_value(c, series)))
395 .collect();
396 pk_builder
397 .append(encode_dense_primary_key(table, &values))
398 .expect("append generated dense primary key to Arrow dictionary builder");
399 }
400 columns.push(Arc::new(pk_builder.finish()));
401 columns.push(Arc::new(UInt64Array::from_value(sequence, rows)));
402 columns.push(Arc::new(UInt8Array::from_value(OpType::Put as u8, rows)));
403 RecordBatch::try_new(flat_schema, columns)
404 .expect("generated fixture columns should match flat SST Arrow schema")
405}
406
407fn file_meta_from_sst_info(
408 info: &SstInfo,
409 region_id: RegionId,
410 file_id: FileId,
411 sequence: u64,
412) -> FileMeta {
413 FileMeta {
414 region_id,
415 file_id,
416 time_range: info.time_range,
417 level: 0,
418 file_size: info.file_size,
419 max_row_group_uncompressed_size: info.max_row_group_uncompressed_size,
420 available_indexes: Default::default(),
421 indexes: Default::default(),
422 index_file_size: 0,
423 index_version: 0,
424 num_rows: info.num_rows as u64,
425 num_row_groups: info.num_row_groups,
426 sequence: NonZeroU64::new(sequence),
427 partition_expr: None,
428 num_series: info.num_series,
429 primary_key_min: None,
430 primary_key_max: None,
431 }
432}
433
434fn deterministic_file_id(seed: u64, index: usize) -> FileId {
435 let seed_hi = ((seed >> 16) & 0xffff) as u16;
436 let seed_lo = (seed & 0xffff) as u16;
437 FileId::parse_str(&format!(
438 "00000000-0000-{seed_hi:04x}-{seed_lo:04x}-{index:012x}"
439 ))
440 .expect("deterministic file id must be a valid UUID")
441}
442
443fn remap_sst_file(
444 object_store_dir: &Path,
445 table_dir: &str,
446 region_id: RegionId,
447 old_file_id: FileId,
448 new_file_id: FileId,
449) {
450 if old_file_id == new_file_id {
451 return;
452 }
453 let old_path = object_store_dir.join(mito2::sst::location::sst_file_path(
454 table_dir,
455 RegionFileId::new(region_id, old_file_id),
456 PathType::Bare,
457 ));
458 let new_path = object_store_dir.join(mito2::sst::location::sst_file_path(
459 table_dir,
460 RegionFileId::new(region_id, new_file_id),
461 PathType::Bare,
462 ));
463 if let Some(parent) = new_path.parent() {
464 fs::create_dir_all(parent).expect("failed to create remapped SST parent directory");
465 }
466 fs::rename(&old_path, &new_path).unwrap_or_else(|err| {
467 panic!(
468 "failed to remap SST file {} -> {}: {err}",
469 old_path.display(),
470 new_path.display()
471 )
472 });
473}
474
475fn case_name_from_path(path: &Path) -> String {
476 path.parent()
477 .and_then(Path::file_name)
478 .and_then(|name| name.to_str())
479 .unwrap_or("query_perf_case")
480 .to_string()
481}
482
483#[tokio::main]
484async fn main() {
485 let args = Args::parse();
486 let case_text = fs::read_to_string(&args.case).expect("failed to read query perf case TOML");
487 let case: CaseFile = toml::from_str(&case_text).expect("failed to parse query perf case TOML");
488 let case_name = case_name_from_path(&args.case);
489 let scenario = case.scenario.direct_readable_sst();
490 if scenario.tables.is_empty() || scenario.layout.regions != 1 {
491 panic!("fixture generator supports one or more tables and exactly one region per table");
492 }
493 if scenario.layout.time_range_layout != "non_overlapping_per_sst" {
494 panic!("MVP supports time_range_layout=non_overlapping_per_sst");
495 }
496 if scenario.layout.sst_count > 1000 && !args.allow_large {
497 panic!("sst_count exceeds 1000; pass --allow-large");
498 }
499 let seed = scenario.seed.unwrap_or(0);
500 let table_index = match args.table.as_deref() {
501 Some(name) => scenario
502 .tables
503 .iter()
504 .position(|table| table.name == name)
505 .unwrap_or_else(|| panic!("--table {name} was not found in case {case_name}")),
506 None if scenario.tables.len() == 1 => 0,
507 None => panic!(
508 "case {} contains {} tables; pass --table <name> to choose one",
509 case_name,
510 scenario.tables.len()
511 ),
512 };
513 let table = &scenario.tables[table_index];
514 if table.engine != "mito" {
515 panic!("MVP supports engine=mito");
516 }
517 let region_id = RegionId::from(args.region_id);
518 let table_dir = args
519 .table_dir
520 .unwrap_or_else(|| format!("data/{}/{}/", table.database, table.name));
521 if Path::new(&table_dir).is_absolute() {
522 panic!("--table-dir must be relative");
523 }
524 if table_dir.trim_end_matches('/').rsplit('/').next()
525 == Some(region_name(region_id.table_id(), region_id.region_sequence()).as_str())
526 {
527 panic!("--table-dir must be table-level directory");
528 }
529 let region_dir =
530 mito2::sst::location::region_dir_from_table_dir(&table_dir, region_id, PathType::Bare);
531 let format = match table.sst_format.as_deref().unwrap_or("flat") {
532 "flat" => FormatType::Flat,
533 "primary_key" | "primary-key" => FormatType::PrimaryKey,
534 other => panic!("unsupported sst_format {other}"),
535 };
536 println!(
537 "query_perf_fixture case={} scenario={} table={}.{} ssts={} rows_per_sst={} format={format:?}",
538 case_name,
539 case.scenario.kind(),
540 table.database,
541 table.name,
542 scenario.layout.sst_count,
543 scenario.layout.rows_per_sst
544 );
545 println!(
546 "out_dir={} table_dir={} region_dir={}",
547 args.out_dir.display(),
548 table_dir,
549 region_dir
550 );
551 if args.dry_run {
552 return;
553 }
554
555 let obj_store_dir = args.out_dir.join("object-store");
556 let manifest_dir = args.out_dir.join("manifest");
557 fs::create_dir_all(&obj_store_dir).expect("failed to create fixture object-store directory");
558 fs::create_dir_all(&manifest_dir).expect("failed to create fixture manifest directory");
559 let ostorage = ObjectStore::new(FsBuilder::default().root(&obj_store_dir.to_string_lossy()))
560 .expect("failed to create filesystem object store for fixture output")
561 .finish();
562 let metadata: RegionMetadataRef = Arc::new(build_region_metadata(table, region_id));
563 let mut files = HashMap::with_capacity(scenario.layout.sst_count);
564 let mut next_file_index = 1;
565 for i in 0..scenario.layout.sst_count {
566 let sequence = 1000 + i as u64;
567 let batch = generate_record_batch(table, &metadata, &scenario.layout, i, sequence);
568 let source =
569 FlatSource::new_iter(batch.schema(), Box::new(vec![batch].into_iter().map(Ok)));
570 let mut metrics = Metrics::new(WriteType::Flush);
571 let mut writer = ParquetWriter::new_with_object_store(
572 ostorage.clone(),
573 metadata.clone(),
574 IndexConfig::default(),
575 NoopIndexBuilder,
576 FixedPathProvider {
577 table_dir: table_dir.clone(),
578 },
579 &mut metrics,
580 )
581 .await;
582 let opts = WriteOptions {
583 write_buffer_size: DEFAULT_WRITE_BUFFER_SIZE,
584 row_group_size: scenario.layout.row_group_size,
585 max_file_size: None,
586 };
587 let infos = match format {
588 FormatType::Flat => writer.write_all_flat(source, Some(sequence), &opts).await,
589 FormatType::PrimaryKey => {
590 writer
591 .write_all_flat_as_primary_key(source, Some(sequence), &opts)
592 .await
593 }
594 }
595 .expect("failed to write fixture SST through Mito parquet writer");
596 for info in infos {
597 let file_id = deterministic_file_id(
598 seed.wrapping_add((table_index as u64) << 16),
599 next_file_index,
600 );
601 next_file_index += 1;
602 remap_sst_file(&obj_store_dir, &table_dir, region_id, info.file_id, file_id);
603 files.insert(
604 file_id,
605 file_meta_from_sst_info(&info, region_id, file_id, sequence),
606 );
607 }
608 }
609 let overall_seq = files
610 .values()
611 .filter_map(|m| m.sequence)
612 .map(|s| s.get())
613 .max()
614 .unwrap_or(0);
615 let manifest = RegionManifest {
616 metadata,
617 files,
618 removed_files: RemovedFilesRecord::default(),
619 flushed_entry_id: overall_seq,
620 flushed_sequence: overall_seq,
621 committed_sequence: Some(overall_seq),
622 manifest_version: args.checkpoint_version,
623 truncated_entry_id: None,
624 compaction_time_window: None,
625 sst_format: format,
626 append_mode: table.append_mode,
627 };
628 let checkpoint = RegionCheckpoint {
629 last_version: args.checkpoint_version,
630 compacted_actions: manifest.files.len(),
631 checkpoint: Some(manifest.clone()),
632 };
633 let checkpoint_bytes = checkpoint
634 .encode()
635 .expect("failed to encode fixture region checkpoint");
636 let checkpoint_path = manifest_dir.join(format!("{:020}.checkpoint", args.checkpoint_version));
637 fs::write(&checkpoint_path, &checkpoint_bytes).expect("failed to write checkpoint file");
638 fs::write(manifest_dir.join("_last_checkpoint"), serde_json::to_vec_pretty(&serde_json::json!({ "size": checkpoint_bytes.len(), "version": args.checkpoint_version, "checksum": null, "extend_metadata": {} })).expect("failed to serialize _last_checkpoint metadata")).expect("failed to write _last_checkpoint file");
639 let files_jsonl_path = args.out_dir.join("files.jsonl");
640 let mut jsonl = BufWriter::new(
641 fs::File::create(&files_jsonl_path).expect("failed to create fixture files.jsonl"),
642 );
643 let mut entries: Vec<_> = manifest.files.iter().collect();
644 entries.sort_by_key(|(id, _)| id.to_string());
645 for (file_id, meta) in entries {
646 writeln!(jsonl, "{}", serde_json::to_string(&serde_json::json!({ "file_id": file_id.to_string(), "region_id": meta.region_id.as_u64(), "object_path": mito2::sst::location::sst_file_path(&table_dir, RegionFileId::new(meta.region_id, *file_id), PathType::Bare), "time_range_start": meta.time_range.0.value(), "time_range_end": meta.time_range.1.value(), "num_rows": meta.num_rows, "num_row_groups": meta.num_row_groups, "file_size": meta.file_size, "num_series": meta.num_series, "sequence": meta.sequence.map(|s| s.get()) })).expect("failed to serialize fixture SST metadata JSONL entry")).expect("failed to write fixture SST metadata JSONL entry");
647 }
648 jsonl.flush().expect("failed to flush fixture files.jsonl");
649 fs::write(args.out_dir.join("summary.json"), serde_json::to_vec_pretty(&serde_json::json!({ "case": case_name, "seed": seed, "table_index": table_index, "table": table.name, "database": table.database, "region_id": region_id.as_u64(), "table_dir": table_dir, "region_dir": region_dir, "sst_format": format!("{format:?}"), "sst_count": scenario.layout.sst_count, "rows_per_sst": scenario.layout.rows_per_sst, "row_group_size": scenario.layout.row_group_size, "total_rows": scenario.layout.sst_count * scenario.layout.rows_per_sst, "checkpoint_path": checkpoint_path, "files_jsonl_path": files_jsonl_path, "readback_validated": false, "metadata_source": "synthetic" })).expect("failed to serialize fixture summary")).expect("failed to write fixture summary.json");
650 println!("Done. wrote {} SST file entries", manifest.files.len());
651}