diff --git a/Cargo.lock b/Cargo.lock index 3b66b13e949..61a9c246ed1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8702,6 +8702,7 @@ dependencies = [ "api", "aquamarine", "arrow-schema 59.2.0", + "arrow-select 59.2.0", "async-channel 1.9.0", "async-stream", "async-trait", @@ -9686,9 +9687,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" [[package]] name = "opendal" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fe43e16d96bed57937eb7c4a28559bf4c5fc4a3951656505d1fdd4f856349b4" +checksum = "3cbd193aec6d62efaff87611dfae52c132aed650bb3cb0c684c0a9394f09a42a" dependencies = [ "ctor", "opendal-core", @@ -9711,9 +9712,9 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de4566a412776e3f65d53dd9fceced5b432a0a9c9a1e8d50ccf58823795e5c5b" +checksum = "54b5555d753dd6a6383dc0891254b72f3cdc40c011ba0372b560e0ae5aa29f5c" dependencies = [ "anyhow", "asyncband", @@ -9737,9 +9738,9 @@ dependencies = [ [[package]] name = "opendal-http-transport-reqwest" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f772ce6c137ab43647726116d505a649a88d41928fcd813438eb30b02312bcbb" +checksum = "4d6588678f3040a3eab1819364dcbc6a746a8286d1ceb23a70a6614faa01dc6c" dependencies = [ "bytes", "futures", @@ -9751,9 +9752,9 @@ dependencies = [ [[package]] name = "opendal-layer-concurrent-limit" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3734c648617e5ad724d49175dde8323550624f9614f50e9e81b7aafae30b1734" +checksum = "db8574309e68bf6d49c5378eaef9d0b3f6786971650b061c69e808254d71cb05" dependencies = [ "asyncband", "futures", @@ -9763,9 +9764,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a8daab5a91eac76f94516e7c71e677195c9221840ce8f5e71815e3fc8207742" +checksum = "4ad43246d2cb7b7ad32b10e22bfa7b6ced8a6677cb6d6f005ae7e1d94cb19c6a" dependencies = [ "log", "opendal-core", @@ -9773,9 +9774,9 @@ dependencies = [ [[package]] name = "opendal-layer-observe-metrics-common" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fbe664be117c293cfa3980a0d2fc238bbb38eae5f6c191f376e88ccb830d26b" +checksum = "3b8bc97d5a00eb41d50f1f6077c4bd5396c1eefea495614084ee9a37202ae06d" dependencies = [ "futures", "http 1.5.0", @@ -9784,9 +9785,9 @@ dependencies = [ [[package]] name = "opendal-layer-prometheus" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b32078d8248555ee6271332ec480b78306a44f0d902ea4721ccac00d9b866a21" +checksum = "404e766cc2e4e4c9225ce08605ae151f768a05a41a448a235e016b1aea9ff98f" dependencies = [ "opendal-core", "opendal-layer-observe-metrics-common", @@ -9795,9 +9796,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb8e22cb1f86ef9af771a8947d3ccf380c8f5b76236f628314dd41c9ba73331a" +checksum = "4aeb54be4940c68609483591934a813f7bdd97d747826783a393914d3139e363" dependencies = [ "backon", "log", @@ -9806,9 +9807,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08ac6c77fe0d1b18e0064a02c19b98dbebcab78bf5e7753ad1f3abbc18850ff4" +checksum = "1d11d53e67473e8914eacb2e14d586779ae57e1a84759ad25232ef339db77d8a" dependencies = [ "opendal-core", "tokio", @@ -9816,9 +9817,9 @@ dependencies = [ [[package]] name = "opendal-layer-tracing" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "603807927a799ea981f62a3b0012e17181e39f468793b78ca8bedda66d19813c" +checksum = "bbe2ebfc795bf08da77d20bed13d149b478cc8e5002df44d237e460134f00be4" dependencies = [ "futures", "http 1.5.0", @@ -9828,9 +9829,9 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6cf31b02e7b44191d97c07254eea109b5da9c17210a29e478c8d0dc26dffb1f" +checksum = "c32c3fc2e507c854ce53615b28fc2208f5e00cbe456c4d344ffdfbf4c9cb976d" dependencies = [ "base64 0.23.1", "bytes", @@ -9849,9 +9850,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8107324c3f3cb9970fe20526c2c341b7c5769acbd4699bab73bd525f5fd63d62" +checksum = "cc93c30bd347a36d93fd46d47f3e2bb6494dcf9458938aa501fe046ff8f7562f" dependencies = [ "http 1.5.0", "opendal-core", @@ -9859,9 +9860,9 @@ dependencies = [ [[package]] name = "opendal-service-fs" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0e7d8974aebd33be899200c9d0d50b2141fd5fa5c462b29e20653e3ddbfe339" +checksum = "ec5c16fe91b77c0364f66546a1c32b2999e9c3c3c75c2e908a282268e0890bac" dependencies = [ "bytes", "log", @@ -9873,9 +9874,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d80eca6f9227bdf65d43e5b8d5fe75dca8cfab1da07571442a0420400847bea" +checksum = "e1b5e42c6606ea1e37fa9cdedf07dc998073a45eb5044317c0fa6f3153e98d6e" dependencies = [ "async-trait", "bytes", @@ -9895,9 +9896,9 @@ dependencies = [ [[package]] name = "opendal-service-hdfs-native" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b84340e11a3bcd7f1f0b55df032d040b2907483c546c3021485aba749c3072c" +checksum = "20fb97b2ac06a87e6720f0f93d5107ac046117deb8ac957030b0962ba4ecc3c5" dependencies = [ "bytes", "futures", @@ -9909,9 +9910,9 @@ dependencies = [ [[package]] name = "opendal-service-http" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ac1553d6b57ca79e200e5c7ded9ab708f2016dadb28ede0f2da71e02fd0b960" +checksum = "65fdde60d1abf14e47eaf19360c08ea1b89c14f03769db448e016781c2e0e32b" dependencies = [ "http 1.5.0", "log", @@ -9921,9 +9922,9 @@ dependencies = [ [[package]] name = "opendal-service-mysql" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e15592be32b105cd152ab398b3db847e22c8c5f230d9b34ff095ea3d0dc6fd97" +checksum = "9ba30e2a0b38aa55579b69bf57f607df7aa613f4d964496407b7998f2dbaefcc" dependencies = [ "asyncband", "opendal-core", @@ -9933,9 +9934,9 @@ dependencies = [ [[package]] name = "opendal-service-oss" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ebfa355b1d4edb0e5b2a5b86d8de9f811fae1d7baca6d65246b230b2afc0bd" +checksum = "d64d2909b444008e42f00137a50df6b55ba599eeb6c8b46c71a2a84948f91880" dependencies = [ "bytes", "http 1.5.0", @@ -9950,9 +9951,9 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.59.2" +version = "0.59.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2942f2d8d3d4953c0e8d07cd1244879628d084948e48ce6b6d808c5938f7858d" +checksum = "1d83e716c25a81fe2ca701894b8e14b9966acca3812af250e50132accb19da02" dependencies = [ "base64 0.23.1", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 549412f369a..f47907aa797 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,6 +107,7 @@ arrow-buffer = "=59.2.0" arrow-flight = "=59.2.0" arrow-ipc = { version = "=59.2.0", default-features = false, features = ["lz4", "zstd"] } arrow-schema = { version = "=59.2.0", features = ["serde"] } +arrow-select = "=59.2.0" async-stream = "0.3" async-trait = "0.1" # Remember to update axum-extra, axum-macros when updating axum diff --git a/docs/rfcs/2026-09-11-metric-export-import.md b/docs/rfcs/2026-09-11-metric-export-import.md index 8ddc31f40b0..57212ea53cb 100644 --- a/docs/rfcs/2026-09-11-metric-export-import.md +++ b/docs/rfcs/2026-09-11-metric-export-import.md @@ -31,13 +31,21 @@ includes reports of exports taking days for roughly 100,000 logical tables with only several GiB of data. Table count, physical schema width and data volume must be measured separately; total database bytes do not describe the transfer cost. -The [experiment conclusions](2026-09-11-metric-export-import/experiment-results.md) -support shared scans, batch DDL, explicit-file COPY and packed-object transfer. -Packing with a bounded range reader substantially reduces storage requests and -improves full export and restore for many small logical tables. Benefits depend -on request latency, and memory use increases. Remaining CREATE work makes batch -DDL part of the first release. These local experiments establish direction; -production V2 recovery and release performance still require acceptance tests. +Local standalone experiments with development builds, local files and Docker +MinIO support shared scans, batch DDL, explicit-file COPY and packed-object +transfer. Packing with a bounded range reader reduces storage requests and +improves full export and restore for many small logical tables; low-latency +export gains are smaller. CPU and request overhead fell while peak process +memory increased. Remaining CREATE work makes batch DDL part of the first +release. Explicit-file COPY removes repeated directory traversal while retaining +ordinary per-table insertion, independently of merged database writes. + +The exercised fixtures preserved schemas, DDL and typed values. Standalone-file +experiments used the existing importer; packing used an independent driver and +prototype range reader. These experiments establish design direction, not +production packed-format compatibility, V2 recovery safety or release performance. +The 8 MiB write/part limit was a design choice, not a measured optimum. Current +acceptance status is maintained in the [tracking issue](https://github.com/GreptimeTeam/greptimedb/issues/9120). # Terminology @@ -552,18 +560,39 @@ batching, still writing standalone table objects; (C) B plus packing and its reader, with identical DDL handling; (D) C plus batch DDL. If general restore request deduplication is available, include it in B as well. -The proposed release target is at least 3x faster full export and full restore -for D versus A on 10,000 and 100,000 small logical tables with MinIO request -latency increased by 5 ms. Fix this target during RFC review, before running -release measurements. The independent local C-versus-B experiment establishes -neither this comparison nor proof of that target. +The release target is at least 3x faster full export and full restore for D +versus A on both 10,000 and 100,000 small logical tables at both +50 ms and ++100 ms added MinIO request latency. Populate every selected table with the same +deterministic data across controls, initially 32 rows per table. Include schema +export, metadata discovery and CLI startup/shutdown in complete wall duration. +The independent local C-versus-B experiment establishes neither this comparison +nor proof of that target. Keep production improvements already shared by all +controls in A, and identify which increments can actually be isolated. -Use Linux release builds and at least three alternating rounds for primary -comparisons. Report medians/spread, schema/data/full durations, CPU, RSS, bytes, -objects, request counts and queue/cache peaks. Include zero-added-latency, -local-file, mixed small/large-table and wide-schema controls. Investigate and -remeasure full-duration regressions exceeding 5% versus A; do not generalize the -latency-injected speedup to these cases. Check values and schemas outside timing. +Start with native optimized release builds on macOS. Record hardware, OS, +MinIO placement, container/VM CPU and memory limits, and connection/storage +settings. Linux deployment-side performance confirmation can follow separately; +Linux root/non-root and Windows correctness checks remain distinct requirements. +Earlier development-build diagnostics, runs with paging interference and sparse +fixtures with only one populated table do not establish this acceptance. + +Calibrate one added delay per object-store HTTP request, including HEAD, GET +(and range GET), PUT, LIST, DELETE and each multipart operation. Do not delay +individual packets, streaming body chunks or database SQL/DDL. Route CLI and +server object access through the same controlled endpoint and report measured +per-method latency distributions. Synthetic delay is not a cloud provider profile. + +Use a small calibration pilot before each scale and at least three alternating +A/B/C/D rounds for primary comparisons, with fresh restore namespaces, equal +concurrency and comparable cache/compaction conditions. Report medians/spread, +schema/data/full durations, CPU, RSS, swap activity, bytes, objects, requests and +queue/cache peaks. Check RAM, disk and MinIO health before large runs; do not +compile concurrently. Fix interfering paging/load or report the scale unverified. +Retain local-file and MinIO 0/+5 ms controls, mixed small/large tables, wide +physical column unions and multiple physical groups. Add +20 ms only if needed +to explain sensitivity. Investigate and remeasure full-duration regressions +exceeding 5% versus A. Check table membership, schemas and typed values outside +timing, and report B/A, C/B and D/C independently where comparable. Validate actual write/part sizes at or below 8 MiB, aggregate concurrency across groups/chunks, and resource release after success, cancellation and failure. diff --git a/docs/rfcs/2026-09-11-metric-export-import/experiment-results.md b/docs/rfcs/2026-09-11-metric-export-import/experiment-results.md deleted file mode 100644 index 9cc7b46464d..00000000000 --- a/docs/rfcs/2026-09-11-metric-export-import/experiment-results.md +++ /dev/null @@ -1,34 +0,0 @@ -# Metric export/import experiment conclusions - -The experiments support shared physical scans, explicit-file COPY, batched -logical DDL, and packing small logical-table Parquet streams into shared objects. - -- **Shared scans reduce repeated query work.** Projecting each logical schema - before dictionary expansion preserves its columns and types, but does not - remove the memory cost of a wide physical schema or distributed ordering. -- **Explicit-file COPY removes repeated directory traversal.** This improvement - uses ordinary per-table insertion; it is independent of database merged writes. -- **Packing and its range reader reduce small-object overhead.** They improve - complete export and restore and sharply reduce object-store requests. Benefits - grow with table count and request latency; low-latency export gains are smaller. - Some reader request savings can also benefit standalone files. -- **Batch DDL remains necessary.** Once data restore is faster, logical-table - creation becomes the main remaining restore cost. The batch-DDL experiment used - an internal interface; production transport and retry integration remain work. -- **Bounded writes allow larger objects.** The packing experiment kept writes - and upload parts within 8 MiB without imposing that limit on whole objects or - splitting logical tables. This was the chosen size, not a measured optimum. -- **Packing has a memory trade-off.** CPU and request overhead fell, while peak - process memory increased. Encoding buffers, upload concurrency, range windows - and table/index metadata need separate accounting. - -The exercised fixtures preserved schemas, DDL and typed values after restore. -Earlier standalone-file experiments used the existing importer; the packing -experiment used an independent driver and prototype range reader. The latter -does not establish production packed-format compatibility or V2 resume safety. - -These were local standalone experiments using development builds, local files -and Docker MinIO. Release acceptance still needs the actual packed V2 path, -request-owned reader isolation, old-reader rejection, large-table throughput, -distributed ordering, cancellation and recovery. See the [RFC](../2026-09-11-metric-export-import.md) -and [tracking issue](https://github.com/GreptimeTeam/greptimedb/issues/9120). diff --git a/src/cli/src/data/snapshot_storage.rs b/src/cli/src/data/snapshot_storage.rs index 17008706066..f8dd22f0307 100644 --- a/src/cli/src/data/snapshot_storage.rs +++ b/src/cli/src/data/snapshot_storage.rs @@ -21,6 +21,7 @@ use std::collections::BTreeSet; use std::path::Component; use async_trait::async_trait; +use common_datasource::packed_writer::WRITE_BYTES; use futures::TryStreamExt; use object_store::services::{Azblob, Fs, Gcs, Oss, S3}; use object_store::util::{with_instrument_layers, with_retry_layers}; @@ -570,13 +571,31 @@ impl OpenDalStorage { /// Writes bytes to a file. async fn write_file(&self, path: &str, data: Vec) -> Result<()> { - self.object_store - .write(path, data) + let mut writer = self + .object_store + .writer_with(path) + .chunk(WRITE_BYTES) + .concurrent(1) .await - .map(|_| ()) - .context(StorageOperationSnafu { - operation: format!("write {}", path), - }) + .with_context(|_| StorageOperationSnafu { + operation: format!("open writer {}", path), + })?; + let data = object_store::Buffer::from(data); + let result = async { + for chunk in data.chunks(WRITE_BYTES) { + writer.write(chunk).await?; + } + writer.close().await.map(|_| ()) + } + .await; + if result.is_err() + && let Err(error) = writer.abort().await + { + common_telemetry::warn!(error; "Failed to abort snapshot metadata write"); + } + result.with_context(|_| StorageOperationSnafu { + operation: format!("write {}", path), + }) } /// Checks if a file exists using stat. @@ -837,6 +856,96 @@ mod tests { ) } + #[tokio::test] + async fn snapshot_metadata_submissions_are_bounded_and_invoke_abort_on_failure() { + use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + + use object_store::layers::mock::{Metadata, ObservedWriter, WriterObservations, oio}; + + struct FailWriter { + inner: oio::Writer, + writes: usize, + fail: Arc, + } + impl oio::Write for FailWriter { + async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> { + self.writes += 1; + if self.fail.load(Ordering::SeqCst) == 1 && self.writes == 2 { + return Err(object_store::Error::new( + ErrorKind::Unexpected, + "injected metadata write failure", + )); + } + self.inner.write(bytes).await + } + async fn close(&mut self) -> object_store::Result { + if self.fail.load(Ordering::SeqCst) == 2 { + return Err(object_store::Error::new( + ErrorKind::Unexpected, + "injected metadata close failure", + )); + } + self.inner.close().await + } + async fn abort(&mut self) -> object_store::Result<()> { + self.inner.abort().await + } + } + + let dir = tempdir().unwrap(); + let mut storage = make_storage_with_rooted_fs(dir.path()); + let observations = Arc::new(Mutex::new(WriterObservations::default())); + let fail = Arc::new(AtomicUsize::new(0)); + storage.object_store = storage.object_store.layer( + MockLayerBuilder::default() + .writer_factory(Arc::new({ + let observations = observations.clone(); + let fail = fail.clone(); + move |_, _, inner| { + Box::new(ObservedWriter::new( + Box::new(FailWriter { + inner, + writes: 0, + fail: fail.clone(), + }), + observations.clone(), + )) + } + })) + .build() + .unwrap(), + ); + for length in [2 * WRITE_BYTES + 17, 1, 0] { + *observations.lock().unwrap() = WriterObservations::default(); + let data = vec![42; length]; + storage.write_file("metadata", data.clone()).await.unwrap(); + assert_eq!(storage.read_file("metadata").await.unwrap(), data); + let observed = observations.lock().unwrap(); + assert_eq!(observed.write_sizes.iter().sum::(), length); + assert!(observed.write_sizes.iter().all(|size| *size <= WRITE_BYTES)); + assert_eq!(observed.close_calls, 1); + assert_eq!(observed.abort_calls, 0); + } + for failure in [1, 2] { + *observations.lock().unwrap() = WriterObservations::default(); + fail.store(failure, Ordering::SeqCst); + assert!( + storage + .write_file("failed", vec![42; 2 * WRITE_BYTES + 17]) + .await + .is_err() + ); + assert_eq!( + observations.lock().unwrap().write_sizes.len(), + if failure == 1 { 2 } else { 3 } + ); + let observed = observations.lock().unwrap(); + assert_eq!(observed.close_calls, usize::from(failure == 2)); + assert_eq!(observed.abort_calls, 1); + } + } + #[test] fn test_validate_uri_valid() { assert_eq!(validate_uri("s3://bucket/path").unwrap(), StorageScheme::S3); diff --git a/src/common/datasource/src/packed_writer.rs b/src/common/datasource/src/packed_writer.rs index 635a1a7a771..d0560fa06f7 100644 --- a/src/common/datasource/src/packed_writer.rs +++ b/src/common/datasource/src/packed_writer.rs @@ -410,7 +410,9 @@ mod tests { use arrow::array::{ArrayRef, Int64Array, StringArray}; use arrow::record_batch::RecordBatch; use futures::TryStreamExt; - use object_store::layers::mock::{Metadata, MockLayerBuilder, MockWriterFactory, oio}; + use object_store::layers::mock::{ + Metadata, MockLayerBuilder, MockWriterFactory, ObservedWriter, WriterObservations, oio, + }; use parquet::arrow::ParquetRecordBatchStreamBuilder; use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder; @@ -424,10 +426,10 @@ mod tests { let store = object_store::secure_fs::SecureFsRoot::open(directory.path()) .unwrap() .build_operator(); - let sizes = Arc::new(std::sync::Mutex::new(Vec::new())); + let observations = Arc::new(std::sync::Mutex::new(WriterObservations::default())); let factory: MockWriterFactory = Arc::new({ - let sizes = sizes.clone(); - move |_, _, writer| Box::new(Observe(writer, sizes.clone())) + let observations = observations.clone(); + move |_, _, writer| Box::new(ObservedWriter::new(writer, observations.clone())) }); let store = store.layer( MockLayerBuilder::default() @@ -483,7 +485,14 @@ mod tests { .validate_membership(["table0", "table1", "table2"]) .unwrap(); assert_eq!(inventory.len(), 3); - assert!(sizes.lock().unwrap().iter().all(|n| *n <= WRITE_BYTES)); + assert!( + observations + .lock() + .unwrap() + .write_sizes + .iter() + .all(|n| *n <= WRITE_BYTES) + ); assert_eq!( index .objects @@ -540,27 +549,13 @@ mod tests { } } - struct Observe(oio::Writer, Arc>>); - impl oio::Write for Observe { - async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> { - self.1.lock().unwrap().push(bytes.len()); - self.0.write(bytes).await - } - async fn close(&mut self) -> object_store::Result { - self.0.close().await - } - async fn abort(&mut self) -> object_store::Result<()> { - self.0.abort().await - } - } - #[tokio::test] async fn rolls_at_table_boundary_and_aborts_standalone_at_part_limit() { let directory = common_test_util::temp_dir::create_temp_dir("packed-limit"); - let sizes = Arc::new(std::sync::Mutex::new(Vec::new())); + let observations = Arc::new(std::sync::Mutex::new(WriterObservations::default())); let factory: MockWriterFactory = Arc::new({ - let sizes = sizes.clone(); - move |_, _, writer| Box::new(Observe(writer, sizes.clone())) + let observations = observations.clone(); + move |_, _, writer| Box::new(ObservedWriter::new(writer, observations.clone())) }); let store = object_store::secure_fs::SecureFsRoot::open(directory.path()) .unwrap() @@ -605,7 +600,14 @@ mod tests { .iter() .all(|t| t.offset == 0 && t.length == 600) ); - assert!(sizes.lock().unwrap().iter().all(|n| *n <= 512)); + assert!( + observations + .lock() + .unwrap() + .write_sizes + .iter() + .all(|n| *n <= 512) + ); } struct FailClose(oio::Writer); diff --git a/src/common/datasource/src/parquet_writer.rs b/src/common/datasource/src/parquet_writer.rs index 243a46b4e1a..157733e856e 100644 --- a/src/common/datasource/src/parquet_writer.rs +++ b/src/common/datasource/src/parquet_writer.rs @@ -500,28 +500,17 @@ mod tests { async fn large_footer_flush_uses_bounded_submissions_in_one_parquet_stream() { use std::sync::Mutex; - use object_store::layers::mock::{Metadata, MockLayerBuilder, MockWriterFactory, oio}; - struct ObservedWriter(oio::Writer, Arc>>); - impl oio::Write for ObservedWriter { - async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> { - self.1.lock().unwrap().push(bytes.len()); - self.0.write(bytes).await - } - async fn close(&mut self) -> object_store::Result { - self.0.close().await - } - async fn abort(&mut self) -> object_store::Result<()> { - self.0.abort().await - } - } + use object_store::layers::mock::{ + MockLayerBuilder, MockWriterFactory, ObservedWriter, WriterObservations, + }; let directory = common_test_util::temp_dir::create_temp_dir("bounded_parquet"); let store = object_store::secure_fs::SecureFsRoot::open(directory.path()) .unwrap() .build_operator(); - let sizes = Arc::new(Mutex::new(Vec::new())); + let observations = Arc::new(Mutex::new(WriterObservations::default())); let factory: MockWriterFactory = Arc::new({ - let sizes = sizes.clone(); - move |_, _, writer| Box::new(ObservedWriter(writer, sizes.clone())) + let observations = observations.clone(); + move |_, _, writer| Box::new(ObservedWriter::new(writer, observations.clone())) }); let store = store.layer( MockLayerBuilder::default() @@ -548,7 +537,7 @@ mod tests { writer.sink = ParquetSink::Object(store.writer("large.parquet").await.unwrap()); writer.write(batch.clone(), None).await.unwrap(); writer.finish(None).await.unwrap(); - let sizes = sizes.lock().unwrap().clone(); + let sizes = observations.lock().unwrap().write_sizes.clone(); let limit = DEFAULT_WRITE_BUFFER_SIZE.as_bytes() as usize; assert!(sizes.iter().sum::() > limit); assert!(sizes.iter().all(|size| *size <= limit), "{sizes:?}"); diff --git a/src/common/function/src/aggrs.rs b/src/common/function/src/aggrs.rs index e618921a18d..bad38a075d4 100644 --- a/src/common/function/src/aggrs.rs +++ b/src/common/function/src/aggrs.rs @@ -14,6 +14,7 @@ pub mod approximate; pub mod count_hash; +pub mod extremum; #[cfg(feature = "geo")] pub mod geo; pub mod vector; diff --git a/src/common/function/src/aggrs/extremum.rs b/src/common/function/src/aggrs/extremum.rs new file mode 100644 index 00000000000..668c307ff36 --- /dev/null +++ b/src/common/function/src/aggrs/extremum.rs @@ -0,0 +1,259 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! PromQL `min` and `max` aggregation operators. +//! +//! PromQL ignores NaN samples unless every sample of a group is NaN. DataFusion's +//! `min`/`max` order NaN differently depending on the code path (its grouped +//! accumulator lets NaN replace the current value and be replaced by the next one), +//! so the result depends on the order of the input. + +use std::sync::Arc; + +use datafusion::arrow::array::{ArrayRef, AsArray}; +use datafusion::arrow::datatypes::{DataType, Field, FieldRef, Float64Type}; +use datafusion::error::Result as DfResult; +use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs}; +use datafusion::logical_expr::{ + Accumulator, AggregateUDF, AggregateUDFImpl, GroupsAccumulator, Signature, Volatility, +}; +use datafusion_common::ScalarValue; +use datafusion_functions_aggregate_common::aggregate::groups_accumulator::prim_op::PrimitiveGroupsAccumulator; + +use crate::function_registry::FunctionRegistry; + +/// `max` or `min` with PromQL NaN semantics. +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct Extremum { + name: &'static str, + is_max: bool, + signature: Signature, +} + +impl Extremum { + pub const MAX_NAME: &'static str = "prom_max"; + pub const MIN_NAME: &'static str = "prom_min"; + + /// Registers both functions; the registry also derives their state functions, which + /// distributed plans need to split the aggregation between datanodes and the frontend. + pub fn register(registry: &FunctionRegistry) { + registry.register_aggr(Self::max_udaf()); + registry.register_aggr(Self::min_udaf()); + } + + pub fn max_udaf() -> AggregateUDF { + AggregateUDF::from(Self::new(Self::MAX_NAME, true)) + } + + pub fn min_udaf() -> AggregateUDF { + AggregateUDF::from(Self::new(Self::MIN_NAME, false)) + } + + fn new(name: &'static str, is_max: bool) -> Self { + Self { + name, + is_max, + signature: Signature::exact(vec![DataType::Float64], Volatility::Immutable), + } + } +} + +/// Folds `value` into `current` like Prometheus' aggregation: a NaN is replaced by any +/// sample, and a NaN sample never replaces a number. +fn fold(is_max: bool, current: &mut f64, value: f64) { + let better = if is_max { + value > *current + } else { + value < *current + }; + if current.is_nan() || better { + *current = value; + } +} + +impl AggregateUDFImpl for Extremum { + fn name(&self) -> &str { + self.name + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> DfResult { + Ok(DataType::Float64) + } + + fn accumulator(&self, _acc_args: AccumulatorArgs) -> DfResult> { + Ok(Box::new(ExtremumAccumulator { + is_max: self.is_max, + value: None, + })) + } + + fn state_fields(&self, args: StateFieldsArgs) -> DfResult> { + Ok(vec![Arc::new(Field::new( + format!("{}[{}]", args.name, self.name), + DataType::Float64, + true, + ))]) + } + + fn groups_accumulator_supported(&self, _args: AccumulatorArgs) -> bool { + true + } + + fn create_groups_accumulator( + &self, + _args: AccumulatorArgs, + ) -> DfResult> { + Ok(groups_accumulator(self.is_max)) + } +} + +fn groups_accumulator(is_max: bool) -> Box { + Box::new( + PrimitiveGroupsAccumulator::::new(&DataType::Float64, move |cur, v| { + fold(is_max, cur, v) + }) + .with_starting_value(f64::NAN), + ) +} + +#[derive(Debug)] +struct ExtremumAccumulator { + is_max: bool, + value: Option, +} + +impl ExtremumAccumulator { + fn update(&mut self, values: &ArrayRef) { + for v in values.as_primitive::().iter().flatten() { + match &mut self.value { + Some(current) => fold(self.is_max, current, v), + None => self.value = Some(v), + } + } + } +} + +impl Accumulator for ExtremumAccumulator { + fn update_batch(&mut self, values: &[ArrayRef]) -> DfResult<()> { + self.update(&values[0]); + Ok(()) + } + + fn merge_batch(&mut self, states: &[ArrayRef]) -> DfResult<()> { + self.update(&states[0]); + Ok(()) + } + + fn state(&mut self) -> DfResult> { + Ok(vec![ScalarValue::Float64(self.value)]) + } + + fn evaluate(&mut self) -> DfResult { + Ok(ScalarValue::Float64(self.value)) + } + + fn size(&self) -> usize { + size_of_val(self) + } +} + +#[cfg(test)] +mod tests { + use datafusion::arrow::array::Float64Array; + use datafusion::logical_expr::EmitTo; + + use super::*; + + fn groups_result( + is_max: bool, + values: Vec>, + groups: Vec, + ) -> Vec> { + let num_groups = groups.iter().max().map_or(0, |g| g + 1); + let mut acc = groups_accumulator(is_max); + let values: ArrayRef = Arc::new(Float64Array::from(values)); + acc.update_batch(&[values], &groups, None, num_groups) + .unwrap(); + let out = acc.evaluate(EmitTo::All).unwrap(); + out.as_primitive::().iter().collect() + } + + fn is_nan(v: Option) -> bool { + v.is_some_and(f64::is_nan) + } + + #[test] + fn nan_is_ignored_unless_the_group_is_all_nan() { + // Group 0 has NaN on both sides of a number; group 1 is all NaN; group 2 has only NULL. + let values = vec![ + Some(f64::NAN), + Some(3.0), + Some(f64::NAN), + Some(f64::NAN), + None, + ]; + let groups = vec![0, 0, 0, 1, 2]; + for is_max in [true, false] { + let out = groups_result(is_max, values.clone(), groups.clone()); + assert_eq!(out[0], Some(3.0)); + assert!(is_nan(out[1])); + assert_eq!(out[2], None); + } + } + + #[test] + fn result_does_not_depend_on_input_order() { + let forward = vec![Some(1.0), Some(f64::NAN), Some(4.0), Some(2.0)]; + let mut backward = forward.clone(); + backward.reverse(); + assert_eq!( + groups_result(true, forward.clone(), vec![0; 4]), + vec![Some(4.0)] + ); + assert_eq!( + groups_result(true, backward.clone(), vec![0; 4]), + vec![Some(4.0)] + ); + assert_eq!(groups_result(false, forward, vec![0; 4]), vec![Some(1.0)]); + assert_eq!(groups_result(false, backward, vec![0; 4]), vec![Some(1.0)]); + } + + #[test] + fn merged_partial_states_keep_nan_semantics() { + let mut partial_nan = ExtremumAccumulator { + is_max: true, + value: None, + }; + partial_nan.update(&(Arc::new(Float64Array::from(vec![f64::NAN])) as ArrayRef)); + let mut partial_num = ExtremumAccumulator { + is_max: true, + value: None, + }; + partial_num.update(&(Arc::new(Float64Array::from(vec![2.0, 5.0])) as ArrayRef)); + + let mut merged = ExtremumAccumulator { + is_max: true, + value: None, + }; + for partial in [&mut partial_nan, &mut partial_num] { + let state = partial.state().unwrap()[0].to_array().unwrap(); + merged.merge_batch(&[state]).unwrap(); + } + assert_eq!(merged.evaluate().unwrap(), ScalarValue::Float64(Some(5.0))); + } +} diff --git a/src/common/function/src/function_registry.rs b/src/common/function/src/function_registry.rs index 6583f7ba607..8657636fe6b 100644 --- a/src/common/function/src/function_registry.rs +++ b/src/common/function/src/function_registry.rs @@ -25,6 +25,7 @@ use crate::admin::AdminFunction; use crate::aggrs::aggr_wrapper::StateMergeHelper; use crate::aggrs::approximate::ApproximateFunction; use crate::aggrs::count_hash::CountHash; +use crate::aggrs::extremum::Extremum; use crate::aggrs::vector::VectorFunction as VectorAggrFunction; use crate::function::{Function, FunctionRef}; use crate::function_factory::ScalarFunctionFactory; @@ -260,6 +261,9 @@ pub static FUNCTION_REGISTRY: LazyLock> = LazyLock::new(|| // CountHash function CountHash::register(&function_registry); + // PromQL min/max; registered before the state functions are derived. + Extremum::register(&function_registry); + // state function of supported aggregate functions StateMergeHelper::register(&function_registry); diff --git a/src/frontend/src/instance.rs b/src/frontend/src/instance.rs index b91290d2a55..df30106c54a 100644 --- a/src/frontend/src/instance.rs +++ b/src/frontend/src/instance.rs @@ -3477,7 +3477,7 @@ mod tests { &instance, ctx.clone(), servers::http::jaeger::QueryTraceParams { - service_name: "service".to_string(), + service_name: Some("service".to_string()), ..Default::default() }, ) diff --git a/src/frontend/src/instance/jaeger.rs b/src/frontend/src/instance/jaeger.rs index c9b5bd9e988..ac59698678b 100644 --- a/src/frontend/src/instance/jaeger.rs +++ b/src/frontend/src/instance/jaeger.rs @@ -202,8 +202,9 @@ impl JaegerQueryHandler for Instance { let mut filters = vec![]; - // `service_name` is already validated in `from_jaeger_query_params()`, so no additional check needed here. - filters.push(col(SERVICE_NAME_COLUMN).eq(lit(query_params.service_name))); + if let Some(service_name) = query_params.service_name { + filters.push(col(SERVICE_NAME_COLUMN).eq(lit(service_name))); + } if let Some(operation_name) = query_params.operation_name { filters.push(col(SPAN_NAME_COLUMN).eq(lit(operation_name))); @@ -254,6 +255,7 @@ impl JaegerQueryHandler for Instance { .await?; // Get all traces that match the trace ids from the previous query. + // API v3 applies the time window only to the search, not to the returned spans. // It's equivalent to the following SQL query: // // ``` @@ -276,11 +278,15 @@ impl JaegerQueryHandler for Instance { ), ]; - if let Some(start_time) = query_params.start_time { + if !query_params.fetch_full_trace + && let Some(start_time) = query_params.start_time + { filters.push(col(TIMESTAMP_COLUMN).gt_eq(lit_timestamp_nano(start_time))); } - if let Some(end_time) = query_params.end_time { + if !query_params.fetch_full_trace + && let Some(end_time) = query_params.end_time + { filters.push(col(TIMESTAMP_COLUMN).lt_eq(lit_timestamp_nano(end_time))); } @@ -540,64 +546,37 @@ fn json_tag_filters( dataframe: &DataFrame, tags: HashMap, ) -> ServerResult> { + let has_resource_attributes = dataframe + .schema() + .has_column_with_unqualified_name(RESOURCE_ATTRIBUTES_COLUMN); let mut filters = vec![]; // NOTE: The key of the tags may contain `.`, for example: `http.status_code`, so we need to use `["http.status_code"]` in json path to access the value. for (key, value) in tags.iter() { - if let JsonValue::String(value) = value { - filters.push( - dataframe - .registry() - .udf(JsonGetString::NAME) - .context(DataFusionSnafu)? - .call(vec![ - col(SPAN_ATTRIBUTES_COLUMN), - lit(format!("[\"{}\"]", key)), - ]) - .eq(lit(value)), - ); - } - if let JsonValue::Number(value) = value { - if value.is_i64() { - filters.push( - dataframe - .registry() - .udf(JsonGetInt::NAME) - .context(DataFusionSnafu)? - .call(vec![ - col(SPAN_ATTRIBUTES_COLUMN), - lit(format!("[\"{}\"]", key)), - ]) - .eq(lit(value.as_i64().unwrap())), - ); + let (udf, value) = match value { + JsonValue::String(value) => (JsonGetString::NAME, lit(value)), + JsonValue::Number(value) if value.is_i64() => { + (JsonGetInt::NAME, lit(value.as_i64().unwrap())) } - if value.is_f64() { - filters.push( - dataframe - .registry() - .udf(JsonGetFloat::NAME) - .context(DataFusionSnafu)? - .call(vec![ - col(SPAN_ATTRIBUTES_COLUMN), - lit(format!("[\"{}\"]", key)), - ]) - .eq(lit(value.as_f64().unwrap())), - ); + JsonValue::Number(value) if value.is_f64() => { + (JsonGetFloat::NAME, lit(value.as_f64().unwrap())) } - } - if let JsonValue::Bool(value) = value { - filters.push( - dataframe - .registry() - .udf(JsonGetBool::NAME) - .context(DataFusionSnafu)? - .call(vec![ - col(SPAN_ATTRIBUTES_COLUMN), - lit(format!("[\"{}\"]", key)), - ]) - .eq(lit(*value)), - ); - } + JsonValue::Bool(value) => (JsonGetBool::NAME, lit(*value)), + _ => continue, + }; + let get = dataframe.registry().udf(udf).context(DataFusionSnafu)?; + let path = lit(format!("[\"{}\"]", key)); + let span_value = get.call(vec![col(SPAN_ATTRIBUTES_COLUMN), path.clone()]); + // Same precedence as `json2_tag_filters`: a span attribute shadows a resource attribute. + let attribute = if has_resource_attributes { + coalesce(vec![ + span_value, + get.call(vec![col(RESOURCE_ATTRIBUTES_COLUMN), path]), + ]) + } else { + span_value + }; + filters.push(attribute.eq(value)); } Ok(filters) diff --git a/src/mito2/Cargo.toml b/src/mito2/Cargo.toml index 39dd96f687b..02c134e964c 100644 --- a/src/mito2/Cargo.toml +++ b/src/mito2/Cargo.toml @@ -19,6 +19,7 @@ workspace = true api.workspace = true aquamarine.workspace = true arrow-schema.workspace = true +arrow-select.workspace = true async-channel = "1.9" async-stream.workspace = true async-trait.workspace = true diff --git a/src/mito2/src/engine/scan_test.rs b/src/mito2/src/engine/scan_test.rs index f86b327735d..f55f084ebd7 100644 --- a/src/mito2/src/engine/scan_test.rs +++ b/src/mito2/src/engine/scan_test.rs @@ -4498,6 +4498,143 @@ async fn test_range_cache_key_separates_sequence_ranges() { ); } +#[tokio::test] +async fn test_single_source_ranges_preserve_batches() { + for (file_count, row_group_size) in [(1, 64), (1, 16), (2, 16)] { + let mut env = TestEnv::new().await; + let engine = env + .create_engine(MitoConfig { + default_flat_format: true, + ..Default::default() + }) + .await; + let region_id = RegionId::new(1, 1); + let request = CreateRequestBuilder::new() + .insert_option("memtable.type", "bulk") + .insert_option("max_row_group_row_count", &row_group_size.to_string()) + .build(); + let schema = test_util::rows_schema(&request); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + let rows_per_file = 64 / file_count; + for file in 0..file_count { + test_util::put_rows( + &engine, + region_id, + Rows { + schema: schema.clone(), + rows: test_util::build_rows(file * rows_per_file, (file + 1) * rows_per_file), + }, + ) + .await; + test_util::flush_region(&engine, region_id, None).await; + } + let Scanner::Seq(scanner) = engine + .scanner(region_id, ScanRequest::default()) + .await + .unwrap() + else { + panic!("expected a sequential scan"); + }; + assert_eq!(file_count, scanner.input().files.len()); + assert_eq!( + 64 / row_group_size, + scanner.properties().partitions[0].len() + ); + let stream = scanner + .scan_partition(&Default::default(), &ExecutionPlanMetricsSet::default(), 0) + .unwrap(); + let batches = stream.try_collect::>().await.unwrap(); + assert_eq!( + vec![row_group_size; 64 / row_group_size], + batches + .iter() + .map(|batch| batch.num_rows()) + .collect::>() + ); + let mut values = scan_field_values(&batches); + values.sort_by(f64::total_cmp); + assert_eq!((0..64).map(|i| i as f64).collect::>(), values); + } +} + +#[tokio::test] +async fn test_short_series_level_zero_merge_preserves_batches_and_latest_values() { + let mut env = TestEnv::new().await; + let engine = env + .create_engine(MitoConfig { + default_flat_format: true, + ..Default::default() + }) + .await; + let region_id = RegionId::new(1, 1); + let request = CreateRequestBuilder::new() + .insert_option("memtable.type", "bulk") + .insert_option("max_row_group_row_count", "65536") + .build(); + let schema = test_util::rows_schema(&request); + engine + .handle_request(region_id, RegionRequest::Create(request)) + .await + .unwrap(); + for (start, end, offset) in [(0, 12_000, 0), (6_000, 18_000, 100_000)] { + let rows = (start..end) + .map(|i| { + row(vec![ + ValueData::StringValue(format!("{i:05}")), + ValueData::F64Value((i + offset) as f64), + ValueData::TimestampMillisecondValue(0), + ]) + }) + .collect(); + test_util::put_rows( + &engine, + region_id, + Rows { + schema: schema.clone(), + rows, + }, + ) + .await; + test_util::flush_region(&engine, region_id, None).await; + } + let Scanner::Seq(scanner) = engine + .scanner(region_id, ScanRequest::default()) + .await + .unwrap() + else { + panic!("expected a sequential scan"); + }; + assert_eq!(2, scanner.input().files.len()); + for file in &scanner.input().files { + assert_eq!(0, file.meta_ref().level); + assert_eq!(12_000, file.meta_ref().num_series); + } + let batches = scanner + .scan_partition(&Default::default(), &ExecutionPlanMetricsSet::default(), 0) + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert!( + batches.len() < 64, + "short series became {} batches", + batches.len() + ); + assert_eq!( + (0..18_000) + .map(|i| if i < 6_000 { + i as f64 + } else { + (i + 100_000) as f64 + }) + .collect::>(), + scan_field_values(&batches), + ); +} + #[tokio::test] async fn test_reset_state_discards_dynamic_field_pruning() { for (append_mode, expected_scanner_name) in [(false, "SeqScan"), (true, "UnorderedScan")] { diff --git a/src/mito2/src/read/flat_projection.rs b/src/mito2/src/read/flat_projection.rs index 2abf6aa48af..1734ca364af 100644 --- a/src/mito2/src/read/flat_projection.rs +++ b/src/mito2/src/read/flat_projection.rs @@ -22,7 +22,7 @@ use common_recordbatch::error::{ ArrowComputeSnafu, DataTypesSnafu, ExternalSnafu, NewDfRecordBatchSnafu, }; use common_recordbatch::{DfRecordBatch, RecordBatch}; -use datatypes::arrow::array::Array; +use datatypes::arrow::array::{Array, AsArray}; use datatypes::arrow::datatypes::{DataType as ArrowDataType, Field}; use datatypes::extension::json::is_json2_extension_type; use datatypes::prelude::{ConcreteDataType, DataType}; @@ -320,6 +320,16 @@ impl FlatProjectionMapper { let mut arrays = Vec::with_capacity(self.output_schema.num_columns()); for (output_idx, index) in self.batch_indices.iter().enumerate() { let mut array = batch.column(*index).clone(); + if matches!( + self.output_schema.arrow_schema().fields()[output_idx].data_type(), + ArrowDataType::Dictionary(_, _) + ) && let Some(dictionary) = array.as_any_dictionary_opt() + && dictionary.values().len() > dictionary.len() + { + // Small scan slices must not retain an entire SST's tag dictionary in merges. + array = arrow_select::dictionary::garbage_collect_any_dictionary(dictionary) + .context(ArrowComputeSnafu)?; + } // Cast dictionary values to the target type. if let ArrowDataType::Dictionary(_key_type, value_type) = array.data_type() && !matches!( @@ -667,6 +677,63 @@ mod tests { ); } + #[test] + fn test_tag_projection_compacts_dictionary_slices() { + use datatypes::arrow::array::{ + DictionaryArray, StringArray, TimestampMillisecondArray, UInt32Array, + }; + use datatypes::arrow::datatypes::UInt32Type; + + let mut builder = RegionMetadataBuilder::new(RegionId::new(1024, 0)); + builder + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new("tag", ConcreteDataType::string_datatype(), true), + semantic_type: SemanticType::Tag, + column_id: 0, + }) + .push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ), + semantic_type: SemanticType::Timestamp, + column_id: 1, + }) + .primary_key(vec![0]); + let metadata = Arc::new(builder.build().unwrap()); + let mapper = FlatProjectionMapper::new(&metadata, [0, 1]) + .unwrap() + .with_pk_dictionary_encoding(); + let values = StringArray::from_iter( + (0..1024).map(|i| (i != 1).then(|| format!("{i}{}", "x".repeat(4096)))), + ); + let dictionary = DictionaryArray::::try_new( + UInt32Array::from(vec![Some(1023), None, Some(1), Some(1023)]), + Arc::new(values), + ) + .unwrap(); + let expected = datatypes::arrow::compute::cast(&dictionary, &ArrowDataType::Utf8).unwrap(); + let batch = DfRecordBatch::try_from_iter([ + ("tag", Arc::new(dictionary) as Arc), + ( + "ts", + Arc::new(TimestampMillisecondArray::from(vec![1, 2, 3, 4])), + ), + ]) + .unwrap(); + let output = mapper.convert(&batch, &CacheStrategy::Disabled).unwrap(); + let actual = output.column(0); + assert!(actual.get_array_memory_size() < 64 * 1024); + assert_eq!(actual.data_type(), batch.column(0).data_type()); + assert_eq!( + datatypes::arrow::compute::cast(actual.as_ref(), &ArrowDataType::Utf8) + .unwrap() + .as_ref(), + expected.as_ref(), + ); + } + #[test] fn test_single_value_dictionary_preserves_string_type() { use datatypes::arrow::array::{DictionaryArray, TimestampMillisecondArray, UInt32Array}; diff --git a/src/mito2/src/read/scan_util.rs b/src/mito2/src/read/scan_util.rs index c5eed8853ae..7224143f2c4 100644 --- a/src/mito2/src/read/scan_util.rs +++ b/src/mito2/src/read/scan_util.rs @@ -1234,7 +1234,15 @@ pub(crate) fn should_split_flat_batches_for_merge( let file = &stream_ctx.input.files[file_index]; let file_meta = file.meta_ref(); if file_meta.level == 0 { - // Always split level 0 files. + // Short series can leave too many tiny batches retained by the merge. + if file_meta.num_series > 0 { + if !can_split_series(file_meta.num_rows, file_meta.num_series) { + return None; + } + total_rows += file_meta.num_rows; + total_series += file_meta.num_series; + } + // Older L0 files may lack series statistics; keep their fallback behavior. num_files_to_split += 1; continue; } else if file_meta.num_rows < SPLIT_ROW_THRESHOLD || file_meta.num_series == 0 { @@ -1913,6 +1921,30 @@ mod tests { ); } + #[tokio::test] + async fn test_should_split_flat_batches_for_merge_uses_level_zero_series_stats() { + for (num_rows, num_series, expected) in [ + (70_144, 35_083, None), + (100_000, 1_000, Some(100)), + (100_000, 0, Some(DEFAULT_READ_BATCH_SIZE / 4)), + ] { + let mut meta = new_test_file(num_rows, num_series).meta_ref().clone(); + meta.level = 0; + let file = FileHandle::new(meta, Arc::new(NoopFilePurger)); + let stream_ctx = new_test_stream_ctx(vec![file], vec![]).await; + let range_meta = new_test_range_meta(smallvec![RowGroupIndex { + index: 0, + row_group_index: 0, + }]); + + assert_eq!( + expected, + should_split_flat_batches_for_merge(&stream_ctx, &range_meta), + "rows={num_rows}, series={num_series}", + ); + } + } + #[tokio::test] async fn test_should_split_flat_batches_for_merge_falls_back_to_memtables() { let stream_ctx = new_test_stream_ctx(vec![], vec![new_test_memtable(5_000, 100)]).await; diff --git a/src/mito2/src/read/seq_scan.rs b/src/mito2/src/read/seq_scan.rs index 323704c31e2..bbf6ff1bf92 100644 --- a/src/mito2/src/read/seq_scan.rs +++ b/src/mito2/src/read/seq_scan.rs @@ -662,8 +662,6 @@ pub(crate) async fn build_flat_sources( return Ok(None); } - let split_batch_size = should_split_flat_batches_for_merge(stream_ctx, range_meta); - let should_split = split_batch_size.is_some(); sources.reserve(num_indices); let mut ordered_sources = Vec::with_capacity(num_indices); ordered_sources.resize_with(num_indices, || None); @@ -737,6 +735,16 @@ pub(crate) async fn build_flat_sources( } } + // Queries merge one range at a time; compaction merges sources from all ranges. + let needs_merge = + ordered_sources.iter().flatten().count() > 1 || (compaction && stream_ctx.ranges.len() > 1); + let split_batch_size = if needs_merge { + should_split_flat_batches_for_merge(stream_ctx, range_meta) + } else { + None + }; + let should_split = split_batch_size.is_some(); + for stream in ordered_sources.into_iter().flatten() { if should_split { sources.push(Box::pin(SplitRecordBatchStream::new(stream))); diff --git a/src/object-store/Cargo.toml b/src/object-store/Cargo.toml index 15e2069e3bd..96785fbf243 100644 --- a/src/object-store/Cargo.toml +++ b/src/object-store/Cargo.toml @@ -24,7 +24,7 @@ common-telemetry.workspace = true derive_builder = { workspace = true, optional = true } hdfs-native = { workspace = true, optional = true } humantime-serde.workspace = true -opendal = { version = "0.59.2", features = [ +opendal = { version = "0.59.4", features = [ "layers-tracing", "layers-prometheus", "services-azblob", @@ -34,7 +34,7 @@ opendal = { version = "0.59.2", features = [ "services-oss", "services-s3", ] } -opendal-http-transport-reqwest = "0.59.2" +opendal-http-transport-reqwest = "0.59.4" prometheus.workspace = true reqwest.workspace = true serde.workspace = true diff --git a/src/object-store/src/layers/mock.rs b/src/object-store/src/layers/mock.rs index 270f776f935..4807a401c95 100644 --- a/src/object-store/src/layers/mock.rs +++ b/src/object-store/src/layers/mock.rs @@ -13,7 +13,7 @@ // limitations under the License. use std::fmt::Debug; -use std::sync::Arc; +use std::sync::{Arc, Mutex}; use derive_builder::Builder; pub use oio::*; @@ -225,6 +225,50 @@ impl oio::Read for MockReader { } } +/// Writer calls observed before forwarding, including calls that fail. +#[derive(Debug, Default)] +pub struct WriterObservations { + pub write_sizes: Vec, + pub close_calls: usize, + pub abort_calls: usize, +} + +/// Records calls without changing the inner writer's completion or abort semantics. +pub struct ObservedWriter { + inner: oio::Writer, + observations: Arc>, +} + +impl ObservedWriter { + pub fn new(inner: oio::Writer, observations: Arc>) -> Self { + Self { + inner, + observations, + } + } +} + +impl oio::Write for ObservedWriter { + async fn write(&mut self, bytes: Buffer) -> Result<()> { + self.observations + .lock() + .unwrap() + .write_sizes + .push(bytes.len()); + self.inner.write(bytes).await + } + + async fn close(&mut self) -> Result { + self.observations.lock().unwrap().close_calls += 1; + self.inner.close().await + } + + async fn abort(&mut self) -> Result<()> { + self.observations.lock().unwrap().abort_calls += 1; + self.inner.abort().await + } +} + pub struct MockWriter { inner: oio::Writer, } diff --git a/src/operator/src/statement/export_logical_tables.rs b/src/operator/src/statement/export_logical_tables.rs index 368335f8450..8789e633995 100644 --- a/src/operator/src/statement/export_logical_tables.rs +++ b/src/operator/src/statement/export_logical_tables.rs @@ -41,7 +41,7 @@ use common_recordbatch::SendableRecordBatchStream; use common_time::range::TimestampRange; use datafusion::datasource::DefaultTableSource; use datafusion_common::TableReference as DfTableReference; -use datafusion_expr::{LogicalPlan, LogicalPlanBuilder, col}; +use datafusion_expr::{LogicalPlan, LogicalPlanBuilder, col, lit}; use futures::StreamExt; use object_store::ObjectStore; use session::context::QueryContextRef; @@ -53,6 +53,7 @@ use store_api::metric_engine_consts::{ use table::TableRef; use table::metadata::TableId; use table::table::adapter::DfTableProviderAdapter; +use tokio::sync::OwnedSemaphorePermit; use tokio_util::sync::CancellationToken; use crate::error::{self, InvalidLogicalTableExportSnafu, LogicalTableExportResourceSnafu, Result}; @@ -79,7 +80,7 @@ impl Default for LogicalTableExportLimits { input_batch_bytes: 64 * 1024 * 1024, conversion_bytes: 1024 * 1024, writer: ParquetWriterLimits { - row_group_rows: 8192, + row_group_rows: 1024 * 1024, flush_threshold_bytes: 8 * 1024 * 1024, max_row_groups: 4096, }, @@ -255,7 +256,7 @@ impl LogicalTableExport { fn build_plan(&self, time_range: Option<&TimestampRange>) -> Result { let info = self.physical_table.table_info(); - let filters = self + let mut filters = self .physical_table .schema() .timestamp_column() @@ -264,6 +265,11 @@ impl LogicalTableExport { }) .into_iter() .collect::>(); + if self.logical_tables.len() == 1 + && let Some((&id, _)) = self.logical_tables.first_key_value() + { + filters.push(col(TABLE_ID).eq(lit(id))); + } let source = Arc::new(DefaultTableSource::new(Arc::new( DfTableProviderAdapter::new(self.physical_table.clone()), ))); @@ -283,11 +289,13 @@ impl LogicalTableExport { .filter(filter) .context(error::BuildDfLogicalPlanSnafu)?; } - builder - .sort(vec![col(TABLE_ID).sort(true, false)]) - .context(error::BuildDfLogicalPlanSnafu)? - .build() - .context(error::BuildDfLogicalPlanSnafu) + // A single selected table has a constant routing key after filtering. + if self.logical_tables.len() > 1 { + builder = builder + .sort(vec![col(TABLE_ID).sort(true, false)]) + .context(error::BuildDfLogicalPlanSnafu)?; + } + builder.build().context(error::BuildDfLogicalPlanSnafu) } } @@ -613,6 +621,46 @@ async fn expand_bounded_slice( batch.num_columns(), requested, )?; + // Parquet accepts compatible string dictionaries without expanding their values. + let fields = schema + .fields() + .iter() + .zip(batch.columns()) + .map(|(field, array)| { + if let DataType::Dictionary(_, value) = array.data_type() + && value.as_ref() == field.data_type() + && matches!(value.as_ref(), DataType::Utf8 | DataType::LargeUtf8) + { + Arc::new( + field + .as_ref() + .clone() + .with_data_type(array.data_type().clone()), + ) + } else { + field.clone() + } + }) + .collect::>(); + let schema = Arc::new(arrow::datatypes::Schema::new_with_metadata( + fields, + schema.metadata().clone(), + )); + // Sharing compatible arrays needs no CPU conversion. Keep the size check on + // this task only when its dictionary inspection is bounded. + if start < end + && batch + .columns() + .iter() + .zip(schema.fields()) + .all(|(array, field)| array.data_type() == field.data_type()) + && let Some(estimated) = estimate_batch_bytes_upper_bound(&batch, end - start, &[])? + && estimated <= conversion + { + let reservation = retained.saturating_add(estimated.saturating_mul(4)); + let permit = budget.reserve(reservation, cancellation).await?; + return convert_slice_payload(batch, schema, start, end - start, reservation, permit); + } let input = batch.clone(); let (len, estimated) = common_runtime::spawn_blocking_global(move || { rows_within_budget(&input, start, end, conversion, &[]) @@ -622,26 +670,36 @@ async fn expand_bounded_slice( let reservation = retained.saturating_add(estimated.saturating_mul(4)); let permit = budget.reserve(reservation, cancellation).await?; common_runtime::spawn_blocking_global(move || { - let slice = batch.slice(start, len); - let expanded = expand_export_batch(&slice, schema)?; - ensure!( - expanded.get_array_memory_size() <= reservation, - LogicalTableExportResourceSnafu { - reason: "converted backing buffers exceed reservation" - } - ); - Ok(( - Payload { - batch: expanded, - permit, - }, - len, - )) + convert_slice_payload(batch, schema, start, len, reservation, permit) }) .await .context(error::JoinTaskSnafu)? } +fn convert_slice_payload( + batch: RecordBatch, + schema: SchemaRef, + start: usize, + len: usize, + reservation: usize, + permit: OwnedSemaphorePermit, +) -> Result<(Payload, usize)> { + let expanded = expand_export_batch(&batch.slice(start, len), schema)?; + ensure!( + expanded.get_array_memory_size() <= reservation, + LogicalTableExportResourceSnafu { + reason: "converted backing buffers exceed reservation" + } + ); + Ok(( + Payload { + batch: expanded, + permit, + }, + len, + )) +} + /// Limits nested casts to selected children: Arrow's List cast otherwise expands /// the entire values array, even when the parent has been sliced. pub(crate) fn expand_export_batch(batch: &RecordBatch, schema: SchemaRef) -> Result { @@ -788,6 +846,41 @@ fn estimate_value_size(array: &dyn Array, row: usize) -> Result { Ok(bytes.saturating_add(16)) } +/// Estimates a conservative byte upper bound with bounded work on an async task. +/// `None` means this path cannot estimate the batch, not that it exceeds a budget. +fn estimate_batch_bytes_upper_bound( + batch: &RecordBatch, + rows: usize, + json_columns: &[usize], +) -> Result> { + let mut dictionary_values = 1024usize; + let mut max_row_bytes = 0usize; + for (index, array) in batch.columns().iter().enumerate() { + let size = if let Some(width) = array.data_type().primitive_width() { + (width + 16).max(32) + } else if let Some(dictionary) = array.as_any_dictionary_opt() + && dictionary.values().len() < rows + && dictionary.values().len() <= dictionary_values + && matches!( + dictionary.values().data_type(), + DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View + ) + { + let values = dictionary.values(); + dictionary_values -= values.len(); + let max = (0..values.len()).try_fold(0usize, |max, row| { + Ok::<_, error::Error>(max.max(estimate_value_size(values.as_ref(), row)?)) + })?; + max.saturating_add(16).max(32) + } else { + return Ok(None); + }; + let expansion = if json_columns.contains(&index) { 8 } else { 1 }; + max_row_bytes = max_row_bytes.saturating_add(size.saturating_mul(expansion)); + } + Ok(max_row_bytes.checked_mul(rows)) +} + pub(crate) fn rows_within_budget( batch: &RecordBatch, start: usize, @@ -795,6 +888,12 @@ pub(crate) fn rows_within_budget( budget: usize, json_columns: &[usize], ) -> Result<(usize, usize)> { + if start < end + && let Some(bytes) = estimate_batch_bytes_upper_bound(batch, end - start, json_columns)? + && bytes <= budget + { + return Ok((end - start, bytes)); + } let mut bytes = 0usize; let mut row = start; while row < end { diff --git a/src/operator/src/statement/export_logical_tables/tests.rs b/src/operator/src/statement/export_logical_tables/tests.rs index 7aa6daab127..6513d7e43fd 100644 --- a/src/operator/src/statement/export_logical_tables/tests.rs +++ b/src/operator/src/statement/export_logical_tables/tests.rs @@ -127,6 +127,188 @@ async fn read(store: &ObjectStore, path: &str) -> (SchemaRef, Vec) ) } +#[tokio::test] +async fn single_table_plan_filters_other_routing_ids_before_export() { + let mut unit = unit(); + unit.logical_tables.retain(|id, _| *id == 1025); + let schema = unit.physical_table.schema(); + let input = expand_export_batch( + &batch( + vec![Some(1027), Some(1025), Some(1024), Some(1025), Some(1026)], + vec![ + Some("other"), + Some("selected-a"), + None, + Some("selected-b"), + None, + ], + ), + schema.arrow_schema().clone(), + ) + .unwrap(); + unit.physical_table = table::test_util::MemTable::table( + "phy", + GreptimeRecordBatch::from_df_record_batch(schema, input), + ); + let context = datafusion::prelude::SessionContext::new_with_config( + datafusion::prelude::SessionConfig::new().with_target_partitions(4), + ); + let batches = context + .execute_logical_plan(unit.build_plan(None).unwrap()) + .await + .unwrap() + .collect() + .await + .unwrap(); + let store = ObjectStore::new(object_store::services::Memory::default()).unwrap(); + let summary = export_stream( + &unit, + stream(batches), + &store, + LogicalTableExportLimits::default(), + &CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(summary.rows, 2); + assert_eq!(summary.files, 1); + let (_, output) = read(&store, "cpu.v1.parquet").await; + let hosts = output + .iter() + .flat_map(|batch| batch.column(0).as_string::().iter()) + .collect::>(); + assert_eq!(hosts, vec![Some("selected-a"), Some("selected-b")]); +} + +#[tokio::test] +async fn dictionary_null_values_write_the_logical_parquet_schema() { + let input = batch(vec![Some(1025); 4], vec![Some("placeholder"); 4]); + let mut columns = input.columns().to_vec(); + columns[2] = Arc::new( + DictionaryArray::::try_new( + UInt32Array::from(vec![Some(0), Some(1), None, Some(0)]), + Arc::new(StringArray::from(vec![Some("host"), None])), + ) + .unwrap(), + ); + let input = RecordBatch::try_new(input.schema(), columns).unwrap(); + let unit = unit(); + let store = ObjectStore::new(object_store::services::Memory::default()).unwrap(); + let summary = export_stream( + &unit, + stream(vec![input]), + &store, + LogicalTableExportLimits::default(), + &CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(summary.rows, 4); + let (schema, output) = read(&store, "cpu.v1.parquet").await; + assert_eq!(schema.fields(), unit.logical_tables[&1025].schema.fields()); + let hosts = output + .iter() + .flat_map(|batch| batch.column(0).as_string::().iter()) + .collect::>(); + assert_eq!(hosts, vec![Some("host"), None, None, Some("host")]); +} + +#[tokio::test] +async fn dictionary_slices_share_backing_and_respect_budget() { + let input = batch(vec![Some(1025); 8192], vec![Some("host"); 8192]) + .project(&[2, 3, 1]) + .unwrap(); + let schema = unit().logical_tables[&1025].schema.clone(); + let budget = ExportWriteBudget::new(1); + let token = CancellationToken::new(); + let conversion = expand_bounded_slice( + input.clone(), + schema.clone(), + 7, + 8192, + 1024 * 1024, + &budget, + &token, + ); + tokio::pin!(conversion); + // Sharing an admitted batch completes without waiting for a conversion worker. + let std::task::Poll::Ready(result) = futures::poll!(&mut conversion) else { + panic!("compatible slice dispatched a conversion worker"); + }; + let (payload, rows) = result.unwrap(); + assert_eq!(rows, 8185); + let input_values = input.column(0).as_dictionary::().values(); + let output_values = payload + .batch + .column(0) + .as_dictionary::() + .values(); + assert!(Arc::ptr_eq(input_values, output_values)); + assert!(64 * 1024 * 1024 - budget.available().1 >= input.get_array_memory_size()); + drop(payload); + assert_eq!(budget.available(), (1, 64 * 1024 * 1024)); + + // A batch which does not fit still takes the bounded conversion path. + let (payload, rows) = + expand_bounded_slice(input.clone(), schema.clone(), 7, 8192, 128, &budget, &token) + .await + .unwrap(); + assert_eq!(rows, 1); + drop(payload); + assert!( + expand_bounded_slice(input, schema, 7, 8192, 1, &budget, &token) + .await + .is_err() + ); + assert_eq!(budget.available(), (1, 64 * 1024 * 1024)); +} + +#[test] +fn dictionary_size_bound_limits_inline_work() { + let values = (0..1025).map(|i| format!("host-{i}")).collect::>(); + let dictionary = DictionaryArray::::new( + UInt32Array::from(vec![0; 2048]), + Arc::new(StringArray::from_iter_values(values.iter())), + ); + let input = RecordBatch::try_from_iter([("host", Arc::new(dictionary) as ArrayRef)]).unwrap(); + assert!( + estimate_batch_bytes_upper_bound(&input, 2048, &[]) + .unwrap() + .is_none() + ); + assert_eq!( + rows_within_budget(&input, 0, 2048, 128, &[]).unwrap(), + (3, 114) + ); +} + +#[tokio::test] +async fn narrow_batches_share_row_groups() { + let store = ObjectStore::new(object_store::services::Memory::default()).unwrap(); + let batches = (0..8) + .map(|_| batch(vec![Some(1025); 8192], vec![Some("host"); 8192])) + .collect(); + let summary = export_stream( + &unit(), + stream(batches), + &store, + LogicalTableExportLimits::default(), + &CancellationToken::new(), + ) + .await + .unwrap(); + assert_eq!(summary.rows, 65536); + let bytes = store.read("cpu.v1.parquet").await.unwrap().to_bytes(); + let reader = ParquetRecordBatchReaderBuilder::try_new(bytes).unwrap(); + assert_eq!(reader.metadata().num_row_groups(), 1); + let rows: usize = reader + .build() + .unwrap() + .map(|batch| batch.unwrap().num_rows()) + .sum(); + assert_eq!(rows, summary.rows); +} + #[tokio::test] async fn routes_across_batches_and_writes_empty_files() { let unit = unit(); @@ -249,6 +431,19 @@ async fn existing_outputs_are_not_overwritten() { ); } +#[test] +fn repeated_dictionary_budget_preserves_null_and_offset_charges() { + let host = "x".repeat(64); + let input = batch(vec![Some(1025); 8], vec![Some(host.as_str()); 8]) + .project(&[2, 4, 1]) + .unwrap(); + // Each row charges 96 bytes for the dictionary value, 32 for null, and 24 for time. + assert_eq!(rows_within_budget(&input, 0, 8, 1215, &[]).unwrap().0, 7); + let (rows, estimated) = rows_within_budget(&input, 0, 8, 1280, &[]).unwrap(); + assert_eq!(rows, 8); + assert!((1216..=1280).contains(&estimated)); +} + #[test] fn dictionary_and_nested_histogram_values_are_bounded_before_expansion() { let dictionary = DictionaryArray::::try_new( @@ -291,6 +486,25 @@ fn validates_selected_schemas_and_projects_only_selected_columns() { .unwrap(); assert_eq!(one.scan_projection, vec![0, 3]); assert_eq!(one.logical_tables[&1025].projection, vec![1]); + let LogicalPlan::Filter(filter) = one.build_plan(None).unwrap() else { + panic!("single-table export must filter the routing key without sorting"); + }; + let datafusion_expr::Expr::BinaryExpr(predicate) = filter.predicate else { + panic!("expected table-id equality"); + }; + assert_eq!(predicate.op, datafusion_expr::Operator::Eq); + assert!(matches!( + predicate.left.as_ref(), + datafusion_expr::Expr::Column(column) if column.name == TABLE_ID + )); + assert!(matches!( + predicate.right.as_ref(), + datafusion_expr::Expr::Literal(datafusion_common::ScalarValue::UInt32(Some(1025)), _) + )); + assert!(matches!( + unit.build_plan(None).unwrap(), + LogicalPlan::Sort(_) + )); assert!( LogicalTableExport::try_new(unit.physical_table.clone(), &[selected.clone(), selected]) .is_err() @@ -333,7 +547,7 @@ async fn cancellation_drops_input_and_aborts_active_upload() { } #[tokio::test] -async fn native_histogram_parquet_roundtrip() { +async fn supported_metric_types_parquet_roundtrip() { fn sample(data_type: &DataType) -> ArrayRef { match data_type { DataType::Struct(fields) => Arc::new(StructArray::new( @@ -359,44 +573,102 @@ async fn native_histogram_parquet_roundtrip() { other => panic!("unexpected histogram field {other}"), } } - let data_type = common_query::native_histogram::native_histogram_arrow_type(); - let field = Field::new("histogram", data_type.clone(), true); - let unit = LogicalTableExport::try_new( - table( - 1024, - "phy", - vec![Field::new(TABLE_ID, DataType::UInt32, false), field.clone()], - true, - ), - &[table(1025, "histogram", vec![field], false)], - ) - .unwrap(); - let batch = RecordBatch::try_from_iter_with_nullable([ - ( - TABLE_ID, - Arc::new(UInt32Array::from(vec![1025; 2])) as ArrayRef, - false, - ), - ("histogram", sample(&data_type), true), - ]) - .unwrap(); - let store = ObjectStore::new(object_store::services::Memory::default()).unwrap(); - let summary = export_stream( - &unit, - stream(vec![batch.clone()]), - &store, - export_limits(), - &CancellationToken::new(), - ) - .await - .unwrap(); - assert_eq!(summary.rows, 2); - let (schema, batches) = read(&store, "histogram.parquet").await; - assert_eq!(schema.fields(), unit.logical_tables[&1025].schema.fields()); - assert_eq!( - arrow::compute::concat_batches(&schema, &batches).unwrap(), - batch.project(&[1]).unwrap() - ); + use common_datasource::file_format::parquet::packed_reader::{ + PackReadWindows, PackedParquetReader, + }; + use common_datasource::packed_snapshot::{PACK_INDEX_FILE, PackIndex}; + use common_datasource::packed_writer::PackedWriter; + use datafusion::parquet::arrow::ParquetRecordBatchStreamBuilder; + use futures::TryStreamExt; + + let timestamps: [ArrayRef; 4] = [ + Arc::new(arrow::array::TimestampSecondArray::from(vec![4, 5])), + Arc::new(arrow::array::TimestampMillisecondArray::from(vec![4, 5])), + Arc::new(arrow::array::TimestampMicrosecondArray::from(vec![4, 5])), + Arc::new(arrow::array::TimestampNanosecondArray::from(vec![4, 5])), + ]; + let values = [ + Arc::new(Float64Array::from(vec![Some(3.5), None])) as ArrayRef, + sample(&common_query::native_histogram::native_histogram_arrow_type()), + ]; + for timestamp in timestamps { + for value in &values { + let logical = RecordBatch::try_from_iter_with_nullable([ + ("ts", timestamp.clone(), false), + ( + "host", + Arc::new(StringArray::from(vec![Some(""), None])) as ArrayRef, + true, + ), + ("value", value.clone(), true), + ]) + .unwrap(); + let mut fields = vec![Field::new(TABLE_ID, DataType::UInt32, false)]; + fields.extend(logical.schema().fields().iter().map(|f| f.as_ref().clone())); + let unit = LogicalTableExport::try_new( + table(1024, "phy", fields.clone(), true), + &[table(1025, "metric", fields[1..].to_vec(), false)], + ) + .unwrap(); + let mut columns = vec![Arc::new(UInt32Array::from(vec![1025; 2])) as ArrayRef]; + columns.extend_from_slice(logical.columns()); + let input = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap(); + for packed in [false, true] { + let store = ObjectStore::new(object_store::services::Memory::default()).unwrap(); + let writer = packed.then(|| PackedWriter::new(store.clone()).unwrap()); + let summary = export_stream_managed( + &unit, + stream(vec![input.clone()]), + &store, + export_limits(), + &CancellationToken::new(), + ExportWriteBudget::new(1), + writer.clone(), + ) + .await + .unwrap(); + assert_eq!(summary.rows, 2); + let batches = if let Some(writer) = writer { + writer + .lock() + .await + .finish(&CancellationToken::new()) + .await + .unwrap(); + let index: PackIndex = serde_json::from_slice( + &store.read(PACK_INDEX_FILE).await.unwrap().to_bytes(), + ) + .unwrap(); + index.validate_membership(["metric"]).unwrap(); + let entry = &index.tables[0]; + let object = &index.objects[0]; + let reader = PackedParquetReader::new( + PackReadWindows::new(store.clone()), + object.path.clone(), + object.length, + entry.offset, + entry.length, + ) + .unwrap(); + ParquetRecordBatchStreamBuilder::new(reader) + .await + .unwrap() + .build() + .unwrap() + .try_collect::>() + .await + .unwrap() + } else { + read(&store, "metric.parquet").await.1 + }; + assert_eq!(batches[0].schema().fields(), logical.schema().fields()); + assert_eq!( + arrow::compute::concat_batches(&logical.schema(), &batches).unwrap(), + logical + ); + } + } + } } struct PausedFileWriter { @@ -748,9 +1020,12 @@ async fn retained_backing_is_reserved_before_conversion_and_until_payload_drop() drop(blocker); let (payload, rows) = convert.await.unwrap(); assert_eq!(rows, 1); - assert!( - 64 * 1024 * 1024 - budget.available().1 >= full + payload.batch.get_array_memory_size() - ); + assert!(matches!( + payload.batch.column(0).data_type(), + DataType::Dictionary(_, _) + )); + // The retained dictionary shares its input buffers, so they are charged once. + assert!(64 * 1024 * 1024 - budget.available().1 >= full); drop(payload); assert_eq!(budget.available(), (1, 64 * 1024 * 1024)); assert!(budget.reserve(64 * 1024 * 1024 + 1, &token).await.is_err()); diff --git a/src/promql/src/functions.rs b/src/promql/src/functions.rs index 34618f0a953..6703505c277 100644 --- a/src/promql/src/functions.rs +++ b/src/promql/src/functions.rs @@ -19,6 +19,7 @@ mod double_exponential_smoothing; mod edge_count; mod extrapolate_rate; mod idelta; +mod ieee_float; mod native_histogram; mod predict_linear; mod quantile; @@ -45,6 +46,7 @@ pub use deriv::Deriv; pub use double_exponential_smoothing::DoubleExponentialSmoothing; pub use extrapolate_rate::{Delta, Increase, Rate}; pub use idelta::IDelta; +pub use ieee_float::{IeeeComparison, IeeeSqrt}; pub use native_histogram::{ MixedRange, NativeHistogramAbsentOverTime, NativeHistogramAdd, NativeHistogramAggAvg, NativeHistogramAggSum, NativeHistogramAvg, NativeHistogramAvgOverTime, NativeHistogramChanges, diff --git a/src/promql/src/functions/ieee_float.rs b/src/promql/src/functions/ieee_float.rs new file mode 100644 index 00000000000..d13480d8071 --- /dev/null +++ b/src/promql/src/functions/ieee_float.rs @@ -0,0 +1,265 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! PromQL comparison operators and `sqrt()`, which follow IEEE 754. +//! +//! Arrow compares floats by total order, where NaN equals NaN and is greater than `+Inf`, and +//! DataFusion's `sqrt` rejects negative input. In PromQL every comparison involving NaN is false +//! except `!=`, and the square root of a negative number is NaN. NULL input gives NULL output. +//! +//! The functions report the column name of the expression they replace (`a < b`, `sqrt(a)`), so +//! query output columns are unchanged. + +use std::sync::Arc; + +use datafusion::arrow::array::{AsArray, BooleanArray, Float64Array}; +use datafusion::arrow::datatypes::{DataType, Float64Type}; +use datafusion::error::Result as DfResult; +use datafusion::logical_expr::{ + ColumnarValue, Operator, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility, +}; +use datafusion_expr::Expr; + +/// A PromQL comparison operator on two float operands. +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct IeeeComparison { + op: Operator, + signature: Signature, +} + +impl IeeeComparison { + /// The comparison UDF for `op`, which must be a comparison operator. + pub fn scalar_udf(op: Operator) -> ScalarUDF { + ScalarUDF::new_from_impl(Self { + op, + signature: Signature::uniform(2, vec![DataType::Float64], Volatility::Immutable), + }) + } + + /// All comparison UDFs, for registering them where plans are decoded. + pub fn all() -> Vec { + [ + Operator::Eq, + Operator::NotEq, + Operator::Lt, + Operator::LtEq, + Operator::Gt, + Operator::GtEq, + ] + .into_iter() + .map(Self::scalar_udf) + .collect() + } + + fn compare(&self, lhs: f64, rhs: f64) -> bool { + match self.op { + Operator::Eq => lhs == rhs, + Operator::NotEq => lhs != rhs, + Operator::Lt => lhs < rhs, + Operator::LtEq => lhs <= rhs, + Operator::Gt => lhs > rhs, + Operator::GtEq => lhs >= rhs, + _ => unreachable!("not a comparison operator: {}", self.op), + } + } +} + +impl ScalarUDFImpl for IeeeComparison { + fn name(&self) -> &str { + match self.op { + Operator::Eq => "prom_eq", + Operator::NotEq => "prom_ne", + Operator::Lt => "prom_lt", + Operator::LtEq => "prom_le", + Operator::Gt => "prom_gt", + Operator::GtEq => "prom_ge", + _ => unreachable!("not a comparison operator: {}", self.op), + } + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> DfResult { + Ok(DataType::Boolean) + } + + fn schema_name(&self, args: &[Expr]) -> DfResult { + Ok(format!( + "{} {} {}", + args[0].schema_name(), + self.op, + args[1].schema_name() + )) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DfResult { + let arrays = ColumnarValue::values_to_arrays(&args.args)?; + let lhs = arrays[0].as_primitive::(); + let rhs = arrays[1].as_primitive::(); + let result: BooleanArray = lhs + .iter() + .zip(rhs.iter()) + .map(|(lhs, rhs)| Some(self.compare(lhs?, rhs?))) + .collect(); + Ok(ColumnarValue::Array(Arc::new(result))) + } +} + +/// PromQL `sqrt()`. +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct IeeeSqrt { + signature: Signature, +} + +impl IeeeSqrt { + pub fn scalar_udf() -> ScalarUDF { + ScalarUDF::new_from_impl(Self { + signature: Signature::uniform(1, vec![DataType::Float64], Volatility::Immutable), + }) + } +} + +impl ScalarUDFImpl for IeeeSqrt { + fn name(&self) -> &str { + "prom_sqrt" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> DfResult { + Ok(DataType::Float64) + } + + fn schema_name(&self, args: &[Expr]) -> DfResult { + Ok(format!("sqrt({})", args[0].schema_name())) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DfResult { + let arrays = ColumnarValue::values_to_arrays(&args.args)?; + let input = arrays[0].as_primitive::(); + let result: Float64Array = input.unary(f64::sqrt); + Ok(ColumnarValue::Array(Arc::new(result))) + } +} + +#[cfg(test)] +mod tests { + use datafusion::arrow::array::{Array, ArrayRef}; + use datafusion::arrow::datatypes::Field; + use datafusion_common::config::ConfigOptions; + + use super::*; + + fn invoke(udf: &ScalarUDF, args: Vec, return_type: DataType) -> ArrayRef { + let number_rows = args[0].len(); + let arg_fields = args + .iter() + .map(|array| Arc::new(Field::new("a", array.data_type().clone(), true))) + .collect(); + let result = udf + .invoke_with_args(ScalarFunctionArgs { + args: args.into_iter().map(ColumnarValue::Array).collect(), + arg_fields, + number_rows, + return_field: Arc::new(Field::new("r", return_type, true)), + config_options: Arc::new(ConfigOptions::default()), + }) + .unwrap(); + match result { + ColumnarValue::Array(array) => array, + ColumnarValue::Scalar(_) => unreachable!(), + } + } + + #[test] + fn comparisons_follow_ieee_and_propagate_null() { + let lhs: ArrayRef = Arc::new(Float64Array::from(vec![ + Some(1.0), + Some(f64::NAN), + Some(f64::NAN), + None, + Some(f64::INFINITY), + ])); + let rhs: ArrayRef = Arc::new(Float64Array::from(vec![ + Some(1.0), + Some(f64::NAN), + Some(1.0), + Some(f64::NAN), + Some(f64::NAN), + ])); + let expected = [ + ( + Operator::Eq, + [Some(true), Some(false), Some(false), None, Some(false)], + ), + ( + Operator::NotEq, + [Some(false), Some(true), Some(true), None, Some(true)], + ), + ( + Operator::Lt, + [Some(false), Some(false), Some(false), None, Some(false)], + ), + ( + Operator::LtEq, + [Some(true), Some(false), Some(false), None, Some(false)], + ), + ( + Operator::Gt, + [Some(false), Some(false), Some(false), None, Some(false)], + ), + ( + Operator::GtEq, + [Some(true), Some(false), Some(false), None, Some(false)], + ), + ]; + for (op, expected) in expected { + let result = invoke( + &IeeeComparison::scalar_udf(op), + vec![lhs.clone(), rhs.clone()], + DataType::Boolean, + ); + let result: Vec<_> = result.as_boolean().iter().collect(); + assert_eq!(result, expected, "{op}"); + } + } + + #[test] + fn sqrt_of_negative_is_nan() { + let input: ArrayRef = Arc::new(Float64Array::from(vec![Some(4.0), Some(-1.0), None])); + let result = invoke(&IeeeSqrt::scalar_udf(), vec![input], DataType::Float64); + let result = result.as_primitive::(); + assert_eq!(result.value(0), 2.0); + assert!(result.value(1).is_nan()); + assert!(result.is_null(2)); + } + + #[test] + fn column_names_match_the_replaced_expressions() { + let a = datafusion_expr::col("a"); + let b = datafusion_expr::lit(1.5_f64); + let comparison = + IeeeComparison::scalar_udf(Operator::LtEq).call(vec![a.clone(), b.clone()]); + assert_eq!( + comparison.schema_name().to_string(), + a.clone().lt_eq(b).schema_name().to_string() + ); + let sqrt = IeeeSqrt::scalar_udf().call(vec![a.clone()]); + assert_eq!(sqrt.schema_name().to_string(), "sqrt(a)"); + } +} diff --git a/src/promql/src/functions/round.rs b/src/promql/src/functions/round.rs index 72736ef6e2e..c5ba4c218c0 100644 --- a/src/promql/src/functions/round.rs +++ b/src/promql/src/functions/round.rs @@ -66,13 +66,7 @@ impl Round { let null_array = Float64Array::new_null(value_array.len()); return Ok(ColumnarValue::Array(Arc::new(null_array))); }; - let op = |a: f64| { - if nearest == 0.0 { - a.round() - } else { - (a / nearest).round() * nearest - } - }; + let op = |a: f64| round_to_nearest(a, nearest); let result: PrimitiveArray = value_array.as_primitive::().unary(op); Ok(ColumnarValue::Array(Arc::new(result) as _)) @@ -90,14 +84,10 @@ impl Round { )?; let result: PrimitiveArray = - datatypes::arrow::compute::binary(value_array, nearest_array, |a, nearest| { - if nearest == 0.0 { - a.round() - } else { - (a / nearest).round() * nearest - } - }) - .map_err(|err: ArrowError| DataFusionError::ArrowError(Box::new(err), None))?; + datatypes::arrow::compute::binary(value_array, nearest_array, round_to_nearest) + .map_err(|err: ArrowError| { + DataFusionError::ArrowError(Box::new(err), None) + })?; Ok(ColumnarValue::Array(Arc::new(result) as _)) } @@ -105,6 +95,13 @@ impl Round { } } +/// Rounds like Prometheus: ties round up (towards +Inf), and multiplying by the inverse +/// of `nearest` follows its float behavior, including NaN for `nearest == 0`. +fn round_to_nearest(value: f64, nearest: f64) -> f64 { + let inverse = 1.0 / nearest; + (value * inverse + 0.5).floor() / inverse +} + #[cfg(test)] mod tests { use datafusion_common::config::ConfigOptions; @@ -143,14 +140,43 @@ mod tests { #[test] fn test_round() { + // Expected values are Prometheus 3.15 `round()` results. test_round_f64(vec![123.456], 0.001, vec![123.456]); - test_round_f64(vec![123.456], 0.01, vec![123.46000000000001]); + test_round_f64(vec![123.456], 0.01, vec![123.46]); test_round_f64(vec![123.456], 0.1, vec![123.5]); - test_round_f64(vec![123.456], 0.0, vec![123.0]); test_round_f64(vec![123.456], 1.0, vec![123.0]); test_round_f64(vec![123.456], 10.0, vec![120.0]); test_round_f64(vec![123.456], 100.0, vec![100.0]); - test_round_f64(vec![123.456], 105.0, vec![105.0]); + test_round_f64(vec![123.456], 105.0, vec![104.99999999999999]); test_round_f64(vec![123.456], 1000.0, vec![0.0]); } + + #[test] + fn test_round_ties_towards_positive_infinity() { + test_round_f64(vec![-0.5], 1.0, vec![0.0]); + test_round_f64(vec![-2.5], 1.0, vec![-2.0]); + test_round_f64(vec![2.5], 1.0, vec![3.0]); + } + + #[test] + fn test_round_to_zero_is_nan() { + let round_udf = Round::scalar_udf(); + let input = vec![ + ColumnarValue::Array(Arc::new(Float64Array::from(vec![123.456]))), + ColumnarValue::Scalar(ScalarValue::Float64(Some(0.0))), + ]; + let arg_fields = vec![ + Arc::new(Field::new("a", input[0].data_type(), false)), + Arc::new(Field::new("b", input[1].data_type(), false)), + ]; + let args = ScalarFunctionArgs { + args: input, + arg_fields, + number_rows: 1, + return_field: Arc::new(Field::new("c", DataType::Float64, false)), + config_options: Arc::new(ConfigOptions::default()), + }; + let result = extract_array(&round_udf.invoke_with_args(args).unwrap()).unwrap(); + assert!(result.as_primitive::().value(0).is_nan()); + } } diff --git a/src/query/src/optimizer/constant_term.rs b/src/query/src/optimizer/constant_term.rs index 70ddcc2f647..6b77a8aa7f6 100644 --- a/src/query/src/optimizer/constant_term.rs +++ b/src/query/src/optimizer/constant_term.rs @@ -201,10 +201,14 @@ impl PhysicalOptimizerRule for MatchesConstantTermOptimizer { })?; if new_pred.transformed { - let exec = FilterExecBuilder::new(new_pred.data, filter.input().clone()) - .with_default_selectivity(filter.default_selectivity()) - .apply_projection_by_ref(filter.projection().as_ref()) - .and_then(|x| x.build())?; + // `FilterExecBuilder::from(&FilterExec)` already carries over the + // input, projection, default selectivity, batch size and fetch, so + // only the predicate has to be replaced. Re-applying the projection + // is not only pointless but wrong: `apply_projection` composes with + // the projection copied from the builder. + let exec = FilterExecBuilder::from(filter) + .with_predicate(new_pred.data) + .build()?; return Ok(Transformed::yes(Arc::new(exec) as _)); } } diff --git a/src/query/src/optimizer/count_nest_aggr.rs b/src/query/src/optimizer/count_nest_aggr.rs index b355dd399b7..70db902d9db 100644 --- a/src/query/src/optimizer/count_nest_aggr.rs +++ b/src/query/src/optimizer/count_nest_aggr.rs @@ -15,6 +15,8 @@ use std::collections::HashSet; use std::sync::Arc; +use common_function::aggrs::aggr_wrapper::get_aggr_func; +use common_function::aggrs::extremum::Extremum; use datafusion::config::ConfigOptions; use datafusion::functions_aggregate::count::count_udaf; use datafusion::logical_expr::{Extension, LogicalPlan, LogicalPlanBuilder, Sort}; @@ -243,9 +245,8 @@ impl CountNestAggrRule { where F: FnOnce(&str) -> bool, { - let Expr::AggregateFunction(func) = expr else { - return None; - }; + // The PromQL planner aliases `prom_min`/`prom_max` to the builtin names. + let func = get_aggr_func(expr)?; let name = func.func.name(); if !accept_name(name) || func.params.filter.is_some() @@ -260,9 +261,19 @@ impl CountNestAggrRule { } fn is_supported_inner_aggregate(name: &str) -> bool { + // PromQL `min`/`max` plan to `prom_min`/`prom_max`, which return NULL for the same + // groups as the builtin ones. matches!( name, - "count" | "sum" | "avg" | "min" | "max" | "stddev_pop" | "var_pop" + "count" + | "sum" + | "avg" + | "min" + | "max" + | Extremum::MIN_NAME + | Extremum::MAX_NAME + | "stddev_pop" + | "var_pop" ) } diff --git a/src/query/src/promql/planner.rs b/src/query/src/promql/planner.rs index 7cc3f88c245..078fe42562d 100644 --- a/src/query/src/promql/planner.rs +++ b/src/query/src/promql/planner.rs @@ -27,6 +27,7 @@ use async_recursion::async_recursion; use catalog::table_source::DfTableSourceProvider; use common_error::ext::ErrorExt; use common_error::status_code::StatusCode; +use common_function::aggrs::extremum::Extremum; use common_function::function::FunctionContext; use common_query::native_histogram::native_histogram_value_type; use common_query::prelude::{ @@ -70,18 +71,18 @@ use promql::extension_plan::{ }; use promql::functions::{ AbsentOverTime, AvgOverTime, Changes, CountOverTime, Delta, Deriv, DoubleExponentialSmoothing, - IDelta, Increase, LastOverTime, MatchGroupViolation, MaxOverTime, MinOverTime, MixedRange, - NativeHistogramAbsentOverTime, NativeHistogramAdd, NativeHistogramAggAvg, - NativeHistogramAggSum, NativeHistogramAvg, NativeHistogramAvgOverTime, NativeHistogramChanges, - NativeHistogramCount, NativeHistogramCountOverTime, NativeHistogramDelta, - NativeHistogramDivScalar, NativeHistogramDrop, NativeHistogramEq, NativeHistogramIDelta, - NativeHistogramIRate, NativeHistogramIncrease, NativeHistogramLastOverTime, - NativeHistogramMulScalar, NativeHistogramNeg, NativeHistogramNotEq, - NativeHistogramPresentOverTime, NativeHistogramRate, NativeHistogramResets, - NativeHistogramScalarMul, NativeHistogramStddev, NativeHistogramStdvar, NativeHistogramSub, - NativeHistogramSum, NativeHistogramSumOverTime, NativeHistogramToString, PredictLinear, - PresentOverTime, PromqlFloatToString, QuantileOverTime, Rate, Resets, Round, StddevOverTime, - StdvarOverTime, SumOverTime, UniqueMatchGroup, quantile_udaf, + IDelta, IeeeComparison, IeeeSqrt, Increase, LastOverTime, MatchGroupViolation, MaxOverTime, + MinOverTime, MixedRange, NativeHistogramAbsentOverTime, NativeHistogramAdd, + NativeHistogramAggAvg, NativeHistogramAggSum, NativeHistogramAvg, NativeHistogramAvgOverTime, + NativeHistogramChanges, NativeHistogramCount, NativeHistogramCountOverTime, + NativeHistogramDelta, NativeHistogramDivScalar, NativeHistogramDrop, NativeHistogramEq, + NativeHistogramIDelta, NativeHistogramIRate, NativeHistogramIncrease, + NativeHistogramLastOverTime, NativeHistogramMulScalar, NativeHistogramNeg, + NativeHistogramNotEq, NativeHistogramPresentOverTime, NativeHistogramRate, + NativeHistogramResets, NativeHistogramScalarMul, NativeHistogramStddev, NativeHistogramStdvar, + NativeHistogramSub, NativeHistogramSum, NativeHistogramSumOverTime, NativeHistogramToString, + PredictLinear, PresentOverTime, PromqlFloatToString, QuantileOverTime, Rate, Resets, Round, + StddevOverTime, StdvarOverTime, SumOverTime, UniqueMatchGroup, quantile_udaf, }; use promql_parser::label::{METRIC_NAME, MatchOp, Matcher, Matchers}; use promql_parser::parser::token::TokenType; @@ -595,9 +596,10 @@ impl PromPlanner { }; let current_interval = self.ctx.interval; - if let Some(step) = step { - self.ctx.interval = step.as_millis() as _; - } + self.ctx.interval = match step { + Some(step) => step.as_millis() as _, + None => DEFAULT_SUBQUERY_STEP_MS, + }; ensure!(self.ctx.interval > 0, ZeroRangeSelectorSnafu); let current_start = self.ctx.start; let current_end = self.ctx.end; @@ -4830,10 +4832,14 @@ impl PromPlanner { } "round" => { if other_input_exprs.is_empty() { - other_input_exprs.push_front(0.0f64.lit()); + other_input_exprs.push_front(1.0f64.lit()); } ScalarFunc::DataFusionUdf(Arc::new(Round::scalar_udf())) } + // DataFusion's `sqrt` rejects negative input; PromQL returns NaN for it. + "sqrt" if !self.all_field_columns_are_native_histograms(input_schema) => { + ScalarFunc::DataFusionUdf(Arc::new(IeeeSqrt::scalar_udf())) + } "rad" | "deg" | "sgn" if self.all_field_columns_are_native_histograms(input_schema) => { ScalarFunc::DataFusionUdf(native_histogram_drop_udf(func.name)) } @@ -5683,19 +5689,41 @@ impl PromPlanner { vec![] }; - // update value column name according to the aggregators, - let mut new_field_columns = Vec::with_capacity(self.ctx.field_columns.len()); + // The alias is built from normalized columns so it matches the builtin's + // qualified name. + let exprs = normalize_cols(exprs, input_plan) + .context(DataFusionPlanningSnafu)? + .into_iter() + .map(Self::alias_extremum_as_builtin) + .collect::>(); - let normalized_exprs = - normalize_cols(exprs.iter().cloned(), input_plan).context(DataFusionPlanningSnafu)?; - for expr in normalized_exprs { - new_field_columns.push(expr.schema_name().to_string()); - } - self.ctx.field_columns = new_field_columns; + // update value column name according to the aggregators, + self.ctx.field_columns = exprs + .iter() + .map(|expr| expr.schema_name().to_string()) + .collect(); Ok((exprs, prev_field_exprs)) } + /// `prom_min`/`prom_max` keep their own names so SQL can use them next to `min`/`max` + /// in one aggregate. PromQL output columns keep the builtin names. + fn alias_extremum_as_builtin(expr: DfExpr) -> DfExpr { + let DfExpr::AggregateFunction(aggr) = &expr else { + return expr; + }; + let builtin = match aggr.func.name() { + Extremum::MIN_NAME => min_udaf(), + Extremum::MAX_NAME => max_udaf(), + _ => return expr, + }; + let name = builtin + .call(aggr.params.args.clone()) + .schema_name() + .to_string(); + expr.alias(name) + } + fn create_numeric_aggregate_expr( op: TokenType, param: &Option>, @@ -5713,8 +5741,8 @@ impl PromPlanner { } token::T_AVG => avg_udaf().call(vec![input]), token::T_COUNT_VALUES | token::T_COUNT => count_udaf().call(vec![input]), - token::T_MIN => min_udaf().call(vec![input]), - token::T_MAX => max_udaf().call(vec![input]), + token::T_MIN => Extremum::min_udaf().call(vec![input]), + token::T_MAX => Extremum::max_udaf().call(vec![input]), // PromQL's `group()` aggregator produces 1 for each group. // Use `max(1.0)` (per-group) to match semantics and output type (Float64). token::T_GROUP => max_udaf().call(vec![lit(1_f64)]), @@ -6058,7 +6086,15 @@ impl PromPlanner { .field_columns .iter() .map(|col| { - let mut sort_exprs = Vec::with_capacity(self.ctx.tag_columns.len() + 1); + let mut sort_exprs = Vec::with_capacity(self.ctx.tag_columns.len() + 2); + // PromQL ranks NaN last for both `topk` and `bottomk`, while DataFusion + // orders NaN above every number. + sort_exprs.push( + datafusion_functions::math::expr_fn::isnan(DfExpr::Column(Column::from_name( + col, + ))) + .sort(true, true), + ); // Order by value in the specific order sort_exprs.push(DfExpr::Column(Column::from_name(col)).sort(asc, true)); // Then tags if the values are equal, @@ -6249,12 +6285,24 @@ impl PromPlanner { token::T_MOD => Ok(Box::new(move |lhs: DfExpr, rhs| { Ok(cast_float(lhs) % cast_float(rhs)) })), - token::T_EQLC => Ok(Box::new(|lhs, rhs| Ok(lhs.eq(rhs)))), - token::T_NEQ => Ok(Box::new(|lhs, rhs| Ok(lhs.not_eq(rhs)))), - token::T_GTR => Ok(Box::new(|lhs, rhs| Ok(lhs.gt(rhs)))), - token::T_LSS => Ok(Box::new(|lhs, rhs| Ok(lhs.lt(rhs)))), - token::T_GTE => Ok(Box::new(|lhs, rhs| Ok(lhs.gt_eq(rhs)))), - token::T_LTE => Ok(Box::new(|lhs, rhs| Ok(lhs.lt_eq(rhs)))), + token::T_EQLC => Ok(Box::new(|lhs, rhs| { + Ok(Self::ieee_comparison(lhs, Operator::Eq, rhs)) + })), + token::T_NEQ => Ok(Box::new(|lhs, rhs| { + Ok(Self::ieee_comparison(lhs, Operator::NotEq, rhs)) + })), + token::T_GTR => Ok(Box::new(|lhs, rhs| { + Ok(Self::ieee_comparison(lhs, Operator::Gt, rhs)) + })), + token::T_LSS => Ok(Box::new(|lhs, rhs| { + Ok(Self::ieee_comparison(lhs, Operator::Lt, rhs)) + })), + token::T_GTE => Ok(Box::new(|lhs, rhs| { + Ok(Self::ieee_comparison(lhs, Operator::GtEq, rhs)) + })), + token::T_LTE => Ok(Box::new(|lhs, rhs| { + Ok(Self::ieee_comparison(lhs, Operator::LtEq, rhs)) + })), token::T_POW => Ok(Box::new(move |lhs, rhs| { Ok(DfExpr::ScalarFunction(ScalarFunction { func: datafusion_functions::math::power(), @@ -6271,6 +6319,14 @@ impl PromPlanner { } } + /// Compares two expressions with IEEE 754 semantics, as PromQL does; see [`IeeeComparison`]. + fn ieee_comparison(lhs: DfExpr, op: Operator, rhs: DfExpr) -> DfExpr { + DfExpr::ScalarFunction(ScalarFunction { + func: Arc::new(IeeeComparison::scalar_udf(op)), + args: vec![lhs, rhs], + }) + } + /// Check if the given op is a [comparison operator](https://prometheus.io/docs/prometheus/latest/querying/operators/#comparison-binary-operators). fn is_token_a_comparison_op(token: TokenType) -> bool { matches!( diff --git a/src/query/src/promql/planner/test.rs b/src/query/src/promql/planner/test.rs index f7cedb8784d..2f5b9b4628b 100644 --- a/src/query/src/promql/planner/test.rs +++ b/src/query/src/promql/planner/test.rs @@ -1827,7 +1827,20 @@ async fn single_sort_desc() { #[tokio::test] async fn single_sqrt() { - do_single_instant_function_call("sqrt", "sqrt").await; + // Negative input is mapped to NaN before DataFusion's `sqrt`, which rejects it. + indie_query_plan_compare( + r#"sqrt(some_metric{tag_0!="bar"})"#, + "Projection: some_metric.timestamp, sqrt(field_0), some_metric.tag_0 [timestamp:Timestamp(ms), sqrt(field_0):Float64;N, tag_0:Utf8]\ + \n Filter: sqrt(field_0) IS NOT NULL [timestamp:Timestamp(ms), sqrt(field_0):Float64;N, tag_0:Utf8, __promql_metric_name:Utf8]\ + \n Projection: some_metric.timestamp, prom_sqrt(some_metric.field_0) AS sqrt(field_0), some_metric.tag_0, __promql_metric_name [timestamp:Timestamp(ms), sqrt(field_0):Float64;N, tag_0:Utf8, __promql_metric_name:Utf8]\ + \n Projection: some_metric.tag_0, some_metric.timestamp, some_metric.field_0, Utf8(\"some_metric\") AS __promql_metric_name [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_metric_name:Utf8]\ + \n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ + \n PromSeriesDivide: tags=[\"tag_0\"] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ + \n Sort: some_metric.tag_0 ASC NULLS FIRST, some_metric.timestamp ASC NULLS FIRST [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ + \n Filter: some_metric.tag_0 != Utf8(\"bar\") AND some_metric.timestamp >= TimestampMillisecond(-999, None) AND some_metric.timestamp <= TimestampMillisecond(100000000, None) [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ + \n TableScan: some_metric [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]", + ) + .await; } #[tokio::test] @@ -1959,6 +1972,19 @@ async fn single_rad() { // }, // }, async fn do_aggregate_expr_plan(fn_name: &str, plan_name: &str) { + do_aggregate_expr_plan_with_display(fn_name, plan_name, plan_name).await; +} + +/// `display_name` is the aggregate function as the plan shows it, `plan_name` the name of the +/// output column; they differ for PromQL `min`/`max`. +async fn do_aggregate_expr_plan_with_display(fn_name: &str, plan_name: &str, display_name: &str) { + let display = |field: &str| { + if display_name == plan_name { + format!("{plan_name}({field})") + } else { + format!("{display_name}({field}) AS {plan_name}({field})") + } + }; let prom_expr = parser::parse(&format!( "{fn_name} by (tag_1)(some_metric{{tag_0!=\"bar\"}})", )) @@ -1985,14 +2011,17 @@ async fn do_aggregate_expr_plan(fn_name: &str, plan_name: &str) { .unwrap(); let expected_no_without = String::from( "Sort: some_metric.tag_1 ASC NULLS LAST, some_metric.timestamp ASC NULLS LAST [tag_1:Utf8, timestamp:Timestamp(ms), TEMPLATE(some_metric.field_0):Float64;N, TEMPLATE(some_metric.field_1):Float64;N]\ - \n Aggregate: groupBy=[[some_metric.tag_1, some_metric.timestamp]], aggr=[[TEMPLATE(some_metric.field_0), TEMPLATE(some_metric.field_1)]] [tag_1:Utf8, timestamp:Timestamp(ms), TEMPLATE(some_metric.field_0):Float64;N, TEMPLATE(some_metric.field_1):Float64;N]\ + \n Aggregate: groupBy=[[some_metric.tag_1, some_metric.timestamp]], aggr=[[DISPLAY_0, DISPLAY_1]] [tag_1:Utf8, timestamp:Timestamp(ms), TEMPLATE(some_metric.field_0):Float64;N, TEMPLATE(some_metric.field_1):Float64;N]\ \n Projection: some_metric.tag_0, some_metric.tag_1, some_metric.timestamp, some_metric.field_0, some_metric.field_1, Utf8(\"some_metric\") AS __promql_metric_name [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N, __promql_metric_name:Utf8]\ \n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]\ \n PromSeriesDivide: tags=[\"tag_0\", \"tag_1\"] [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]\ \n Sort: some_metric.tag_0 ASC NULLS FIRST, some_metric.tag_1 ASC NULLS FIRST, some_metric.timestamp ASC NULLS FIRST [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]\ \n Filter: some_metric.tag_0 != Utf8(\"bar\") AND some_metric.timestamp >= TimestampMillisecond(-999, None) AND some_metric.timestamp <= TimestampMillisecond(100000000, None) [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]\ \n TableScan: some_metric [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]" - ).replace("TEMPLATE", plan_name); + ) + .replace("TEMPLATE", plan_name) + .replace("DISPLAY_0", &display("some_metric.field_0")) + .replace("DISPLAY_1", &display("some_metric.field_1")); assert_eq!( plan.display_indent_schema().to_string(), expected_no_without @@ -2015,14 +2044,16 @@ async fn do_aggregate_expr_plan(fn_name: &str, plan_name: &str) { .unwrap(); let expected_without = String::from( "Sort: some_metric.tag_0 ASC NULLS LAST, some_metric.timestamp ASC NULLS LAST [tag_0:Utf8, timestamp:Timestamp(ms), TEMPLATE(some_metric.field_0):Float64;N, TEMPLATE(some_metric.field_1):Float64;N]\ - \n Aggregate: groupBy=[[some_metric.tag_0, some_metric.timestamp]], aggr=[[TEMPLATE(some_metric.field_0), TEMPLATE(some_metric.field_1)]] [tag_0:Utf8, timestamp:Timestamp(ms), TEMPLATE(some_metric.field_0):Float64;N, TEMPLATE(some_metric.field_1):Float64;N]\ + \n Aggregate: groupBy=[[some_metric.tag_0, some_metric.timestamp]], aggr=[[DISPLAY_0, DISPLAY_1]] [tag_0:Utf8, timestamp:Timestamp(ms), TEMPLATE(some_metric.field_0):Float64;N, TEMPLATE(some_metric.field_1):Float64;N]\ \n Projection: some_metric.tag_0, some_metric.tag_1, some_metric.timestamp, some_metric.field_0, some_metric.field_1, Utf8(\"some_metric\") AS __promql_metric_name [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N, __promql_metric_name:Utf8]\ \n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]\ \n PromSeriesDivide: tags=[\"tag_0\", \"tag_1\"] [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]\ \n Sort: some_metric.tag_0 ASC NULLS FIRST, some_metric.tag_1 ASC NULLS FIRST, some_metric.timestamp ASC NULLS FIRST [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]\ \n Filter: some_metric.tag_0 != Utf8(\"bar\") AND some_metric.timestamp >= TimestampMillisecond(-999, None) AND some_metric.timestamp <= TimestampMillisecond(100000000, None) [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]\ \n TableScan: some_metric [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]" - ).replace("TEMPLATE", plan_name); + ).replace("TEMPLATE", plan_name) + .replace("DISPLAY_0", &display("some_metric.field_0")) + .replace("DISPLAY_1", &display("some_metric.field_1")); assert_eq!(plan.display_indent_schema().to_string(), expected_without); } @@ -4048,12 +4079,12 @@ async fn aggregate_count() { #[tokio::test] async fn aggregate_min() { - do_aggregate_expr_plan("min", "min").await; + do_aggregate_expr_plan_with_display("min", "min", "prom_min").await; } #[tokio::test] async fn aggregate_max() { - do_aggregate_expr_plan("max", "max").await; + do_aggregate_expr_plan_with_display("max", "max", "prom_max").await; } #[tokio::test] @@ -4240,7 +4271,7 @@ async fn simple_bool_grammar() { let query = "some_metric != bool 1.2345"; let expected = String::from( "Projection: some_metric.tag_0, some_metric.timestamp, field_0 != Float64(1.2345) [tag_0:Utf8, timestamp:Timestamp(ms), field_0 != Float64(1.2345):Float64;N]\ - \n Projection: some_metric.tag_0, __promql_metric_name, some_metric.timestamp, CAST(some_metric.field_0 != Float64(1.2345) AS Float64) AS field_0 != Float64(1.2345) [tag_0:Utf8, __promql_metric_name:Utf8, timestamp:Timestamp(ms), field_0 != Float64(1.2345):Float64;N]\ + \n Projection: some_metric.tag_0, __promql_metric_name, some_metric.timestamp, CAST(prom_ne(some_metric.field_0, Float64(1.2345)) AS Float64) AS field_0 != Float64(1.2345) [tag_0:Utf8, __promql_metric_name:Utf8, timestamp:Timestamp(ms), field_0 != Float64(1.2345):Float64;N]\ \n Projection: some_metric.tag_0, some_metric.timestamp, some_metric.field_0, Utf8(\"some_metric\") AS __promql_metric_name [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_metric_name:Utf8]\ \n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ \n PromSeriesDivide: tags=[\"tag_0\"] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ @@ -4257,7 +4288,7 @@ async fn bool_with_additional_arithmetic() { let query = "some_metric + (1 == bool 2)"; let expected = String::from( "Projection: some_metric.tag_0, some_metric.timestamp, field_0 + Float64(1) = Float64(2) [tag_0:Utf8, timestamp:Timestamp(ms), field_0 + Float64(1) = Float64(2):Float64;N]\ - \n Projection: some_metric.tag_0, __promql_metric_name, some_metric.timestamp, CAST(some_metric.field_0 AS Float64) + CAST(Float64(1) = Float64(2) AS Float64) AS field_0 + Float64(1) = Float64(2) [tag_0:Utf8, __promql_metric_name:Utf8, timestamp:Timestamp(ms), field_0 + Float64(1) = Float64(2):Float64;N]\ + \n Projection: some_metric.tag_0, __promql_metric_name, some_metric.timestamp, CAST(some_metric.field_0 AS Float64) + CAST(prom_eq(Float64(1), Float64(2)) AS Float64) AS field_0 + Float64(1) = Float64(2) [tag_0:Utf8, __promql_metric_name:Utf8, timestamp:Timestamp(ms), field_0 + Float64(1) = Float64(2):Float64;N]\ \n Projection: some_metric.tag_0, some_metric.timestamp, some_metric.field_0, Utf8(\"some_metric\") AS __promql_metric_name [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_metric_name:Utf8]\ \n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ \n PromSeriesDivide: tags=[\"tag_0\"] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ @@ -5239,7 +5270,7 @@ async fn native_histogram_mixed_field_table_behaves() { async fn less_filter_on_value() { let query = "some_metric < 1.2345"; let expected = String::from( - "Filter: some_metric.field_0 < Float64(1.2345) [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_metric_name:Utf8]\ + "Filter: prom_lt(some_metric.field_0, Float64(1.2345)) [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_metric_name:Utf8]\ \n Projection: some_metric.tag_0, some_metric.timestamp, some_metric.field_0, Utf8(\"some_metric\") AS __promql_metric_name [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_metric_name:Utf8]\ \n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ \n PromSeriesDivide: tags=[\"tag_0\"] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\ @@ -5864,14 +5895,6 @@ async fn subquery_rejects_invalid_range_and_zero_step() { matches!(&err, crate::promql::error::Error::ZeroRangeSelector { .. }), "{err}" ); - // An omitted child step inherits the zero parent step. - let err = plan_subquery_offset_probe("sum_over_time(subquery_offset_offgrid[5s:])", 60, 60, 0) - .await - .unwrap_err(); - assert!( - matches!(&err, crate::promql::error::Error::ZeroRangeSelector { .. }), - "{err}" - ); } /// Rejects nonempty child windows whose derived millisecond bounds are unrepresentable. @@ -6882,9 +6905,9 @@ async fn test_topk_expr() { .await .unwrap(); let expected = "Projection: sum(prometheus_tsdb_head_series.greptime_value), prometheus_tsdb_head_series.ip, prometheus_tsdb_head_series.greptime_timestamp [sum(prometheus_tsdb_head_series.greptime_value):Float64;N, ip:Utf8, greptime_timestamp:Timestamp(ms)]\ - \n Sort: prometheus_tsdb_head_series.greptime_timestamp ASC NULLS LAST, row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ASC NULLS LAST [ip:Utf8, greptime_timestamp:Timestamp(ms), sum(prometheus_tsdb_head_series.greptime_value):Float64;N, row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW:UInt64]\ - \n Filter: row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW <= Float64(10) [ip:Utf8, greptime_timestamp:Timestamp(ms), sum(prometheus_tsdb_head_series.greptime_value):Float64;N, row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW:UInt64]\ - \n WindowAggr: windowExpr=[[row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]] [ip:Utf8, greptime_timestamp:Timestamp(ms), sum(prometheus_tsdb_head_series.greptime_value):Float64;N, row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW:UInt64]\ + \n Sort: prometheus_tsdb_head_series.greptime_timestamp ASC NULLS LAST, row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [isnan(sum(prometheus_tsdb_head_series.greptime_value)) ASC NULLS FIRST, sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ASC NULLS LAST [ip:Utf8, greptime_timestamp:Timestamp(ms), sum(prometheus_tsdb_head_series.greptime_value):Float64;N, row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [isnan(sum(prometheus_tsdb_head_series.greptime_value)) ASC NULLS FIRST, sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW:UInt64]\ + \n Filter: row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [isnan(sum(prometheus_tsdb_head_series.greptime_value)) ASC NULLS FIRST, sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW <= Float64(10) [ip:Utf8, greptime_timestamp:Timestamp(ms), sum(prometheus_tsdb_head_series.greptime_value):Float64;N, row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [isnan(sum(prometheus_tsdb_head_series.greptime_value)) ASC NULLS FIRST, sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW:UInt64]\ + \n WindowAggr: windowExpr=[[row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [isnan(sum(prometheus_tsdb_head_series.greptime_value)) ASC NULLS FIRST, sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]] [ip:Utf8, greptime_timestamp:Timestamp(ms), sum(prometheus_tsdb_head_series.greptime_value):Float64;N, row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [isnan(sum(prometheus_tsdb_head_series.greptime_value)) ASC NULLS FIRST, sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW:UInt64]\ \n Sort: prometheus_tsdb_head_series.ip ASC NULLS LAST, prometheus_tsdb_head_series.greptime_timestamp ASC NULLS LAST [ip:Utf8, greptime_timestamp:Timestamp(ms), sum(prometheus_tsdb_head_series.greptime_value):Float64;N]\ \n Aggregate: groupBy=[[prometheus_tsdb_head_series.ip, prometheus_tsdb_head_series.greptime_timestamp]], aggr=[[sum(prometheus_tsdb_head_series.greptime_value)]] [ip:Utf8, greptime_timestamp:Timestamp(ms), sum(prometheus_tsdb_head_series.greptime_value):Float64;N]\ \n Projection: prometheus_tsdb_head_series.ip, prometheus_tsdb_head_series.greptime_timestamp, prometheus_tsdb_head_series.greptime_value, Utf8(\"prometheus_tsdb_head_series\") AS __promql_metric_name [ip:Utf8, greptime_timestamp:Timestamp(ms), greptime_value:Float64;N, __promql_metric_name:Utf8]\ diff --git a/src/query/src/query_engine/default_serializer.rs b/src/query/src/query_engine/default_serializer.rs index 63ee0856aaf..ced633382d3 100644 --- a/src/query/src/query_engine/default_serializer.rs +++ b/src/query/src/query_engine/default_serializer.rs @@ -32,7 +32,7 @@ use datafusion_expr::{Expr, UserDefinedLogicalNode}; use greptime_proto::substrait_extension::MergeScan as PbMergeScan; use promql::functions::{ AbsentOverTime, AvgOverTime, Changes, CountOverTime, Delta, Deriv, DoubleExponentialSmoothing, - IDelta, Increase, LastOverTime, MaxOverTime, MinOverTime, MixedRange, + IDelta, IeeeComparison, IeeeSqrt, Increase, LastOverTime, MaxOverTime, MinOverTime, MixedRange, NativeHistogramAbsentOverTime, NativeHistogramAdd, NativeHistogramAggAvg, NativeHistogramAggSum, NativeHistogramAvg, NativeHistogramAvgOverTime, NativeHistogramChanges, NativeHistogramCount, NativeHistogramCountOverTime, NativeHistogramDelta, @@ -334,6 +334,10 @@ fn register_greptime_functions( } let _ = session_state.register_udaf(quantile_udaf()); + for udf in IeeeComparison::all() { + let _ = session_state.register_udf(Arc::new(udf)); + } + let _ = session_state.register_udf(Arc::new(IeeeSqrt::scalar_udf())); let _ = session_state.register_udf(Arc::new(IDelta::::scalar_udf())); let _ = session_state.register_udf(Arc::new(IDelta::::scalar_udf())); diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs index 2653ceddbe5..210fe0cee00 100644 --- a/src/servers/src/http.rs +++ b/src/servers/src/http.rs @@ -1576,6 +1576,26 @@ impl HttpServer { "/api/traces/{trace_id}", routing::get(jaeger::handle_get_trace), ) + .route( + "/api/v3/services", + routing::get(jaeger::v3::handle_get_services), + ) + .route( + "/api/v3/operations", + routing::get(jaeger::v3::handle_get_operations), + ) + .route( + "/api/v3/traces", + routing::get(jaeger::v3::handle_find_traces), + ) + .route( + "/api/v3/trace-summaries", + routing::get(jaeger::v3::handle_find_trace_summaries), + ) + .route( + "/api/v3/traces/{trace_id}", + routing::get(jaeger::v3::handle_get_trace), + ) .with_state(handler) } diff --git a/src/servers/src/http/jaeger.rs b/src/servers/src/http/jaeger.rs index 16dbaf76e53..f636ccb3df8 100644 --- a/src/servers/src/http/jaeger.rs +++ b/src/servers/src/http/jaeger.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod v3; + use std::collections::{BTreeMap, HashMap}; use std::fmt; use std::str::FromStr; @@ -289,9 +291,9 @@ fn update_query_context(query_ctx: &mut QueryContext, table_name: Option impl QueryTraceParams { fn from_jaeger_query_params(query_params: JaegerQueryParams) -> Result { let mut internal_query_params: QueryTraceParams = QueryTraceParams { - service_name: query_params.service_name.context(InvalidJaegerQuerySnafu { + service_name: Some(query_params.service_name.context(InvalidJaegerQuerySnafu { reason: "service_name is required".to_string(), - })?, + })?), operation_name: query_params.operation_name, // Convert start time from microseconds to nanoseconds. start_time: query_params.start.map(|start| start * 1000), @@ -347,7 +349,7 @@ impl QueryTraceParams { #[derive(Debug, Default, PartialEq)] pub struct QueryTraceParams { - pub service_name: String, + pub service_name: Option, pub operation_name: Option, // The limit of the number of traces to return. @@ -362,6 +364,9 @@ pub struct QueryTraceParams { pub min_duration: Option, pub max_duration: Option, + /// Fetch all spans of matching traces, including spans outside the search window. + pub fetch_full_trace: bool, + // The user agent of the trace query, mainly find traces pub user_agent: TraceUserAgent, } @@ -1647,7 +1652,7 @@ mod tests { ..Default::default() }, QueryTraceParams { - service_name: "test-service-0".to_string(), + service_name: Some("test-service-0".to_string()), ..Default::default() }, ), @@ -1664,7 +1669,7 @@ mod tests { ..Default::default() }, QueryTraceParams { - service_name: "test-service-0".to_string(), + service_name: Some("test-service-0".to_string()), operation_name: Some("access-mysql".to_string()), start_time: Some(1738726754492422000), end_time: Some(1738726754642422000), @@ -1679,6 +1684,7 @@ mod tests { ("http.path".to_string(), JsonValue::String("/api/v1/users".to_string())), ])), user_agent: TraceUserAgent::Jaeger, + fetch_full_trace: false, }, ), ]; diff --git a/src/servers/src/http/jaeger/v3.rs b/src/servers/src/http/jaeger/v3.rs new file mode 100644 index 00000000000..ccb8a235fd3 --- /dev/null +++ b/src/servers/src/http/jaeger/v3.rs @@ -0,0 +1,542 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod otlp; +mod summary; + +use std::collections::HashMap; +use std::sync::Arc; + +use axum::extract::rejection::QueryRejection; +use axum::extract::{Path, Query, State}; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; +use axum::{Extension, Json}; +use common_error::ext::ErrorExt; +use common_error::status_code::StatusCode as GreptimeStatusCode; +use common_query::Output; +use common_telemetry::error; +use rust_decimal::Decimal; +use rust_decimal::prelude::ToPrimitive; +use serde::Deserialize; +use serde_json::{Value, json}; +use session::context::QueryContext; + +use crate::error::{Error, status_code_to_http_status}; +use crate::http::extractor::TraceTableName; +use crate::http::jaeger::{ + QueryTraceParams, convert_string_to_boolean, convert_string_to_number, covert_to_records, + empty_string_as_none, operations_from_records, services_from_records, update_query_context, +}; +use crate::metrics::METRIC_JAEGER_QUERY_ELAPSED; +use crate::query_handler::JaegerQueryHandlerRef; + +/// Jaeger API v3 uses an error envelope rather than the legacy `errors` array. +#[derive(Debug)] +pub struct ApiError { + status: StatusCode, + message: String, +} + +impl ApiError { + fn invalid(message: impl Into) -> Self { + Self { + status: StatusCode::BAD_REQUEST, + message: message.into(), + } + } + + fn not_found() -> Self { + Self { + status: StatusCode::NOT_FOUND, + message: "No traces found".into(), + } + } +} + +impl From for ApiError { + fn from(err: Error) -> Self { + error!(err; "Jaeger v3 query failed"); + Self { + status: status_code_to_http_status(&err.status_code()), + message: err.output_msg(), + } + } +} + +impl From for ApiError { + fn from(err: QueryRejection) -> Self { + Self::invalid(err.body_text()) + } +} + +impl IntoResponse for ApiError { + fn into_response(self) -> Response { + ( + self.status, + Json(json!({"error": {"httpCode": self.status.as_u16(), "message": self.message}})), + ) + .into_response() + } +} + +#[derive(Debug, Default, Deserialize)] +pub struct OperationsParams { + service: String, + #[serde(rename = "spanKind", alias = "span_kind")] + span_kind: Option, +} + +#[derive(Debug, Default, Deserialize)] +pub struct GetTraceParams { + #[serde(rename = "startTime", alias = "start_time")] + #[serde(default, deserialize_with = "empty_string_as_none")] + start_time: Option, + #[serde(rename = "endTime", alias = "end_time")] + #[serde(default, deserialize_with = "empty_string_as_none")] + end_time: Option, + // GreptimeDB returns stored spans without Jaeger's clock-skew enrichment. + #[serde(rename = "rawTraces", alias = "raw_traces")] + #[serde(default, deserialize_with = "empty_string_as_none")] + _raw_traces: Option, +} + +#[derive(Debug, Default, Deserialize)] +pub struct FindTracesParams { + #[serde(rename = "query.serviceName", alias = "query.service_name")] + service_name: Option, + #[serde(rename = "query.operationName", alias = "query.operation_name")] + operation_name: Option, + #[serde(rename = "query.startTimeMin", alias = "query.start_time_min")] + start_time_min: String, + #[serde(rename = "query.startTimeMax", alias = "query.start_time_max")] + start_time_max: String, + #[serde(rename = "query.durationMin", alias = "query.duration_min")] + #[serde(default, deserialize_with = "empty_string_as_none")] + duration_min: Option, + #[serde(rename = "query.durationMax", alias = "query.duration_max")] + #[serde(default, deserialize_with = "empty_string_as_none")] + duration_max: Option, + #[serde( + rename = "query.searchDepth", + alias = "query.search_depth", + alias = "query.num_traces" + )] + #[serde(default, deserialize_with = "empty_string_as_none")] + search_depth: Option, + #[serde(rename = "query.attributes")] + #[serde(default, deserialize_with = "empty_string_as_none")] + attributes: Option, + #[serde(rename = "query.rawTraces", alias = "query.raw_traces")] + #[serde(default, deserialize_with = "empty_string_as_none")] + _raw_traces: Option, + #[serde(rename = "query.filter")] + filter: Option, +} + +impl FindTracesParams { + fn into_query(self) -> Result { + if self.filter.is_some() { + return Err(ApiError::invalid("query.filter is not supported")); + } + let start = parse_timestamp(&self.start_time_min)?; + let end = parse_timestamp(&self.start_time_max)?; + if start >= end { + return Err(ApiError::invalid( + "query.startTimeMin must be before query.startTimeMax", + )); + } + let limit = match self.search_depth.unwrap_or(0) { + 0 => 100, + limit @ 1..=10_000 => limit, + _ => { + return Err(ApiError::invalid( + "query.searchDepth must be between 0 and 10000", + )); + } + }; + let tags = self + .attributes + .map(|attributes| { + let values = serde_json::from_str::>(&attributes) + .map_err(|err| ApiError::invalid(format!("Invalid query.attributes: {err}")))?; + Ok::<_, ApiError>( + values + .into_iter() + .map(|(key, value)| { + let value = Value::String(value); + let value = convert_string_to_number(&value) + .or_else(|| convert_string_to_boolean(&value)) + .unwrap_or(value); + (key, value) + }) + .collect(), + ) + }) + .transpose()?; + let query = QueryTraceParams { + service_name: self.service_name.filter(|name| !name.is_empty()), + operation_name: self.operation_name.filter(|name| !name.is_empty()), + start_time: Some(start), + // The backend uses an inclusive bound; API v3 specifies an exclusive bound. + end_time: Some(end - 1), + min_duration: self + .duration_min + .as_deref() + .map(parse_duration) + .transpose()?, + max_duration: self + .duration_max + .as_deref() + .map(parse_duration) + .transpose()?, + limit: Some(limit as usize), + tags, + fetch_full_trace: true, + ..Default::default() + }; + if let (Some(min), Some(max)) = (query.min_duration, query.max_duration) + && min > max + { + return Err(ApiError::invalid( + "query.durationMin must not exceed query.durationMax", + )); + } + Ok(query) + } +} + +fn parse_timestamp(value: &str) -> Result { + chrono::DateTime::parse_from_rfc3339(value) + .map_err(|err| ApiError::invalid(format!("Invalid RFC3339 timestamp '{value}': {err}")))? + .timestamp_nanos_opt() + .ok_or_else(|| ApiError::invalid("Timestamp is outside the supported nanosecond range")) +} + +fn parse_duration(value: &str) -> Result { + let invalid = || ApiError::invalid(format!("Invalid Go duration '{value}'")); + let mut rest = value.strip_prefix('+').unwrap_or(value); + if rest == "0" { + return Ok(0); + } + if rest.is_empty() { + return Err(invalid()); + } + let mut total = 0_u64; + while !rest.is_empty() { + let number_end = rest + .find(|ch: char| !ch.is_ascii_digit() && ch != '.') + .ok_or_else(invalid)?; + let amount: Decimal = rest[..number_end].parse().map_err(|_| invalid())?; + rest = &rest[number_end..]; + let unit_end = rest + .find(|ch: char| ch.is_ascii_digit() || ch == '.') + .unwrap_or(rest.len()); + let unit = match &rest[..unit_end] { + "ns" => 1_u64, + "us" | "µs" | "μs" => 1_000, + "ms" => 1_000_000, + "s" => 1_000_000_000, + "m" => 60_000_000_000, + "h" => 3_600_000_000_000, + _ => return Err(invalid()), + }; + let nanos = amount + .checked_mul(Decimal::from(unit)) + .and_then(|amount| amount.trunc().to_u64()) + .ok_or_else(invalid)?; + total = total + .checked_add(nanos) + .filter(|total| *total <= i64::MAX as u64) + .ok_or_else(invalid)?; + rest = &rest[unit_end..]; + } + Ok(total) +} + +pub async fn handle_get_services( + State(handler): State, + Extension(mut ctx): Extension, + TraceTableName(table): TraceTableName, +) -> Result, ApiError> { + update_query_context(&mut ctx, table); + let _timer = METRIC_JAEGER_QUERY_ELAPSED + .with_label_values(&[&ctx.get_db_string(), "/api/v3/services"]) + .start_timer(); + let services = match query_records(handler.get_services(Arc::new(ctx)).await).await? { + Some(records) => services_from_records(records)?, + None => vec![], + }; + Ok(Json(json!({"services": services}))) +} + +pub async fn handle_get_operations( + State(handler): State, + params: Result, QueryRejection>, + Extension(mut ctx): Extension, + TraceTableName(table): TraceTableName, +) -> Result, ApiError> { + let Query(params) = params?; + if params.service.is_empty() { + return Err(ApiError::invalid("service is required")); + } + update_query_context(&mut ctx, table); + let _timer = METRIC_JAEGER_QUERY_ELAPSED + .with_label_values(&[&ctx.get_db_string(), "/api/v3/operations"]) + .start_timer(); + let operations = match query_records( + handler + .get_operations( + Arc::new(ctx), + ¶ms.service, + params.span_kind.as_deref().filter(|kind| !kind.is_empty()), + ) + .await, + ) + .await? + { + Some(records) => operations_from_records(records, true)?, + None => vec![], + }; + let operations: Vec<_> = operations.into_iter().map(|operation| json!({ + "name": operation.name, + "spanKind": operation.span_kind.filter(|kind| !kind.is_empty()).unwrap_or_else(|| "internal".into()), + })).collect(); + Ok(Json(json!({"operations": operations}))) +} + +pub async fn handle_get_trace( + State(handler): State, + Path(trace_id): Path, + params: Result, QueryRejection>, + Extension(mut ctx): Extension, + TraceTableName(table): TraceTableName, +) -> Result, ApiError> { + let Query(params) = params?; + if !matches!(trace_id.len(), 16 | 32) || !trace_id.bytes().all(|ch| ch.is_ascii_hexdigit()) { + return Err(ApiError::invalid( + "trace_id must be a 64-bit or 128-bit hexadecimal ID", + )); + } + let trace_id = format!("{:0>32}", trace_id.to_ascii_lowercase()); + let start = params + .start_time + .as_deref() + .map(parse_timestamp) + .transpose()?; + let end = params + .end_time + .as_deref() + .map(parse_timestamp) + .transpose()?; + if let (Some(start), Some(end)) = (start, end) + && start > end + { + return Err(ApiError::invalid("startTime must not exceed endTime")); + } + update_query_context(&mut ctx, table); + let _timer = METRIC_JAEGER_QUERY_ELAPSED + .with_label_values(&[&ctx.get_db_string(), "/api/v3/traces/{trace_id}"]) + .start_timer(); + // API v3 time bounds are lookup hints. Applying them as span filters would truncate the trace, + // so they are only validated. + trace_response( + handler + .get_trace(Arc::new(ctx), &trace_id, None, None, None) + .await, + ) + .await +} + +pub async fn handle_find_traces( + State(handler): State, + params: Result, QueryRejection>, + Extension(mut ctx): Extension, + TraceTableName(table): TraceTableName, +) -> Result, ApiError> { + let Query(params) = params?; + let query = params.into_query()?; + update_query_context(&mut ctx, table); + let _timer = METRIC_JAEGER_QUERY_ELAPSED + .with_label_values(&[&ctx.get_db_string(), "/api/v3/traces"]) + .start_timer(); + trace_response(handler.find_traces(Arc::new(ctx), query).await).await +} + +pub async fn handle_find_trace_summaries( + State(handler): State, + params: Result, QueryRejection>, + Extension(mut ctx): Extension, + TraceTableName(table): TraceTableName, +) -> Result, ApiError> { + let Query(params) = params?; + let query = params.into_query()?; + update_query_context(&mut ctx, table); + let _timer = METRIC_JAEGER_QUERY_ELAPSED + .with_label_values(&[&ctx.get_db_string(), "/api/v3/trace-summaries"]) + .start_timer(); + let summaries = match query_records(handler.find_traces(Arc::new(ctx), query).await).await? { + Some(records) => summary::summarize(otlp::traces_from_records(records)?), + None => vec![], + }; + Ok(Json(json!({"summaries": summaries}))) +} + +async fn query_records( + output: crate::error::Result, +) -> Result, ApiError> { + match output { + Ok(output) => covert_to_records(output) + .await? + .map(Some) + .ok_or_else(|| ApiError { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: "Unexpected Jaeger query output".into(), + }), + Err(err) if err.status_code() == GreptimeStatusCode::TableNotFound => Ok(None), + Err(err) => Err(err.into()), + } +} + +async fn trace_response(output: crate::error::Result) -> Result, ApiError> { + let records = query_records(output) + .await? + .ok_or_else(ApiError::not_found)?; + if records.rows.is_empty() { + return Err(ApiError::not_found()); + } + Ok(Json(json!({"result": otlp::traces_from_records(records)?}))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_find_traces_parameters() { + for (start, end, service, operation, depth, duration) in [ + ( + "start_time_min", + "start_time_max", + "service_name", + "operation_name", + "num_traces", + "duration_min", + ), + ( + "startTimeMin", + "startTimeMax", + "serviceName", + "operationName", + "searchDepth", + "durationMin", + ), + ] { + let uri = format!("/api/v3/traces?query.{start}=2026-01-01T00:00:00.000000001Z&query.{end}=2026-01-01T00:00:01Z&query.{service}=checkout&query.{operation}=GET&query.{depth}=7&query.{duration}=1.5ms&query.attributes=%7B%22error%22%3A%22true%22%7D").parse().unwrap(); + let Query(params) = Query::::try_from_uri(&uri).unwrap(); + let query = params.into_query().unwrap(); + assert_eq!(query.service_name.as_deref(), Some("checkout")); + assert_eq!(query.operation_name.as_deref(), Some("GET")); + assert_eq!(query.limit, Some(7)); + assert_eq!(query.min_duration, Some(1_500_000)); + assert_eq!(query.start_time, Some(1767225600000000001)); + assert_eq!(query.end_time, Some(1767225600999999999)); + assert_eq!(query.tags.unwrap()["error"], json!(true)); + assert!(query.fetch_full_trace); + } + let uri = "/api/v3/traces?query.startTimeMin=2026-01-01T00:00:00Z&query.startTimeMax=2026-01-01T01:00:00Z&query.durationMin=&query.durationMax=&query.searchDepth=&query.rawTraces=&query.attributes=".parse().unwrap(); + let Query(params) = Query::::try_from_uri(&uri).unwrap(); + let query = params.into_query().unwrap(); + assert_eq!(query.service_name, None); + assert_eq!(query.limit, Some(100)); + for name in [ + "query.searchDepth", + "query.search_depth", + "query.num_traces", + ] { + for (depth, expected) in [(0, 100), (1, 1), (10_000, 10_000)] { + let uri = format!("/api/v3/traces?query.startTimeMin=2026-01-01T00:00:00Z&query.startTimeMax=2026-01-01T01:00:00Z&{name}={depth}").parse().unwrap(); + let Query(params) = Query::::try_from_uri(&uri).unwrap(); + assert_eq!(params.into_query().unwrap().limit, Some(expected)); + } + } + } + + #[test] + fn test_go_durations() { + for (value, expected) in [ + ("0", 0), + ("1.5ms", 1_500_000), + ("1h2m3.5s", 3_723_500_000_000), + ("+.5us", 500), + ("1.µs", 1_000), + ("1μs", 1_000), + ("0.1ns", 0), + ("9223372036854775807ns", i64::MAX as u64), + ] { + assert_eq!(parse_duration(value).unwrap(), expected, "{value}"); + } + for value in [ + "", + "1", + "-1s", + "1.2.3s", + "s", + "1 s", + "1d", + "1s-1s", + "9223372036854775808ns", + ] { + assert!(parse_duration(value).is_err(), "{value}"); + } + } + + #[test] + fn test_invalid_find_traces_parameters() { + for extra in [ + "query.searchDepth=-1", + "query.searchDepth=10001", + "query.searchDepth=2147483647", + "query.durationMin=-1s", + "query.durationMin=2s&query.durationMax=1s", + "query.durationMax=999999999999h", + "query.attributes=%7B%22error%22%3Atrue%7D", + "query.attributes=invalid", + "query.filter=%7B%7D", + ] { + let uri = format!("/api/v3/traces?query.startTimeMin=2026-01-01T00:00:00Z&query.startTimeMax=2026-01-01T01:00:00Z&{extra}").parse().unwrap(); + let Query(params) = Query::::try_from_uri(&uri).unwrap(); + assert_eq!( + params.into_query().unwrap_err().status, + StatusCode::BAD_REQUEST, + "{extra}" + ); + } + for (start, end) in [ + ("invalid", "2026-01-01T00:00:00Z"), + ("2500-01-01T00:00:00Z", "2500-01-02T00:00:00Z"), + ("2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z"), + ] { + assert!( + FindTracesParams { + start_time_min: start.into(), + start_time_max: end.into(), + ..Default::default() + } + .into_query() + .is_err() + ); + } + } +} diff --git a/src/servers/src/http/jaeger/v3/otlp.rs b/src/servers/src/http/jaeger/v3/otlp.rs new file mode 100644 index 00000000000..4d09924ad39 --- /dev/null +++ b/src/servers/src/http/jaeger/v3/otlp.rs @@ -0,0 +1,294 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::{HashMap, HashSet}; + +use axum::http::StatusCode; +use opentelemetry_proto::tonic::common::v1::{ + AnyValue, ArrayValue, InstrumentationScope, KeyValue, KeyValueList, any_value, +}; +use opentelemetry_proto::tonic::resource::v1::Resource; +use opentelemetry_proto::tonic::trace::v1::span::{Event, Link, SpanKind}; +use opentelemetry_proto::tonic::trace::v1::status::StatusCode as SpanStatusCode; +use opentelemetry_proto::tonic::trace::v1::{ResourceSpans, ScopeSpans, Span, Status, TracesData}; +use serde_json::Value; + +use crate::http::HttpRecordsOutput; +use crate::http::jaeger::v3::ApiError; +use crate::otlp::trace::{ + KEY_SERVICE_NAME, PARENT_SPAN_ID_COLUMN, RESOURCE_ATTRIBUTES_COLUMN, SCOPE_ATTRIBUTES_COLUMN, + SCOPE_NAME_COLUMN, SCOPE_VERSION_COLUMN, SERVICE_NAME_COLUMN, SPAN_ATTRIBUTES_COLUMN, + SPAN_EVENTS_COLUMN, SPAN_ID_COLUMN, SPAN_KIND_COLUMN, SPAN_LINKS_COLUMN, SPAN_NAME_COLUMN, + SPAN_STATUS_CODE, SPAN_STATUS_MESSAGE_COLUMN, TIMESTAMP_COLUMN, TIMESTAMP_END_COLUMN, + TRACE_ID_COLUMN, TRACE_STATE_COLUMN, +}; + +/// Converts stored trace rows directly to OTLP, without the legacy microsecond conversion. +pub fn traces_from_records(records: HttpRecordsOutput) -> Result { + let binary_columns: HashSet<_> = records + .schema + .column_schemas + .iter() + .filter(|column| column.data_type == "Binary") + .map(|column| column.name.as_str()) + .collect(); + let mut resource_spans = Vec::with_capacity(records.rows.len()); + for row in &records.rows { + let cells: HashMap<_, _> = records + .schema + .column_schemas + .iter() + .zip(row) + .map(|(column, cell)| (column.name.as_str(), cell)) + .collect(); + let string = |name| cells.get(name).and_then(|v| v.as_str()).unwrap_or_default(); + let timestamp = |name| { + cells + .get(name) + .and_then(|v| v.as_u64()) + .ok_or_else(|| invalid_data(format!("Invalid timestamp column: {name}"))) + }; + let span = Span { + trace_id: decode_id(string(TRACE_ID_COLUMN), 16)?, + span_id: decode_id(string(SPAN_ID_COLUMN), 8)?, + parent_span_id: if string(PARENT_SPAN_ID_COLUMN).is_empty() { + vec![] + } else { + decode_id(string(PARENT_SPAN_ID_COLUMN), 8)? + }, + trace_state: string(TRACE_STATE_COLUMN).into(), + name: string(SPAN_NAME_COLUMN).into(), + kind: SpanKind::from_str_name(string(SPAN_KIND_COLUMN)).unwrap_or(SpanKind::Unspecified) + as i32, + start_time_unix_nano: timestamp(TIMESTAMP_COLUMN)?, + end_time_unix_nano: timestamp(TIMESTAMP_END_COLUMN)?, + attributes: attributes(&cells, SPAN_ATTRIBUTES_COLUMN, &binary_columns)?, + events: events(cells.get(SPAN_EVENTS_COLUMN).copied())?, + links: links(cells.get(SPAN_LINKS_COLUMN).copied())?, + status: Some(Status { + code: SpanStatusCode::from_str_name(string(SPAN_STATUS_CODE)) + .unwrap_or(SpanStatusCode::Unset) as i32, + message: string(SPAN_STATUS_MESSAGE_COLUMN).into(), + }), + ..Default::default() + }; + let mut resource_attributes = + attributes(&cells, RESOURCE_ATTRIBUTES_COLUMN, &binary_columns)?; + if !resource_attributes + .iter() + .any(|attr| attr.key == KEY_SERVICE_NAME) + && !string(SERVICE_NAME_COLUMN).is_empty() + { + resource_attributes.push(KeyValue { + key: KEY_SERVICE_NAME.into(), + value: Some(AnyValue { + value: Some(any_value::Value::StringValue( + string(SERVICE_NAME_COLUMN).into(), + )), + }), + ..Default::default() + }); + } + // Each row retains its own resource and scope. Service name alone does not identify + // a resource: different instances of one service can have different attributes. + resource_spans.push(ResourceSpans { + resource: Some(Resource { + attributes: resource_attributes, + ..Default::default() + }), + scope_spans: vec![ScopeSpans { + scope: Some(InstrumentationScope { + name: string(SCOPE_NAME_COLUMN).into(), + version: string(SCOPE_VERSION_COLUMN).into(), + attributes: attributes(&cells, SCOPE_ATTRIBUTES_COLUMN, &binary_columns)?, + ..Default::default() + }), + spans: vec![span], + ..Default::default() + }], + ..Default::default() + }); + } + Ok(TracesData { resource_spans }) +} + +fn invalid_data(message: impl Into) -> ApiError { + ApiError { + status: StatusCode::INTERNAL_SERVER_ERROR, + message: message.into(), + } +} + +fn decode_id(value: &str, length: usize) -> Result, ApiError> { + let bytes = hex::decode(value).map_err(|_| invalid_data("Invalid stored span or trace ID"))?; + if bytes.len() != length { + return Err(invalid_data("Invalid stored span or trace ID length")); + } + Ok(bytes) +} + +fn attributes( + cells: &HashMap<&str, &Value>, + name: &str, + binary_columns: &HashSet<&str>, +) -> Result, ApiError> { + if let Some(value) = cells.get(name) { + return object_attributes(value); + } + let prefix = format!("{name}."); + let mut attrs = cells + .iter() + .filter_map(|(column, value)| { + column + .strip_prefix(&prefix) + .filter(|_| !value.is_null()) + .map(|key| (key, *value, binary_columns.contains(column))) + }) + .map(|(key, value, is_binary)| { + let value = if is_binary { + AnyValue { + value: Some(any_value::Value::BytesValue( + serde_json::from_value(value.clone()) + .map_err(|_| invalid_data("Invalid stored binary attribute"))?, + )), + } + } else { + any_value(value)? + }; + Ok(KeyValue { + key: key.into(), + value: Some(value), + ..Default::default() + }) + }) + .collect::, ApiError>>()?; + attrs.sort_unstable_by(|a, b| a.key.cmp(&b.key)); + Ok(attrs) +} + +fn object_attributes(value: &Value) -> Result, ApiError> { + match value { + Value::Null => Ok(vec![]), + Value::Object(values) => values + .iter() + .map(|(key, value)| { + Ok(KeyValue { + key: key.clone(), + value: Some(any_value(value)?), + ..Default::default() + }) + }) + .collect(), + _ => Err(invalid_data("Stored attributes must be a JSON object")), + } +} + +fn any_value(value: &Value) -> Result { + use any_value::Value as OtlpValue; + + let value = match value { + Value::Null => None, + Value::String(value) => Some(OtlpValue::StringValue(value.clone())), + Value::Bool(value) => Some(OtlpValue::BoolValue(*value)), + Value::Number(value) => Some(if let Some(value) = value.as_i64() { + OtlpValue::IntValue(value) + } else { + OtlpValue::DoubleValue( + value + .as_f64() + .ok_or_else(|| invalid_data("Invalid numeric attribute"))?, + ) + }), + Value::Array(values) => Some(OtlpValue::ArrayValue(ArrayValue { + values: values.iter().map(any_value).collect::>()?, + })), + Value::Object(_) => Some(OtlpValue::KvlistValue(KeyValueList { + values: object_attributes(value)?, + })), + }; + Ok(AnyValue { value }) +} + +fn events(value: Option<&Value>) -> Result, ApiError> { + let Some(value) = value.filter(|value| !value.is_null()) else { + return Ok(vec![]); + }; + let values = value + .as_array() + .ok_or_else(|| invalid_data("Invalid stored span events"))?; + values + .iter() + .map(|event| { + let time = event["time"] + .as_str() + .ok_or_else(|| invalid_data("Missing span event time"))?; + let time = chrono::DateTime::parse_from_str(time, "%Y-%m-%d %H:%M:%S%.f%z") + .ok() + .and_then(|time| time.timestamp_nanos_opt()) + .and_then(|time| u64::try_from(time).ok()) + .ok_or_else(|| invalid_data("Invalid stored span event time"))?; + Ok(Event { + time_unix_nano: time, + name: event["name"] + .as_str() + .ok_or_else(|| invalid_data("Missing span event name"))? + .into(), + attributes: object_attributes(&event["attributes"])?, + ..Default::default() + }) + }) + .collect() +} + +fn links(value: Option<&Value>) -> Result, ApiError> { + let Some(value) = value.filter(|value| !value.is_null()) else { + return Ok(vec![]); + }; + let values = value + .as_array() + .ok_or_else(|| invalid_data("Invalid stored span links"))?; + values + .iter() + .map(|link| { + Ok(Link { + trace_id: decode_id(link["trace_id"].as_str().unwrap_or_default(), 16)?, + span_id: decode_id(link["span_id"].as_str().unwrap_or_default(), 8)?, + trace_state: link["trace_state"].as_str().unwrap_or_default().into(), + attributes: object_attributes(&link["attributes"])?, + ..Default::default() + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn test_event_timestamp_precision() { + for (time, expected) in [ + ("1970-01-01 00:00:01+0000", 1_000_000_000), + ("1970-01-01 00:00:01.123+0000", 1_123_000_000), + ("1970-01-01 00:00:01.123456+0000", 1_123_456_000), + ("1970-01-01 00:00:01.123456789+0000", 1_123_456_789), + ] { + let value = json!([{"name":"event", "time":time, "attributes":{}}]); + let events = events(Some(&value)).unwrap(); + assert_eq!(events[0].time_unix_nano, expected); + } + } +} diff --git a/src/servers/src/http/jaeger/v3/summary.rs b/src/servers/src/http/jaeger/v3/summary.rs new file mode 100644 index 00000000000..14273c3a705 --- /dev/null +++ b/src/servers/src/http/jaeger/v3/summary.rs @@ -0,0 +1,142 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::{BTreeMap, HashSet}; + +use opentelemetry_proto::tonic::common::v1::any_value; +use opentelemetry_proto::tonic::trace::v1::status::StatusCode; +use opentelemetry_proto::tonic::trace::v1::{Span, TracesData}; +use serde_json::{Value, json}; + +use crate::otlp::trace::KEY_SERVICE_NAME; + +pub fn summarize(traces: TracesData) -> Vec { + let mut by_trace: BTreeMap<&[u8], Vec<(&str, &Span)>> = BTreeMap::new(); + for resource in &traces.resource_spans { + let service = resource + .resource + .as_ref() + .and_then(|resource| { + resource + .attributes + .iter() + .find(|attr| attr.key == KEY_SERVICE_NAME) + .and_then(|attr| attr.value.as_ref()) + .and_then(|value| match &value.value { + Some(any_value::Value::StringValue(name)) => Some(name.as_str()), + _ => None, + }) + }) + .unwrap_or_default(); + for scope in &resource.scope_spans { + for span in &scope.spans { + by_trace + .entry(&span.trace_id) + .or_default() + .push((service, span)); + } + } + } + by_trace + .into_iter() + .map(|(trace_id, spans)| { + let ids: HashSet<_> = spans + .iter() + .map(|(_, span)| span.span_id.as_slice()) + .collect(); + let mut services: BTreeMap<&str, (usize, usize)> = BTreeMap::new(); + let mut min_start = u64::MAX; + let mut max_end = 0; + let mut errors = 0; + let mut orphans = 0; + let mut root: Option<(&str, &Span)> = None; + for &(service, span) in &spans { + let stats = services.entry(service).or_default(); + stats.0 += 1; + if span + .status + .as_ref() + .is_some_and(|status| status.code == StatusCode::Error as i32) + { + stats.1 += 1; + errors += 1; + } + min_start = min_start.min(span.start_time_unix_nano); + max_end = max_end.max(span.end_time_unix_nano); + if span.parent_span_id.iter().all(|byte| *byte == 0) { + if root.is_none_or(|(_, root)| { + span.start_time_unix_nano < root.start_time_unix_nano + }) { + root = Some((service, span)); + } + } else if !ids.contains(span.parent_span_id.as_slice()) { + orphans += 1; + } + } + let (root_service, root_operation) = + root.map_or(("", ""), |(service, span)| (service, span.name.as_str())); + let services: Vec<_> = services + .into_iter() + .map(|(name, (spans, errors))| { + json!({ + "name": name, "spanCount": spans, "errorSpanCount": errors, + }) + }) + .collect(); + json!({ + "traceId": hex::encode(trace_id), + "rootServiceName": root_service, + "rootOperationName": root_operation, + "minStartTimeUnixNano": min_start.to_string(), + "maxEndTimeUnixNano": max_end.to_string(), + "spanCount": spans.len(), + "errorSpanCount": errors, + "orphanSpanCount": orphans, + "services": services, + }) + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_summary_roots_and_orphans() { + let traces: TracesData = serde_json::from_value(json!({"resourceSpans":[ + {"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"b"}}]},"scopeSpans":[{"spans":[ + {"traceId":"00000000000000000000000000000001","spanId":"0000000000000001","name":"late-root","startTimeUnixNano":"20","endTimeUnixNano":"50"}, + {"traceId":"00000000000000000000000000000001","spanId":"0000000000000002","parentSpanId":"0000000000000099","name":"orphan","startTimeUnixNano":"1","endTimeUnixNano":"60","status":{"code":2}}, + {"traceId":"00000000000000000000000000000002","spanId":"0000000000000003","parentSpanId":"0000000000000001","startTimeUnixNano":"1","endTimeUnixNano":"2"} + ]}]}, + {"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"a"}}]},"scopeSpans":[{"spans":[ + {"traceId":"00000000000000000000000000000001","spanId":"0000000000000004","parentSpanId":"0000000000000000","name":"early-root","startTimeUnixNano":"10","endTimeUnixNano":"20"}, + {"traceId":"00000000000000000000000000000001","spanId":"0000000000000005","parentSpanId":"0000000000000004","startTimeUnixNano":"11","endTimeUnixNano":"15","status":{"code":2}} + ]}]} + ]})).unwrap(); + let summaries = summarize(traces); + assert_eq!( + summaries[0], + json!({ + "traceId":"00000000000000000000000000000001", "rootServiceName":"a", "rootOperationName":"early-root", + "minStartTimeUnixNano":"1", "maxEndTimeUnixNano":"60", "spanCount":4, "errorSpanCount":2, "orphanSpanCount":1, + "services":[{"name":"a","spanCount":2,"errorSpanCount":1},{"name":"b","spanCount":2,"errorSpanCount":1}] + }) + ); + assert_eq!(summaries[1]["orphanSpanCount"], 1); + assert_eq!(summaries[1]["rootServiceName"], ""); + assert_eq!(summaries[1]["rootOperationName"], ""); + } +} diff --git a/tests-integration/src/tests/instance_test.rs b/tests-integration/src/tests/instance_test.rs index 89706b53764..1a9cc1790f7 100644 --- a/tests-integration/src/tests/instance_test.rs +++ b/tests-integration/src/tests/instance_test.rs @@ -415,6 +415,125 @@ LEFT JOIN (SELECT k, count(*) n FROM join_repro WHERE kind = 'child' AND k < '8' } } +/// Regression test for `MatchesConstantTermOptimizer` dropping `FilterExec`'s `fetch`. +/// +/// On a single-partition plan the `LIMIT` is enforced by the `fetch` that the physical +/// `LimitPushdown` rule leaves on the `FilterExec`, so a rule rebuilding that `FilterExec` +/// has to carry the `fetch` over. While it does not, the limit is silently lifted for +/// `matches_term` filters and +/// `SELECT count(*) FROM (SELECT ts, msg FROM t WHERE matches_term(msg, 'foo') LIMIT 1) s` +/// returns the number of matching rows (2) instead of 1. +/// +/// `query_parallelism=4` is expected to stay correct, because the limit is then materialized +/// above the scan instead of being delegated to the `FilterExec`; the test pins down that +/// contrast. The metric-engine logical table covers the same filter on a shared physical +/// region. +/// +/// Both matching rows share `ts` and `msg`, so the limited output does not depend on the +/// order in which the scan returns them. +#[apply(both_instances_cases)] +async fn test_matches_term_subquery_limit_count(instance: Arc) { + let frontend = instance.frontend(); + + execute_sql( + &frontend, + r#" +CREATE TABLE f07_mito_msgs ( + ts TIMESTAMP TIME INDEX, + host STRING PRIMARY KEY, + msg STRING, + val DOUBLE +) +ENGINE = mito +"#, + ) + .await; + execute_sql( + &frontend, + r#"INSERT INTO f07_mito_msgs (ts, host, msg, val) VALUES + (0, 'host1', 'foo bar', 1.0), + (0, 'host2', 'foo bar', 2.0), + (0, 'host3', 'other baz', 3.0)"#, + ) + .await; + + execute_sql( + &frontend, + r#" +CREATE TABLE f07_metric_phy ( + ts TIMESTAMP TIME INDEX, + val DOUBLE, + host STRING, + msg STRING, + PRIMARY KEY (host, msg) +) +ENGINE = metric +WITH ('physical_metric_table' = '') +"#, + ) + .await; + execute_sql( + &frontend, + r#" +CREATE TABLE f07_metric_msgs ( + ts TIMESTAMP TIME INDEX, + val DOUBLE, + host STRING, + msg STRING, + PRIMARY KEY (host, msg) +) +ENGINE = metric +WITH ('on_physical_table' = 'f07_metric_phy') +"#, + ) + .await; + execute_sql( + &frontend, + r#"INSERT INTO f07_metric_msgs (ts, host, msg, val) VALUES + (0, 'host1', 'foo bar', 1.0), + (0, 'host2', 'foo bar', 2.0), + (0, 'host3', 'other baz', 3.0)"#, + ) + .await; + + for table in ["f07_mito_msgs", "f07_metric_msgs"] { + let limited = format!("SELECT ts, msg FROM {table} WHERE matches_term(msg, 'foo') LIMIT 1"); + let count = format!("SELECT count(*) FROM ({limited}) s"); + + let limit_rows = execute_sql_with_query_parallelism(&frontend, &limited, 1) + .await + .data + .pretty_print() + .await; + assert_eq!( + limit_rows, + r#"+---------------------+---------+ +| ts | msg | ++---------------------+---------+ +| 1970-01-01T00:00:00 | foo bar | ++---------------------+---------+"#, + "query_parallelism=1: `{limited}` must return exactly the single limited row" + ); + + for parallelism in [1, 4] { + let count_rows = execute_sql_with_query_parallelism(&frontend, &count, parallelism) + .await + .data + .pretty_print() + .await; + assert_eq!( + count_rows, + r#"+----------+ +| count(*) | ++----------+ +| 1 | ++----------+"#, + "query_parallelism={parallelism}: the LIMIT 1 of the subquery must cap `{count}` to one row" + ); + } + } +} + #[apply(both_instances_cases)] async fn test_partitioned_merge_scan_semi_join_repartitions(instance: Arc) { let frontend = instance.frontend(); diff --git a/tests-integration/tests/export_logical_tables.rs b/tests-integration/tests/export_logical_tables.rs index 8e3057bb7aa..13f9384cb5c 100644 --- a/tests-integration/tests/export_logical_tables.rs +++ b/tests-integration/tests/export_logical_tables.rs @@ -1426,6 +1426,7 @@ struct ImportRequests { async fn import_http( instance: Arc, older: bool, + packed: bool, ) -> (String, tokio::task::JoinHandle<()>, Arc) { use axum::response::IntoResponse; use std::sync::atomic::Ordering; @@ -1446,6 +1447,12 @@ async fn import_http( if request.uri().path() == "/v1/capabilities" { observed.capabilities.fetch_add(1, Ordering::SeqCst); if older { + if packed { + return axum::Json(serde_json::json!({ + "metric_packed_import": 1 + })) + .into_response(); + } return axum::http::StatusCode::NOT_FOUND.into_response(); } } @@ -1528,6 +1535,11 @@ async fn packed_export_v2_cli_local_roundtrip() { metric_export_v2_cli_roundtrip(false, true, false).await; } +#[tokio::test(flavor = "multi_thread")] +async fn packed_import_v2_without_batch_ddl_roundtrip() { + metric_export_v2_cli_roundtrip(false, true, true).await; +} + #[tokio::test(flavor = "multi_thread")] async fn packed_export_v2_cli_s3_roundtrip() { if std::env::var("GT_S3_ENDPOINT_URL").is_ok_and(|e| !e.is_empty()) @@ -1606,7 +1618,7 @@ async fn metric_export_v2_cli_roundtrip(s3: bool, packed: bool, older: bool) { &format!("CREATE VIEW \"{quoted_special}\" AS SELECT * FROM audit"), ) .await; - if packed && !s3 { + if packed && !s3 && !older { for id in 0..128 { let name = format!("batch_{id:03}"); sql(instance, &format!("CREATE TABLE {name} (host STRING PRIMARY KEY, val DOUBLE, ts TIMESTAMP TIME INDEX) ENGINE=metric WITH (on_physical_table='v2_a')")).await; @@ -1630,7 +1642,7 @@ async fn metric_export_v2_cli_roundtrip(s3: bool, packed: bool, older: bool) { "CREATE TABLE bulk (host STRING PRIMARY KEY, payload STRING, ts TIMESTAMP TIME INDEX)", ) .await; - for batch in 0..64 { + for batch in 0..96 { let rows = (0..64) .map(|row| { let payload = (0..128) @@ -1813,7 +1825,7 @@ async fn metric_export_v2_cli_roundtrip(s3: bool, packed: bool, older: bool) { .await .unwrap() .content_length() - > 5 * 1024 * 1024 + > common_datasource::packed_writer::WRITE_BYTES as u64 ); } let mut preserved = Vec::new(); @@ -1923,7 +1935,7 @@ async fn metric_export_v2_cli_roundtrip(s3: bool, packed: bool, older: bool) { .await; } let (target_addr, target_server, requests) = - import_http(target.fe_instance().clone(), older).await; + import_http(target.fe_instance().clone(), older, packed).await; let state = destination.path().join("restore-state.json"); let mut import = vec![ "import-v2", @@ -1959,7 +1971,11 @@ async fn metric_export_v2_cli_roundtrip(s3: bool, packed: bool, older: bool) { requests.capabilities.load(Ordering::SeqCst), if older { 1 } else { 2 } ); - assert_eq!(requests.batches.load(Ordering::SeqCst) > 1, !older); + if older { + assert_eq!(requests.batches.load(Ordering::SeqCst), 0); + } else { + assert!(requests.batches.load(Ordering::SeqCst) > 1); + } assert!(!state.exists()); for name in names { let source_table = table(instance, &name).await; @@ -2003,7 +2019,7 @@ async fn metric_export_v2_cli_roundtrip(s3: bool, packed: bool, older: bool) { .column_schemas() ); } - if packed { + if packed && !older { let schema_uri = format!("{uri}/schema-only"); let mut schema_args = vec![ "export-v2", @@ -2039,7 +2055,7 @@ async fn metric_export_v2_cli_roundtrip(s3: bool, packed: bool, older: bool) { .build() .await; let (schema_addr, schema_server, schema_requests) = - import_http(schema_target.fe_instance().clone(), false).await; + import_http(schema_target.fe_instance().clone(), false, false).await; schema_requests.fail_batch.store(false, Ordering::SeqCst); let schema_state = destination.path().join("schema-only-state.json"); let mut schema_import = vec![ diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 9f0973a5fcb..d71bd06555d 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -227,6 +227,7 @@ macro_rules! http_tests { test_splunk_raw, test_log_query, test_jaeger_query_api, + test_jaeger_v3_query_api, test_jaeger_query_api_for_trace_v1, test_jaeger_query_api_for_trace_v2, @@ -11494,6 +11495,324 @@ pub async fn test_log_query(store_type: StorageType) { guard.remove_all().await; } +pub async fn test_jaeger_v3_query_api( + store_type: StorageType, +) -> Result<(), Box> { + let (app, mut guard) = + setup_test_http_app_with_frontend(store_type, "test_jaeger_v3_query_api").await; + let client = TestClient::new(app).await; + for (path, expected_status, expected) in [ + ("services", StatusCode::OK, json!({"services":[]})), + ( + "trace-summaries?query.startTimeMin=2026-01-01T00:00:00Z&query.startTimeMax=2026-01-02T00:00:00Z", + StatusCode::OK, + json!({"summaries":[]}), + ), + ( + "operations?service=checkout", + StatusCode::OK, + json!({"operations":[]}), + ), + ( + "traces/00000000000000000000000000000001", + StatusCode::NOT_FOUND, + json!({"error":{"httpCode":404,"message":"No traces found"}}), + ), + ] { + let response = client + .get(&format!("/v1/jaeger/api/v3/{path}")) + .send() + .await; + assert_eq!(response.status(), expected_status, "{path}"); + assert_eq!( + serde_json::from_str::(&response.text().await)?, + expected + ); + } + for path in [ + "operations", + "traces", + "traces/not-hex", + "traces/0000000000000001?startTime=invalid", + "traces/0000000000000001?rawTraces=invalid", + "traces?query.startTimeMin=2026-01-01T00:00:00Z&query.startTimeMax=2026-01-02T00:00:00Z&query.searchDepth=10001", + "trace-summaries?query.startTimeMin=2026-01-01T00:00:00Z&query.startTimeMax=2026-01-02T00:00:00Z&query.searchDepth=10001", + ] { + let response = client + .get(&format!("/v1/jaeger/api/v3/{path}")) + .send() + .await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{path}"); + let body: Value = serde_json::from_str(&response.text().await)?; + assert_eq!(body["error"]["httpCode"], 400); + } + + // The parent starts before the search window. A matching child must return the whole trace. + let start = chrono::Utc::now() + .timestamp_nanos_opt() + .ok_or("Timestamp out of range")? as u64; + let start = start / 1_000_000_000 * 1_000_000_000 + 123; + let timestamp = |ns: u64| { + chrono::DateTime::from_timestamp_nanos(ns as i64) + .to_rfc3339_opts(chrono::SecondsFormat::Nanos, true) + }; + let request: ExportTraceServiceRequest = serde_json::from_value(json!({"resourceSpans":[ + {"resource":{"attributes":[make_string_attr("service.name","checkout"), make_string_attr("service.instance.id","one")]}, + "scopeSpans":[{"scope":{"name":"sdk","version":"1.0","attributes":[make_string_attr("scope.key","value"), {"key":"scope.bytes","value":{"bytesValue":"AQID/w=="}}]},"spans":[{ + "traceId":"000000000000000000000000000000ab", "spanId":"0000000000000001", "name":"parent", "kind":2, + "startTimeUnixNano":start.to_string(), "endTimeUnixNano":(start+2000).to_string() + }]}]}, + {"resource":{"attributes":[make_string_attr("service.name","checkout"), make_string_attr("service.instance.id","two"), {"key":"resource.bytes","value":{"bytesValue":"AQID/w=="}}]}, + "scopeSpans":[{"scope":{"name":"sdk","version":"1.0","attributes":[make_string_attr("scope.key","value"), {"key":"scope.bytes","value":{"bytesValue":"AQID/w=="}}]},"spans":[{ + "traceId":"000000000000000000000000000000ab", "spanId":"0000000000000002", "parentSpanId":"0000000000000001", "name":"child", "kind":3, + "traceState":"vendor=value", "startTimeUnixNano":(start+1000).to_string(), "endTimeUnixNano":(start+1999).to_string(), + "attributes":[make_int_attr("http.status_code",500),make_bool_attr("failed",true), + {"key":"bytes","value":{"bytesValue":"AQID/w=="}}, + {"key":"integers","value":{"arrayValue":{"values":[{"intValue":"1"},{"intValue":"2"},{"intValue":"3"},{"intValue":"255"}]}}}, + {"key":"nested","value":{"arrayValue":{"values":[{"intValue":"9223372036854775807"},{"kvlistValue":{"values":[make_string_attr("key","value")]}}]}}}], + "events":[{"name":"exception","timeUnixNano":(start+1011).to_string(),"attributes":[make_string_attr("message","failure")]}], + "links":[{"traceId":"000000000000000000000000000000cd","spanId":"0000000000000003","traceState":"link=value","attributes":[make_bool_attr("remote",true)]}], + "status":{"code":2,"message":"failure"} + }, { + "traceId":"000000000000000000000000000000cd", "spanId":"0000000000000003", "name":"boundary", "kind":2, + "startTimeUnixNano":(start+2000).to_string(), "endTimeUnixNano":(start+3000).to_string() + }]}]} + ]}))?; + for pipeline in [ + "greptime_trace_v0", + GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME, + GREPTIME_INTERNAL_TRACE_PIPELINE_V2_NAME, + ] { + let table = format!("jaeger_api_v3_{pipeline}"); + let response = send_req( + &client, + vec![ + ( + HeaderName::from_static("content-type"), + HeaderValue::from_static("application/x-protobuf"), + ), + ( + HeaderName::from_static("x-greptime-pipeline-name"), + HeaderValue::from_str(pipeline)?, + ), + ( + HeaderName::from_static("x-greptime-trace-table-name"), + HeaderValue::from_str(&table)?, + ), + ], + "/v1/otlp/v1/traces", + request.encode_to_vec(), + false, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + for (path, expected) in [ + ("services", json!({"services":["checkout"]})), + ( + "operations?service=checkout&span_kind=client", + json!({"operations":[{"name":"child","spanKind":"client"}]}), + ), + ( + "operations?service=checkout&spanKind=client", + json!({"operations":[{"name":"child","spanKind":"client"}]}), + ), + ] { + let response = client + .get(&format!("/v1/jaeger/api/v3/{path}")) + .header("x-greptime-trace-table-name", &table) + .send() + .await; + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + serde_json::from_str::(&response.text().await)?, + expected + ); + } + for flushed in [false, true] { + if flushed { + let response = client + .post("/v1/sql") + .form(&[("sql", format!("ADMIN FLUSH_TABLE('{table}')"))]) + .send() + .await; + assert_eq!(response.status(), StatusCode::OK); + } + let response = client.get(&format!( + "/v1/jaeger/api/v3/trace-summaries?query.serviceName=checkout&query.startTimeMin={}&query.startTimeMax={}", + timestamp(start+1000), timestamp(start+2000), + )).header("x-greptime-trace-table-name", &table).send().await; + assert_eq!(response.status(), StatusCode::OK); + let body: Value = serde_json::from_str(&response.text().await)?; + assert_eq!( + body, + json!({"summaries":[{ + "traceId":"000000000000000000000000000000ab", + "rootServiceName":"checkout", "rootOperationName":"parent", + "minStartTimeUnixNano":start.to_string(), "maxEndTimeUnixNano":(start+2000).to_string(), + "spanCount":2, "errorSpanCount":1, "orphanSpanCount":0, + "services":[{"name":"checkout", "spanCount":2, "errorSpanCount":1}], + }]}) + ); + for path in [ + "traces/00000000000000AB".to_string(), + format!( + "traces/00000000000000AB?startTime={}", + timestamp(start + 1000) + ), + format!("traces/00000000000000AB?endTime={}", timestamp(start + 500)), + format!( + "traces/00000000000000AB?start_time={}&end_time={}", + timestamp(start + 500), + timestamp(start + 999) + ), + format!( + "traces?query.startTimeMin={}&query.startTimeMax={}&query.searchDepth=0", + timestamp(start + 1000), + timestamp(start + 2000) + ), + format!( + "traces?query.service_name=checkout&query.operation_name=child&query.start_time_min={}&query.start_time_max={}&query.num_traces=1&query.duration_min=999ns&query.duration_max=999ns&query.attributes={}", + timestamp(start + 1000), + timestamp(start + 2000), + encode(r#"{"http.status_code":"500","failed":"true"}"#) + ), + format!( + "traces?query.startTimeMin={}&query.startTimeMax={}&query.attributes={}", + timestamp(start + 1000), + timestamp(start + 2000), + encode(r#"{"service.instance.id":"two"}"#) + ), + ] { + let response = client + .get(&format!("/v1/jaeger/api/v3/{path}")) + .header("x-greptime-trace-table-name", &table) + .send() + .await; + let status = response.status(); + let body: Value = serde_json::from_str(&response.text().await)?; + assert_eq!(status, StatusCode::OK, "{pipeline}: {path}: {body}"); + let resources = body["result"]["resourceSpans"] + .as_array() + .ok_or("Missing resource spans")?; + let spans: Vec<_> = resources + .iter() + .flat_map(|r| r["scopeSpans"][0]["spans"].as_array().unwrap()) + .collect(); + assert_eq!(spans.len(), 2, "{body}"); + assert!( + spans + .iter() + .all(|span| span["traceId"] == "000000000000000000000000000000ab") + ); + let child = spans + .iter() + .find(|span| span["name"] == "child") + .ok_or("Missing child span")?; + assert_eq!(child["startTimeUnixNano"], (start + 1000).to_string()); + assert_eq!(child["endTimeUnixNano"], (start + 1999).to_string()); + assert_eq!(child["parentSpanId"], "0000000000000001"); + assert_eq!(child["kind"], 3); + assert_eq!(child["status"]["code"], 2); + assert_eq!(child["traceState"], "vendor=value"); + assert_eq!( + child["events"][0]["timeUnixNano"], + (start + 1011).to_string() + ); + assert_eq!(child["links"][0]["traceState"], "link=value"); + assert_eq!( + child["links"][0]["attributes"][0]["value"]["boolValue"], + true + ); + let nested = child["attributes"] + .as_array() + .unwrap() + .iter() + .find(|attr| attr["key"] == "nested") + .unwrap(); + assert_eq!( + nested["value"]["arrayValue"]["values"][0]["intValue"], + "9223372036854775807" + ); + let attribute = |values: &Value, key: &str| { + values + .as_array() + .unwrap() + .iter() + .find(|attr| attr["key"] == key) + .unwrap()["value"] + .clone() + }; + assert_eq!( + attribute(&child["attributes"], "integers"), + json!({"arrayValue":{"values":[{"intValue":"1"},{"intValue":"2"},{"intValue":"3"},{"intValue":"255"}]}}) + ); + if pipeline == GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME { + assert_eq!( + attribute(&child["attributes"], "bytes"), + json!({"bytesValue":"AQID/w=="}) + ); + } + for resource in resources { + let span = &resource["scopeSpans"][0]["spans"][0]; + let attrs = resource["resource"]["attributes"].as_array().unwrap(); + let instance = attrs + .iter() + .find(|attr| attr["key"] == "service.instance.id") + .unwrap(); + assert_eq!( + instance["value"]["stringValue"], + if span["name"] == "parent" { + "one" + } else { + "two" + } + ); + let scope_attrs = &resource["scopeSpans"][0]["scope"]["attributes"]; + assert_eq!( + attribute(scope_attrs, "scope.key"), + json!({"stringValue":"value"}) + ); + if pipeline == GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME { + assert_eq!( + attribute(scope_attrs, "scope.bytes"), + json!({"bytesValue":"AQID/w=="}) + ); + if span["name"] == "child" { + assert_eq!( + attribute(&resource["resource"]["attributes"], "resource.bytes"), + json!({"bytesValue":"AQID/w=="}) + ); + } + } + } + } + // Instance `one` is on the parent, which starts before the search window. + let response = client + .get(&format!( + "/v1/jaeger/api/v3/traces?query.startTimeMin={}&query.startTimeMax={}&query.attributes={}", + timestamp(start + 1000), + timestamp(start + 2000), + encode(r#"{"service.instance.id":"one"}"#) + )) + .header("x-greptime-trace-table-name", &table) + .send() + .await; + assert_eq!(response.status(), StatusCode::NOT_FOUND, "{pipeline}"); + } + // Legacy clients continue receiving Jaeger JSON at the old URL. + let response = client + .get("/v1/jaeger/api/traces/000000000000000000000000000000ab") + .header("x-greptime-trace-table-name", &table) + .send() + .await; + assert_eq!(response.status(), StatusCode::OK); + let body: Value = serde_json::from_str(&response.text().await)?; + assert_eq!(body["data"][0]["spans"].as_array().unwrap().len(), 2); + } + guard.remove_all().await; + Ok(()) +} + pub async fn test_jaeger_query_api(store_type: StorageType) { common_telemetry::init_default_ut_logging(); let (app, mut guard) = diff --git a/tests/cases/distributed/explain/step_aggr_advance.result b/tests/cases/distributed/explain/step_aggr_advance.result index 348021360ff..11adf22f866 100644 --- a/tests/cases/distributed/explain/step_aggr_advance.result +++ b/tests/cases/distributed/explain/step_aggr_advance.result @@ -21,26 +21,26 @@ Affected Rows: 0 -- SQLNESS REPLACE (Hash.*) REDACTED tql explain (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr_optimize_not [2m])); -+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| plan_type | plan | -+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| logical_plan | MergeSort: aggr_optimize_not.a ASC NULLS LAST, aggr_optimize_not.b ASC NULLS LAST, aggr_optimize_not.c ASC NULLS LAST, aggr_optimize_not.greptime_timestamp ASC NULLS LAST | -| | MergeScan [is_placeholder=false, remote_input=[ | -| | Sort: aggr_optimize_not.a ASC NULLS LAST, aggr_optimize_not.b ASC NULLS LAST, aggr_optimize_not.c ASC NULLS LAST, aggr_optimize_not.greptime_timestamp ASC NULLS LAST | -| | Aggregate: groupBy=[[aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.greptime_timestamp]], aggr=[[max(prom_max_over_time(greptime_timestamp_range,greptime_value))]] | -| | Filter: prom_max_over_time(greptime_timestamp_range,greptime_value) IS NOT NULL | -| | Projection: aggr_optimize_not.greptime_timestamp, prom_max_over_time(greptime_timestamp_range, greptime_value) AS prom_max_over_time(greptime_timestamp_range,greptime_value), aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d | -| | PromRangeManipulate: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp], values=["greptime_value"] | -| | PromSeriesNormalize: offset=[0], time index=[greptime_timestamp], filter NaN: [true] | -| | PromSeriesDivide: tags=["a", "b", "c", "d"] | -| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST | -| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) | -| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] | -| | ]] | -| physical_plan | MergeSortExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] | ++---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | MergeSort: aggr_optimize_not.a ASC NULLS LAST, aggr_optimize_not.b ASC NULLS LAST, aggr_optimize_not.c ASC NULLS LAST, aggr_optimize_not.greptime_timestamp ASC NULLS LAST | +| | MergeScan [is_placeholder=false, remote_input=[ | +| | Sort: aggr_optimize_not.a ASC NULLS LAST, aggr_optimize_not.b ASC NULLS LAST, aggr_optimize_not.c ASC NULLS LAST, aggr_optimize_not.greptime_timestamp ASC NULLS LAST | +| | Aggregate: groupBy=[[aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.greptime_timestamp]], aggr=[[prom_max(prom_max_over_time(greptime_timestamp_range,greptime_value)) AS max(prom_max_over_time(greptime_timestamp_range,greptime_value))]] | +| | Filter: prom_max_over_time(greptime_timestamp_range,greptime_value) IS NOT NULL | +| | Projection: aggr_optimize_not.greptime_timestamp, prom_max_over_time(greptime_timestamp_range, greptime_value) AS prom_max_over_time(greptime_timestamp_range,greptime_value), aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d | +| | PromRangeManipulate: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp], values=["greptime_value"] | +| | PromSeriesNormalize: offset=[0], time index=[greptime_timestamp], filter NaN: [true] | +| | PromSeriesDivide: tags=["a", "b", "c", "d"] | +| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST | +| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) | +| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] | +| | ]] | +| physical_plan | MergeSortExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] | | | MergeScanExec: REDACTED -| | | -+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| | | ++---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED @@ -57,11 +57,12 @@ tql analyze (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr | 0_| 0_|_MergeSortExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| -| 1_| 0_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED +| 1_| 0_|_ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp, prom_max(prom_max_over_time(greptime_timestamp_range,greptime_value))@4 as max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +|_|_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp], aggr=[max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp], aggr=[prom_max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[a@2 as a, b@3 as b, c@4 as c, greptime_timestamp@0 as greptime_timestamp], aggr=[max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[a@2 as a, b@3 as b, c@4 as c, greptime_timestamp@0 as greptime_timestamp], aggr=[prom_max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_FilterExec: prom_max_over_time(greptime_timestamp_range,greptime_value)@1 IS NOT NULL REDACTED |_|_|_ProjectionExec: expr=[greptime_timestamp@4 as greptime_timestamp, prom_max_over_time(greptime_timestamp_range@6, greptime_value@5) as prom_max_over_time(greptime_timestamp_range,greptime_value), a@0 as a, b@1 as b, c@2 as c] REDACTED |_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED @@ -69,11 +70,12 @@ tql analyze (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED |_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED |_|_|_| -| 1_| 1_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED +| 1_| 1_|_ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp, prom_max(prom_max_over_time(greptime_timestamp_range,greptime_value))@4 as max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +|_|_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp], aggr=[max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp], aggr=[prom_max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[a@2 as a, b@3 as b, c@4 as c, greptime_timestamp@0 as greptime_timestamp], aggr=[max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[a@2 as a, b@3 as b, c@4 as c, greptime_timestamp@0 as greptime_timestamp], aggr=[prom_max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_FilterExec: prom_max_over_time(greptime_timestamp_range,greptime_value)@1 IS NOT NULL REDACTED |_|_|_ProjectionExec: expr=[greptime_timestamp@4 as greptime_timestamp, prom_max_over_time(greptime_timestamp_range@6, greptime_value@5) as prom_max_over_time(greptime_timestamp_range,greptime_value), a@0 as a, b@1 as b, c@2 as c] REDACTED |_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED @@ -305,31 +307,31 @@ tql analyze (1752591864, 1752592164, '30s') count by (a, b, c, d) (max_over_time -- SQLNESS REPLACE (Hash.*) REDACTED tql explain (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr_optimize_not [2m])); -+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| plan_type | plan | -+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| logical_plan | Sort: aggr_optimize_not.b ASC NULLS LAST, aggr_optimize_not.c ASC NULLS LAST, aggr_optimize_not.d ASC NULLS LAST, aggr_optimize_not.greptime_timestamp ASC NULLS LAST | -| | Aggregate: groupBy=[[aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d, aggr_optimize_not.greptime_timestamp]], aggr=[[__min_merge(__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) AS min(prom_max_over_time(greptime_timestamp_range,greptime_value))]] | -| | MergeScan [is_placeholder=false, remote_input=[ | -| | Aggregate: groupBy=[[aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d, aggr_optimize_not.greptime_timestamp]], aggr=[[__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))]] | -| | Filter: prom_max_over_time(greptime_timestamp_range,greptime_value) IS NOT NULL | -| | Projection: aggr_optimize_not.greptime_timestamp, prom_max_over_time(greptime_timestamp_range, greptime_value) AS prom_max_over_time(greptime_timestamp_range,greptime_value), aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d | -| | PromRangeManipulate: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp], values=["greptime_value"] | -| | PromSeriesNormalize: offset=[0], time index=[greptime_timestamp], filter NaN: [true] | -| | PromSeriesDivide: tags=["a", "b", "c", "d"] | -| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST | -| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) | -| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] | -| | ]] | -| physical_plan | SortPreservingMergeExec: [b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] | -| | SortExec: expr=[b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] | -| | AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__min_merge(__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] | ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| plan_type | plan | ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| logical_plan | Sort: aggr_optimize_not.b ASC NULLS LAST, aggr_optimize_not.c ASC NULLS LAST, aggr_optimize_not.d ASC NULLS LAST, aggr_optimize_not.greptime_timestamp ASC NULLS LAST | +| | Aggregate: groupBy=[[aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d, aggr_optimize_not.greptime_timestamp]], aggr=[[__prom_min_merge(__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) AS min(prom_max_over_time(greptime_timestamp_range,greptime_value))]] | +| | MergeScan [is_placeholder=false, remote_input=[ | +| | Aggregate: groupBy=[[aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d, aggr_optimize_not.greptime_timestamp]], aggr=[[__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))]] | +| | Filter: prom_max_over_time(greptime_timestamp_range,greptime_value) IS NOT NULL | +| | Projection: aggr_optimize_not.greptime_timestamp, prom_max_over_time(greptime_timestamp_range, greptime_value) AS prom_max_over_time(greptime_timestamp_range,greptime_value), aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d | +| | PromRangeManipulate: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp], values=["greptime_value"] | +| | PromSeriesNormalize: offset=[0], time index=[greptime_timestamp], filter NaN: [true] | +| | PromSeriesDivide: tags=["a", "b", "c", "d"] | +| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST | +| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) | +| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] | +| | ]] | +| physical_plan | SortPreservingMergeExec: [b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] | +| | SortExec: expr=[b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] | +| | AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__prom_min_merge(__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] | | | RepartitionExec: partitioning=REDACTED -| | AggregateExec: mode=Partial, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__min_merge(__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] | +| | AggregateExec: mode=Partial, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__prom_min_merge(__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] | | | RepartitionExec: partitioning=REDACTED | | MergeScanExec: REDACTED -| | | -+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| | | ++---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -- SQLNESS REPLACE (metrics.*) REDACTED -- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED @@ -345,15 +347,15 @@ tql analyze (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr +-+-+-+ | 0_| 0_|_SortPreservingMergeExec: [b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED |_|_|_SortExec: expr=[b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] REDACTED -|_|_|_AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__min_merge(__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +|_|_|_AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__prom_min_merge(__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__min_merge(__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__prom_min_merge(__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED |_|_|_MergeScanExec: REDACTED |_|_|_| -| 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +| 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[b@2 as b, c@3 as c, d@4 as d, greptime_timestamp@0 as greptime_timestamp], aggr=[__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[b@2 as b, c@3 as c, d@4 as d, greptime_timestamp@0 as greptime_timestamp], aggr=[__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_FilterExec: prom_max_over_time(greptime_timestamp_range,greptime_value)@1 IS NOT NULL REDACTED |_|_|_ProjectionExec: expr=[greptime_timestamp@4 as greptime_timestamp, prom_max_over_time(greptime_timestamp_range@6, greptime_value@5) as prom_max_over_time(greptime_timestamp_range,greptime_value), b@1 as b, c@2 as c, d@3 as d] REDACTED |_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED @@ -361,9 +363,9 @@ tql analyze (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr |_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED |_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED |_|_|_| -| 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +| 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_RepartitionExec: partitioning=REDACTED -|_|_|_AggregateExec: mode=Partial, gby=[b@2 as b, c@3 as c, d@4 as d, greptime_timestamp@0 as greptime_timestamp], aggr=[__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED +|_|_|_AggregateExec: mode=Partial, gby=[b@2 as b, c@3 as c, d@4 as d, greptime_timestamp@0 as greptime_timestamp], aggr=[__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED |_|_|_FilterExec: prom_max_over_time(greptime_timestamp_range,greptime_value)@1 IS NOT NULL REDACTED |_|_|_ProjectionExec: expr=[greptime_timestamp@4 as greptime_timestamp, prom_max_over_time(greptime_timestamp_range@6, greptime_value@5) as prom_max_over_time(greptime_timestamp_range,greptime_value), b@1 as b, c@2 as c, d@3 as d] REDACTED |_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED diff --git a/tests/cases/standalone/common/promql/count_values.result b/tests/cases/standalone/common/promql/count_values.result index 886c3a143b7..1d04301d73b 100644 --- a/tests/cases/standalone/common/promql/count_values.result +++ b/tests/cases/standalone/common/promql/count_values.result @@ -84,7 +84,7 @@ TQL EVAL (0, 15, '5s') abs(count_values("v", http_requests)); TQL EVAL (0, 15, '5s') round(count_values("v", http_requests)); +---------------------+-------------------------------------------------+-----+ -| ts | prom_round(count(http_requests.val),Float64(0)) | v | +| ts | prom_round(count(http_requests.val),Float64(1)) | v | +---------------------+-------------------------------------------------+-----+ | 1970-01-01T00:00:00 | 1.0 | 401 | | 1970-01-01T00:00:00 | 3.0 | 200 | diff --git a/tests/cases/standalone/common/promql/delta_temporality.result b/tests/cases/standalone/common/promql/delta_temporality.result index 787e72e8dc8..ac32a907cb9 100644 --- a/tests/cases/standalone/common/promql/delta_temporality.result +++ b/tests/cases/standalone/common/promql/delta_temporality.result @@ -162,7 +162,7 @@ TQL EVAL (180, 180, '1m') round(sum(rate(delta_temporality[3m])), 0.000001); +---------------------+----------------------------------------------------------------------------------------+ | ts | prom_round(sum(prom_rate(ts_range,greptime_value,ts,Int64(180000))),Float64(0.000001)) | +---------------------+----------------------------------------------------------------------------------------+ -| 1970-01-01T00:03:00 | 0.45555599999999996 | +| 1970-01-01T00:03:00 | 0.455556 | +---------------------+----------------------------------------------------------------------------------------+ CREATE TABLE delta_marker_only ( diff --git a/tests/cases/standalone/common/promql/dotted_column_names.result b/tests/cases/standalone/common/promql/dotted_column_names.result index c855a5ba5c7..bb4912bc94e 100644 --- a/tests/cases/standalone/common/promql/dotted_column_names.result +++ b/tests/cases/standalone/common/promql/dotted_column_names.result @@ -424,9 +424,9 @@ TQL EXPLAIN (0, 10, '5s') topk(1, {"otel.m"}); | plan_type_| plan_| +-+-+ | logical_plan_| Projection: otel.m.v.val, otel.m.service.name, otel.m.Host, otel.m.host, otel.m.ts.time_| -|_|_Sort: otel.m.ts.time ASC NULLS LAST, row_number() PARTITION BY [otel.m.ts.time] ORDER BY [otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ASC NULLS LAST_| -|_|_Filter: CAST(row_number() PARTITION BY [otel.m.ts.time] ORDER BY [otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS Float64) <= Float64(1)_| -|_|_WindowAggr: windowExpr=[[row_number() PARTITION BY [otel.m.ts.time] ORDER BY [otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]_| +|_|_Sort: otel.m.ts.time ASC NULLS LAST, row_number() PARTITION BY [otel.m.ts.time] ORDER BY [isnan(otel.m.v.val) ASC NULLS FIRST, otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ASC NULLS LAST_| +|_|_Filter: CAST(row_number() PARTITION BY [otel.m.ts.time] ORDER BY [isnan(otel.m.v.val) ASC NULLS FIRST, otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS Float64) <= Float64(1)_| +|_|_WindowAggr: windowExpr=[[row_number() PARTITION BY [otel.m.ts.time] ORDER BY [isnan(otel.m.v.val) ASC NULLS FIRST, otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]_| |_|_MergeScan [is_placeholder=false, remote_input=[_| |_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[ts.time]_| |_|_PromSeriesDivide: tags=["service.name", "Host", "host"]_| @@ -435,10 +435,10 @@ TQL EXPLAIN (0, 10, '5s') topk(1, {"otel.m"}); |_|_TableScan: otel.m, partial_filters=[otel.m.ts.time >= TimestampMillisecond(-299999, None), otel.m.ts.time <= TimestampMillisecond(10000, None)]_| |_| ]]_| | physical_plan | ProjectionExec: expr=[v.val@4 as v.val, service.name@1 as service.name, Host@2 as Host, host@3 as host, ts.time@0 as ts.time]_| -|_|_SortPreservingMergeExec: [ts.time@0 ASC NULLS LAST, row_number() PARTITION BY [otel.m.ts.time] ORDER BY [otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 ASC NULLS LAST]_| -|_|_FilterExec: CAST(row_number() PARTITION BY [otel.m.ts.time] ORDER BY [otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 AS Float64) <= 1_| -|_|_BoundedWindowAggExec: wdw=[row_number() PARTITION BY [otel.m.ts.time] ORDER BY [otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [otel.m.ts.time] ORDER BY [otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] | -|_|_SortExec: expr=[ts.time@0 ASC NULLS LAST, v.val@4 DESC, service.name@1 DESC, Host@2 DESC, host@3 DESC], preserve_REDACTED +|_|_SortPreservingMergeExec: [ts.time@0 ASC NULLS LAST, row_number() PARTITION BY [otel.m.ts.time] ORDER BY [isnan(otel.m.v.val) ASC NULLS FIRST, otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 ASC NULLS LAST]_| +|_|_FilterExec: CAST(row_number() PARTITION BY [otel.m.ts.time] ORDER BY [isnan(otel.m.v.val) ASC NULLS FIRST, otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 AS Float64) <= 1_| +|_|_BoundedWindowAggExec: wdw=[row_number() PARTITION BY [otel.m.ts.time] ORDER BY [isnan(otel.m.v.val) ASC NULLS FIRST, otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [otel.m.ts.time] ORDER BY [isnan(otel.m.v.val) ASC NULLS FIRST, otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] | +|_|_SortExec: expr=[ts.time@0 ASC NULLS LAST, isnan(v.val@4) ASC, v.val@4 DESC, service.name@1 DESC, Host@2 DESC, host@3 DESC], preserve_REDACTED |_|_RepartitionExec: REDACTED |_|_MergeScanExec: REDACTED |_|_| diff --git a/tests/cases/standalone/common/promql/float_semantics.result b/tests/cases/standalone/common/promql/float_semantics.result new file mode 100644 index 00000000000..e56db8b0ba6 --- /dev/null +++ b/tests/cases/standalone/common/promql/float_semantics.result @@ -0,0 +1,115 @@ +-- NaN and rounding follow Prometheus (IEEE 754) rather than SQL ordering. +-- Expected results are from Prometheus 3. +create table float_semantics ( + ts timestamp time index, + val double, + host string primary key +); + +Affected Rows: 0 + +insert into float_semantics values + (0, 1.0, 'a'), + (0, 2.0, 'b'), + (0, 'NaN'::double, 'c'), + (0, -2.5, 'd'); + +Affected Rows: 4 + +tql eval (0, 0, '1s') max(float_semantics); + ++---------------------+--------------------------+ +| ts | max(float_semantics.val) | ++---------------------+--------------------------+ +| 1970-01-01T00:00:00 | 2.0 | ++---------------------+--------------------------+ + +tql eval (0, 0, '1s') min(float_semantics); + ++---------------------+--------------------------+ +| ts | min(float_semantics.val) | ++---------------------+--------------------------+ +| 1970-01-01T00:00:00 | -2.5 | ++---------------------+--------------------------+ + +tql eval (0, 0, '1s') topk(1, float_semantics); + ++-----+------+---------------------+ +| val | host | ts | ++-----+------+---------------------+ +| 2.0 | b | 1970-01-01T00:00:00 | ++-----+------+---------------------+ + +-- SQLNESS SORT_RESULT 3 1 +tql eval (0, 0, '1s') float_semantics <= NaN; + ++----+-----+------+ +| ts | val | host | ++----+-----+------+ ++----+-----+------+ + +-- SQLNESS SORT_RESULT 3 1 +tql eval (0, 0, '1s') float_semantics != bool NaN; + ++------+---------------------+---------------------+ +| host | ts | val != Float64(NaN) | ++------+---------------------+---------------------+ +| a | 1970-01-01T00:00:00 | 1.0 | +| b | 1970-01-01T00:00:00 | 1.0 | +| c | 1970-01-01T00:00:00 | 1.0 | +| d | 1970-01-01T00:00:00 | 1.0 | ++------+---------------------+---------------------+ + +-- SQLNESS SORT_RESULT 3 1 +tql eval (0, 0, '1s') round(float_semantics); + ++---------------------+----------------------------+------+ +| ts | prom_round(val,Float64(1)) | host | ++---------------------+----------------------------+------+ +| 1970-01-01T00:00:00 | -2.0 | d | +| 1970-01-01T00:00:00 | 1.0 | a | +| 1970-01-01T00:00:00 | 2.0 | b | +| 1970-01-01T00:00:00 | NaN | c | ++---------------------+----------------------------+------+ + +-- SQLNESS SORT_RESULT 3 1 +tql eval (0, 0, '1s') sqrt(float_semantics); + ++---------------------+--------------------+------+ +| ts | sqrt(val) | host | ++---------------------+--------------------+------+ +| 1970-01-01T00:00:00 | 1.0 | a | +| 1970-01-01T00:00:00 | 1.4142135623730951 | b | +| 1970-01-01T00:00:00 | NaN | c | +| 1970-01-01T00:00:00 | NaN | d | ++---------------------+--------------------+------+ + +-- prom_min/prom_max have their own column names, so SQL can use them next to min/max. +select min(val) as sql_lo, prom_min(val) as prom_lo from float_semantics; + ++--------+---------+ +| sql_lo | prom_lo | ++--------+---------+ +| -2.5 | -2.5 | ++--------+---------+ + +select max(val) as sql_hi, prom_max(val) as prom_hi from float_semantics; + ++--------+---------+ +| sql_hi | prom_hi | ++--------+---------+ +| NaN | 2.0 | ++--------+---------+ + +select prom_min(val) as lo from float_semantics having min(val) < 0; + ++------+ +| lo | ++------+ +| -2.5 | ++------+ + +drop table float_semantics; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/promql/float_semantics.sql b/tests/cases/standalone/common/promql/float_semantics.sql new file mode 100644 index 00000000000..f23e95141c4 --- /dev/null +++ b/tests/cases/standalone/common/promql/float_semantics.sql @@ -0,0 +1,41 @@ +-- NaN and rounding follow Prometheus (IEEE 754) rather than SQL ordering. +-- Expected results are from Prometheus 3. + +create table float_semantics ( + ts timestamp time index, + val double, + host string primary key +); + +insert into float_semantics values + (0, 1.0, 'a'), + (0, 2.0, 'b'), + (0, 'NaN'::double, 'c'), + (0, -2.5, 'd'); + +tql eval (0, 0, '1s') max(float_semantics); + +tql eval (0, 0, '1s') min(float_semantics); + +tql eval (0, 0, '1s') topk(1, float_semantics); + +-- SQLNESS SORT_RESULT 3 1 +tql eval (0, 0, '1s') float_semantics <= NaN; + +-- SQLNESS SORT_RESULT 3 1 +tql eval (0, 0, '1s') float_semantics != bool NaN; + +-- SQLNESS SORT_RESULT 3 1 +tql eval (0, 0, '1s') round(float_semantics); + +-- SQLNESS SORT_RESULT 3 1 +tql eval (0, 0, '1s') sqrt(float_semantics); + +-- prom_min/prom_max have their own column names, so SQL can use them next to min/max. +select min(val) as sql_lo, prom_min(val) as prom_lo from float_semantics; + +select max(val) as sql_hi, prom_max(val) as prom_hi from float_semantics; + +select prom_min(val) as lo from float_semantics having min(val) < 0; + +drop table float_semantics; diff --git a/tests/cases/standalone/common/promql/functions.result b/tests/cases/standalone/common/promql/functions.result index 325fbccd17d..1867413aec8 100644 --- a/tests/cases/standalone/common/promql/functions.result +++ b/tests/cases/standalone/common/promql/functions.result @@ -109,7 +109,7 @@ tql eval (4, 4, '1s') quantile_over_time(0.4 + 0.1, prom_series_q[4s]); tql eval (3, 3, '1s') round(prom_series_q); +---------------------+----------------------------+------+ -| ts | prom_round(val,Float64(0)) | host | +| ts | prom_round(val,Float64(1)) | host | +---------------------+----------------------------+------+ | 1970-01-01T00:00:03 | 346.0 | q | +---------------------+----------------------------+------+ @@ -133,8 +133,8 @@ tql eval (1, 4, '1s') round(prom_series_q, 0.05 + 0.05); | ts | prom_round(val,Float64(0.05) + Float64(0.05)) | host | +---------------------+-----------------------------------------------+------+ | 1970-01-01T00:00:01 | 123.5 | q | -| 1970-01-01T00:00:02 | 234.60000000000002 | q | -| 1970-01-01T00:00:03 | 345.70000000000005 | q | +| 1970-01-01T00:00:02 | 234.6 | q | +| 1970-01-01T00:00:03 | 345.7 | q | | 1970-01-01T00:00:04 | 456.8 | q | +---------------------+-----------------------------------------------+------+ @@ -156,10 +156,10 @@ tql eval (1, 4, '1s') round(prom_series_q, 100.0 + 3.0); +---------------------+-------------------------------------------+------+ | ts | prom_round(val,Float64(100) + Float64(3)) | host | +---------------------+-------------------------------------------+------+ -| 1970-01-01T00:00:01 | 103.0 | q | -| 1970-01-01T00:00:02 | 206.0 | q | +| 1970-01-01T00:00:01 | 103.00000000000001 | q | +| 1970-01-01T00:00:02 | 206.00000000000003 | q | | 1970-01-01T00:00:03 | 309.0 | q | -| 1970-01-01T00:00:04 | 412.0 | q | +| 1970-01-01T00:00:04 | 412.00000000000006 | q | +---------------------+-------------------------------------------+------+ -- SQLNESS SORT_RESULT 3 1 diff --git a/tests/cases/standalone/common/promql/subquery_default_step.result b/tests/cases/standalone/common/promql/subquery_default_step.result new file mode 100644 index 00000000000..47008660cad --- /dev/null +++ b/tests/cases/standalone/common/promql/subquery_default_step.result @@ -0,0 +1,53 @@ +-- A subquery without a step uses the evaluation interval (one minute by +-- default in Prometheus), not the query step. Expected results are from Prometheus 3. +create table subquery_default_step ( + ts timestamp time index, + val double, + host string primary key +); + +Affected Rows: 0 + +insert into subquery_default_step values + (0, 1.0, 'a'), + (60000, 1.0, 'a'), + (120000, 1.0, 'a'), + (180000, 1.0, 'a'), + (240000, 1.0, 'a'), + (300000, 1.0, 'a'); + +Affected Rows: 6 + +tql eval (300, 300, '1s') count_over_time(subquery_default_step[5m:]); + ++---------------------+------------------------------------+------+ +| ts | prom_count_over_time(ts_range,val) | host | ++---------------------+------------------------------------+------+ +| 1970-01-01T00:05:00 | 5.0 | a | ++---------------------+------------------------------------+------+ + +tql eval (300, 300, '1s') count_over_time(subquery_default_step[5m:30s]); + ++---------------------+------------------------------------+------+ +| ts | prom_count_over_time(ts_range,val) | host | ++---------------------+------------------------------------+------+ +| 1970-01-01T00:05:00 | 10.0 | a | ++---------------------+------------------------------------+------+ + +-- The outer range query keeps its own step around the subquery's default step. +tql eval (300, 360, '15s') count_over_time(subquery_default_step[5m:]); + ++---------------------+------------------------------------+------+ +| ts | prom_count_over_time(ts_range,val) | host | ++---------------------+------------------------------------+------+ +| 1970-01-01T00:05:00 | 5.0 | a | +| 1970-01-01T00:05:15 | 5.0 | a | +| 1970-01-01T00:05:30 | 5.0 | a | +| 1970-01-01T00:05:45 | 5.0 | a | +| 1970-01-01T00:06:00 | 5.0 | a | ++---------------------+------------------------------------+------+ + +drop table subquery_default_step; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/promql/subquery_default_step.sql b/tests/cases/standalone/common/promql/subquery_default_step.sql new file mode 100644 index 00000000000..fb1acb5f45b --- /dev/null +++ b/tests/cases/standalone/common/promql/subquery_default_step.sql @@ -0,0 +1,25 @@ +-- A subquery without a step uses the evaluation interval (one minute by +-- default in Prometheus), not the query step. Expected results are from Prometheus 3. + +create table subquery_default_step ( + ts timestamp time index, + val double, + host string primary key +); + +insert into subquery_default_step values + (0, 1.0, 'a'), + (60000, 1.0, 'a'), + (120000, 1.0, 'a'), + (180000, 1.0, 'a'), + (240000, 1.0, 'a'), + (300000, 1.0, 'a'); + +tql eval (300, 300, '1s') count_over_time(subquery_default_step[5m:]); + +tql eval (300, 300, '1s') count_over_time(subquery_default_step[5m:30s]); + +-- The outer range query keeps its own step around the subquery's default step. +tql eval (300, 360, '15s') count_over_time(subquery_default_step[5m:]); + +drop table subquery_default_step; diff --git a/tests/cases/standalone/common/promql/tsid_binary_join_regression.result b/tests/cases/standalone/common/promql/tsid_binary_join_regression.result index 4a75252e6ee..c3810ea94b0 100644 --- a/tests/cases/standalone/common/promql/tsid_binary_join_regression.result +++ b/tests/cases/standalone/common/promql/tsid_binary_join_regression.result @@ -346,7 +346,7 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > tsid_binary_join_right; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@3, __tsid@1), (ts@4, ts@2)], filter=greptime_value@1 < greptime_value@0, projection=[ts@4, greptime_value@0, host@1, job@2, __tsid@3], NullsEqual: true REDACTED +| 0_| 0_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@3, __tsid@1), (ts@4, ts@2)], filter=prom_gt(greptime_value@0, greptime_value@1), projection=[ts@4, greptime_value@0, host@1, job@2, __tsid@3], NullsEqual: true REDACTED |_|_|_RepartitionExec: partitioning=Hash([__tsid@3, ts@4],REDACTED |_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, host@1 as host, job@2 as job, __tsid@3 as __tsid, ts@4 as ts] REDACTED |_|_|_MergeScanExec: REDACTED @@ -387,8 +387,8 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > bool tsid_binary_join_right; +-+-+-+ | stage | node | plan_| +-+-+-+ -| 0_| 0_|_ProjectionExec: expr=[host@0 as host, job@1 as job, ts@2 as ts, __tsid@3 as __tsid, CAST(greptime_value@4 < greptime_value@5 AS Float64) as tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value] REDACTED -|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@2, ts@4)], projection=[host@4, job@5, ts@7, __tsid@6, greptime_value@3, greptime_value@0], NullsEqual: true REDACTED +| 0_| 0_|_ProjectionExec: expr=[host@0 as host, job@1 as job, ts@2 as ts, __tsid@3 as __tsid, CAST(prom_gt(greptime_value@4, greptime_value@5) AS Float64) as tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@2, ts@4)], projection=[host@4, job@5, ts@7, __tsid@6, greptime_value@0, greptime_value@3], NullsEqual: true REDACTED |_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED |_|_|_CooperativeExec REDACTED |_|_|_MergeScanExec: REDACTED @@ -432,7 +432,7 @@ TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > tsid_binary_join_right) / tsi | 0_| 0_|_ProjectionExec: expr=[host@0 as host, job@1 as job, ts@2 as ts, __tsid@3 as __tsid, greptime_value@4 / greptime_value@5 * 100 as .greptime_value / tsid_binary_join_left.greptime_value * Float64(100)] REDACTED |_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@2, __tsid@3), (ts@0, ts@4)], projection=[host@4, job@5, ts@7, __tsid@6, greptime_value@1, greptime_value@3], NullsEqual: true REDACTED |_|_|_CoalescePartitionsExec REDACTED -|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@2, ts@2)], filter=greptime_value@1 < greptime_value@0, projection=[ts@2, greptime_value@0, __tsid@1], NullsEqual: true REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@2, ts@2)], filter=prom_gt(greptime_value@0, greptime_value@1), projection=[ts@2, greptime_value@0, __tsid@1], NullsEqual: true REDACTED |_|_|_RepartitionExec: partitioning=Hash([REDACTED |_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED |_|_|_MergeScanExec: REDACTED @@ -487,8 +487,8 @@ TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > bool tsid_binary_join_right) |_|_|_CoalescePartitionsExec REDACTED |_|_|_ProjectionExec: expr=[ts@0 as ts, __tsid@1 as __tsid, tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value@2 + greptime_value@3 as tsid_binary_join_right.tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value + tsid_binary_join_left.greptime_value] REDACTED |_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@0, ts@2)], projection=[ts@5, __tsid@4, tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value@2, greptime_value@3], NullsEqual: true REDACTED -|_|_|_ProjectionExec: expr=[ts@0 as ts, __tsid@1 as __tsid, CAST(greptime_value@2 < greptime_value@3 AS Float64) as tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value] REDACTED -|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@2, ts@2)], projection=[ts@5, __tsid@4, greptime_value@3, greptime_value@0], NullsEqual: true REDACTED +|_|_|_ProjectionExec: expr=[ts@0 as ts, __tsid@1 as __tsid, CAST(prom_gt(greptime_value@2, greptime_value@3) AS Float64) as tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value] REDACTED +|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@2, ts@2)], projection=[ts@5, __tsid@4, greptime_value@0, greptime_value@3], NullsEqual: true REDACTED |_|_|_RepartitionExec: partitioning=Hash([REDACTED |_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED |_|_|_MergeScanExec: REDACTED