rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-10-12 23:12:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(promql): sync capability with current main

Browse Source 
Signed-off-by: discord9 <55937128+discord9@users.noreply.github.com>
This commit is contained in:
discord9
2026-10-12 17:27:05 +08:00
parent c750cc1fd2 b33dcc4b02
commit 432a92f529
47 changed files with 3521 additions and 433 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+39 -38
View File
@@ -8702,6 +8702,7 @@ dependencies = [
"api",
"aquamarine",
"arrow-schema 59.2.0",
"arrow-select 59.2.0",
"async-channel 1.9.0",
"async-stream",
"async-trait",
@@ -9686,9 +9687,9 @@ checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"
[[package]]
name = "opendal"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9fe43e16d96bed57937eb7c4a28559bf4c5fc4a3951656505d1fdd4f856349b4"
checksum = "3cbd193aec6d62efaff87611dfae52c132aed650bb3cb0c684c0a9394f09a42a"
dependencies = [
"ctor",
"opendal-core",
@@ -9711,9 +9712,9 @@ dependencies = [
[[package]]
name = "opendal-core"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "de4566a412776e3f65d53dd9fceced5b432a0a9c9a1e8d50ccf58823795e5c5b"
checksum = "54b5555d753dd6a6383dc0891254b72f3cdc40c011ba0372b560e0ae5aa29f5c"
dependencies = [
"anyhow",
"asyncband",
@@ -9737,9 +9738,9 @@ dependencies = [
[[package]]
name = "opendal-http-transport-reqwest"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f772ce6c137ab43647726116d505a649a88d41928fcd813438eb30b02312bcbb"
checksum = "4d6588678f3040a3eab1819364dcbc6a746a8286d1ceb23a70a6614faa01dc6c"
dependencies = [
"bytes",
"futures",
@@ -9751,9 +9752,9 @@ dependencies = [
[[package]]
name = "opendal-layer-concurrent-limit"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3734c648617e5ad724d49175dde8323550624f9614f50e9e81b7aafae30b1734"
checksum = "db8574309e68bf6d49c5378eaef9d0b3f6786971650b061c69e808254d71cb05"
dependencies = [
"asyncband",
"futures",
@@ -9763,9 +9764,9 @@ dependencies = [
[[package]]
name = "opendal-layer-logging"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a8daab5a91eac76f94516e7c71e677195c9221840ce8f5e71815e3fc8207742"
checksum = "4ad43246d2cb7b7ad32b10e22bfa7b6ced8a6677cb6d6f005ae7e1d94cb19c6a"
dependencies = [
"log",
"opendal-core",
@@ -9773,9 +9774,9 @@ dependencies = [
[[package]]
name = "opendal-layer-observe-metrics-common"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fbe664be117c293cfa3980a0d2fc238bbb38eae5f6c191f376e88ccb830d26b"
checksum = "3b8bc97d5a00eb41d50f1f6077c4bd5396c1eefea495614084ee9a37202ae06d"
dependencies = [
"futures",
"http 1.5.0",
@@ -9784,9 +9785,9 @@ dependencies = [
[[package]]
name = "opendal-layer-prometheus"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b32078d8248555ee6271332ec480b78306a44f0d902ea4721ccac00d9b866a21"
checksum = "404e766cc2e4e4c9225ce08605ae151f768a05a41a448a235e016b1aea9ff98f"
dependencies = [
"opendal-core",
"opendal-layer-observe-metrics-common",
@@ -9795,9 +9796,9 @@ dependencies = [
[[package]]
name = "opendal-layer-retry"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb8e22cb1f86ef9af771a8947d3ccf380c8f5b76236f628314dd41c9ba73331a"
checksum = "4aeb54be4940c68609483591934a813f7bdd97d747826783a393914d3139e363"
dependencies = [
"backon",
"log",
@@ -9806,9 +9807,9 @@ dependencies = [
[[package]]
name = "opendal-layer-timeout"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08ac6c77fe0d1b18e0064a02c19b98dbebcab78bf5e7753ad1f3abbc18850ff4"
checksum = "1d11d53e67473e8914eacb2e14d586779ae57e1a84759ad25232ef339db77d8a"
dependencies = [
"opendal-core",
"tokio",
@@ -9816,9 +9817,9 @@ dependencies = [
[[package]]
name = "opendal-layer-tracing"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "603807927a799ea981f62a3b0012e17181e39f468793b78ca8bedda66d19813c"
checksum = "bbe2ebfc795bf08da77d20bed13d149b478cc8e5002df44d237e460134f00be4"
dependencies = [
"futures",
"http 1.5.0",
@@ -9828,9 +9829,9 @@ dependencies = [
[[package]]
name = "opendal-service-azblob"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c6cf31b02e7b44191d97c07254eea109b5da9c17210a29e478c8d0dc26dffb1f"
checksum = "c32c3fc2e507c854ce53615b28fc2208f5e00cbe456c4d344ffdfbf4c9cb976d"
dependencies = [
"base64 0.23.1",
"bytes",
@@ -9849,9 +9850,9 @@ dependencies = [
[[package]]
name = "opendal-service-azure-common"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8107324c3f3cb9970fe20526c2c341b7c5769acbd4699bab73bd525f5fd63d62"
checksum = "cc93c30bd347a36d93fd46d47f3e2bb6494dcf9458938aa501fe046ff8f7562f"
dependencies = [
"http 1.5.0",
"opendal-core",
@@ -9859,9 +9860,9 @@ dependencies = [
[[package]]
name = "opendal-service-fs"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0e7d8974aebd33be899200c9d0d50b2141fd5fa5c462b29e20653e3ddbfe339"
checksum = "ec5c16fe91b77c0364f66546a1c32b2999e9c3c3c75c2e908a282268e0890bac"
dependencies = [
"bytes",
"log",
@@ -9873,9 +9874,9 @@ dependencies = [
[[package]]
name = "opendal-service-gcs"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d80eca6f9227bdf65d43e5b8d5fe75dca8cfab1da07571442a0420400847bea"
checksum = "e1b5e42c6606ea1e37fa9cdedf07dc998073a45eb5044317c0fa6f3153e98d6e"
dependencies = [
"async-trait",
"bytes",
@@ -9895,9 +9896,9 @@ dependencies = [
[[package]]
name = "opendal-service-hdfs-native"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b84340e11a3bcd7f1f0b55df032d040b2907483c546c3021485aba749c3072c"
checksum = "20fb97b2ac06a87e6720f0f93d5107ac046117deb8ac957030b0962ba4ecc3c5"
dependencies = [
"bytes",
"futures",
@@ -9909,9 +9910,9 @@ dependencies = [
[[package]]
name = "opendal-service-http"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ac1553d6b57ca79e200e5c7ded9ab708f2016dadb28ede0f2da71e02fd0b960"
checksum = "65fdde60d1abf14e47eaf19360c08ea1b89c14f03769db448e016781c2e0e32b"
dependencies = [
"http 1.5.0",
"log",
@@ -9921,9 +9922,9 @@ dependencies = [
[[package]]
name = "opendal-service-mysql"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e15592be32b105cd152ab398b3db847e22c8c5f230d9b34ff095ea3d0dc6fd97"
checksum = "9ba30e2a0b38aa55579b69bf57f607df7aa613f4d964496407b7998f2dbaefcc"
dependencies = [
"asyncband",
"opendal-core",
@@ -9933,9 +9934,9 @@ dependencies = [
[[package]]
name = "opendal-service-oss"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0ebfa355b1d4edb0e5b2a5b86d8de9f811fae1d7baca6d65246b230b2afc0bd"
checksum = "d64d2909b444008e42f00137a50df6b55ba599eeb6c8b46c71a2a84948f91880"
dependencies = [
"bytes",
"http 1.5.0",
@@ -9950,9 +9951,9 @@ dependencies = [
[[package]]
name = "opendal-service-s3"
version = "0.59.2"
version = "0.59.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2942f2d8d3d4953c0e8d07cd1244879628d084948e48ce6b6d808c5938f7858d"
checksum = "1d83e716c25a81fe2ca701894b8e14b9966acca3812af250e50132accb19da02"
dependencies = [
"base64 0.23.1",
"bytes",
Cargo.toml
+1
View File
@@ -107,6 +107,7 @@ arrow-buffer = "=59.2.0"
arrow-flight = "=59.2.0"
arrow-ipc = { version = "=59.2.0", default-features = false, features = ["lz4", "zstd"] }
arrow-schema = { version = "=59.2.0", features = ["serde"] }
arrow-select = "=59.2.0"
async-stream = "0.3"
async-trait = "0.1"
# Remember to update axum-extra, axum-macros when updating axum
docs/rfcs/2026-09-11-metric-export-import.md
+47 -18
View File
@@ -31,13 +31,21 @@ includes reports of exports taking days for roughly 100,000 logical tables with
only several GiB of data. Table count, physical schema width and data volume must
be measured separately; total database bytes do not describe the transfer cost.
The [experiment conclusions](2026-09-11-metric-export-import/experiment-results.md)
support shared scans, batch DDL, explicit-file COPY and packed-object transfer.
Packing with a bounded range reader substantially reduces storage requests and
improves full export and restore for many small logical tables. Benefits depend
on request latency, and memory use increases. Remaining CREATE work makes batch
DDL part of the first release. These local experiments establish direction;
production V2 recovery and release performance still require acceptance tests.
Local standalone experiments with development builds, local files and Docker
MinIO support shared scans, batch DDL, explicit-file COPY and packed-object
transfer. Packing with a bounded range reader reduces storage requests and
improves full export and restore for many small logical tables; low-latency
export gains are smaller. CPU and request overhead fell while peak process
memory increased. Remaining CREATE work makes batch DDL part of the first
release. Explicit-file COPY removes repeated directory traversal while retaining
ordinary per-table insertion, independently of merged database writes.
The exercised fixtures preserved schemas, DDL and typed values. Standalone-file
experiments used the existing importer; packing used an independent driver and
prototype range reader. These experiments establish design direction, not
production packed-format compatibility, V2 recovery safety or release performance.
The 8 MiB write/part limit was a design choice, not a measured optimum. Current
acceptance status is maintained in the [tracking issue](https://github.com/GreptimeTeam/greptimedb/issues/9120).
# Terminology
@@ -552,18 +560,39 @@ batching, still writing standalone table objects; (C) B plus packing and its
reader, with identical DDL handling; (D) C plus batch DDL. If general restore
request deduplication is available, include it in B as well.
The proposed release target is at least 3x faster full export and full restore
for D versus A on 10,000 and 100,000 small logical tables with MinIO request
latency increased by 5 ms. Fix this target during RFC review, before running
release measurements. The independent local C-versus-B experiment establishes
neither this comparison nor proof of that target.
The release target is at least 3x faster full export and full restore for D
versus A on both 10,000 and 100,000 small logical tables at both +50 ms and
+100 ms added MinIO request latency. Populate every selected table with the same
deterministic data across controls, initially 32 rows per table. Include schema
export, metadata discovery and CLI startup/shutdown in complete wall duration.
The independent local C-versus-B experiment establishes neither this comparison
nor proof of that target. Keep production improvements already shared by all
controls in A, and identify which increments can actually be isolated.
Use Linux release builds and at least three alternating rounds for primary
comparisons. Report medians/spread, schema/data/full durations, CPU, RSS, bytes,
objects, request counts and queue/cache peaks. Include zero-added-latency,
local-file, mixed small/large-table and wide-schema controls. Investigate and
remeasure full-duration regressions exceeding 5% versus A; do not generalize the
latency-injected speedup to these cases. Check values and schemas outside timing.
Start with native optimized release builds on macOS. Record hardware, OS,
MinIO placement, container/VM CPU and memory limits, and connection/storage
settings. Linux deployment-side performance confirmation can follow separately;
Linux root/non-root and Windows correctness checks remain distinct requirements.
Earlier development-build diagnostics, runs with paging interference and sparse
fixtures with only one populated table do not establish this acceptance.
Calibrate one added delay per object-store HTTP request, including HEAD, GET
(and range GET), PUT, LIST, DELETE and each multipart operation. Do not delay
individual packets, streaming body chunks or database SQL/DDL. Route CLI and
server object access through the same controlled endpoint and report measured
per-method latency distributions. Synthetic delay is not a cloud provider profile.
Use a small calibration pilot before each scale and at least three alternating
A/B/C/D rounds for primary comparisons, with fresh restore namespaces, equal
concurrency and comparable cache/compaction conditions. Report medians/spread,
schema/data/full durations, CPU, RSS, swap activity, bytes, objects, requests and
queue/cache peaks. Check RAM, disk and MinIO health before large runs; do not
compile concurrently. Fix interfering paging/load or report the scale unverified.
Retain local-file and MinIO 0/+5 ms controls, mixed small/large tables, wide
physical column unions and multiple physical groups. Add +20 ms only if needed
to explain sensitivity. Investigate and remeasure full-duration regressions
exceeding 5% versus A. Check table membership, schemas and typed values outside
timing, and report B/A, C/B and D/C independently where comparable.
Validate actual write/part sizes at or below 8 MiB, aggregate concurrency across
groups/chunks, and resource release after success, cancellation and failure.
docs/rfcs/2026-09-11-metric-export-import/experiment-results.md
-34
View File
@@ -1,34 +0,0 @@
# Metric export/import experiment conclusions
The experiments support shared physical scans, explicit-file COPY, batched
logical DDL, and packing small logical-table Parquet streams into shared objects.
- **Shared scans reduce repeated query work.** Projecting each logical schema
before dictionary expansion preserves its columns and types, but does not
remove the memory cost of a wide physical schema or distributed ordering.
- **Explicit-file COPY removes repeated directory traversal.** This improvement
uses ordinary per-table insertion; it is independent of database merged writes.
- **Packing and its range reader reduce small-object overhead.** They improve
complete export and restore and sharply reduce object-store requests. Benefits
grow with table count and request latency; low-latency export gains are smaller.
Some reader request savings can also benefit standalone files.
- **Batch DDL remains necessary.** Once data restore is faster, logical-table
creation becomes the main remaining restore cost. The batch-DDL experiment used
an internal interface; production transport and retry integration remain work.
- **Bounded writes allow larger objects.** The packing experiment kept writes
and upload parts within 8 MiB without imposing that limit on whole objects or
splitting logical tables. This was the chosen size, not a measured optimum.
- **Packing has a memory trade-off.** CPU and request overhead fell, while peak
process memory increased. Encoding buffers, upload concurrency, range windows
and table/index metadata need separate accounting.
The exercised fixtures preserved schemas, DDL and typed values after restore.
Earlier standalone-file experiments used the existing importer; the packing
experiment used an independent driver and prototype range reader. The latter
does not establish production packed-format compatibility or V2 resume safety.
These were local standalone experiments using development builds, local files
and Docker MinIO. Release acceptance still needs the actual packed V2 path,
request-owned reader isolation, old-reader rejection, large-table throughput,
distributed ordering, cancellation and recovery. See the [RFC](../2026-09-11-metric-export-import.md)
and [tracking issue](https://github.com/GreptimeTeam/greptimedb/issues/9120).
src/cli/src/data/snapshot_storage.rs
+115 -6
View File
@@ -21,6 +21,7 @@ use std::collections::BTreeSet;
use std::path::Component;
use async_trait::async_trait;
use common_datasource::packed_writer::WRITE_BYTES;
use futures::TryStreamExt;
use object_store::services::{Azblob, Fs, Gcs, Oss, S3};
use object_store::util::{with_instrument_layers, with_retry_layers};
@@ -570,13 +571,31 @@ impl OpenDalStorage {
/// Writes bytes to a file.
async fn write_file(&self, path: &str, data: Vec<u8>) -> Result<()> {
self.object_store
.write(path, data)
let mut writer = self
.object_store
.writer_with(path)
.chunk(WRITE_BYTES)
.concurrent(1)
.await
.map(|_| ())
.context(StorageOperationSnafu {
operation: format!("write {}", path),
})
.with_context(|_| StorageOperationSnafu {
operation: format!("open writer {}", path),
})?;
let data = object_store::Buffer::from(data);
let result = async {
for chunk in data.chunks(WRITE_BYTES) {
writer.write(chunk).await?;
}
writer.close().await.map(|_| ())
}
.await;
if result.is_err()
&& let Err(error) = writer.abort().await
{
common_telemetry::warn!(error; "Failed to abort snapshot metadata write");
}
result.with_context(|_| StorageOperationSnafu {
operation: format!("write {}", path),
})
}
/// Checks if a file exists using stat.
@@ -837,6 +856,96 @@ mod tests {
)
}
#[tokio::test]
async fn snapshot_metadata_submissions_are_bounded_and_invoke_abort_on_failure() {
use std::sync::Mutex;
use std::sync::atomic::{AtomicUsize, Ordering};
use object_store::layers::mock::{Metadata, ObservedWriter, WriterObservations, oio};
struct FailWriter {
inner: oio::Writer,
writes: usize,
fail: Arc<AtomicUsize>,
}
impl oio::Write for FailWriter {
async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> {
self.writes += 1;
if self.fail.load(Ordering::SeqCst) == 1 && self.writes == 2 {
return Err(object_store::Error::new(
ErrorKind::Unexpected,
"injected metadata write failure",
));
}
self.inner.write(bytes).await
}
async fn close(&mut self) -> object_store::Result<Metadata> {
if self.fail.load(Ordering::SeqCst) == 2 {
return Err(object_store::Error::new(
ErrorKind::Unexpected,
"injected metadata close failure",
));
}
self.inner.close().await
}
async fn abort(&mut self) -> object_store::Result<()> {
self.inner.abort().await
}
}
let dir = tempdir().unwrap();
let mut storage = make_storage_with_rooted_fs(dir.path());
let observations = Arc::new(Mutex::new(WriterObservations::default()));
let fail = Arc::new(AtomicUsize::new(0));
storage.object_store = storage.object_store.layer(
MockLayerBuilder::default()
.writer_factory(Arc::new({
let observations = observations.clone();
let fail = fail.clone();
move |_, _, inner| {
Box::new(ObservedWriter::new(
Box::new(FailWriter {
inner,
writes: 0,
fail: fail.clone(),
}),
observations.clone(),
))
}
}))
.build()
.unwrap(),
);
for length in [2 * WRITE_BYTES + 17, 1, 0] {
*observations.lock().unwrap() = WriterObservations::default();
let data = vec![42; length];
storage.write_file("metadata", data.clone()).await.unwrap();
assert_eq!(storage.read_file("metadata").await.unwrap(), data);
let observed = observations.lock().unwrap();
assert_eq!(observed.write_sizes.iter().sum::<usize>(), length);
assert!(observed.write_sizes.iter().all(|size| *size <= WRITE_BYTES));
assert_eq!(observed.close_calls, 1);
assert_eq!(observed.abort_calls, 0);
}
for failure in [1, 2] {
*observations.lock().unwrap() = WriterObservations::default();
fail.store(failure, Ordering::SeqCst);
assert!(
storage
.write_file("failed", vec![42; 2 * WRITE_BYTES + 17])
.await
.is_err()
);
assert_eq!(
observations.lock().unwrap().write_sizes.len(),
if failure == 1 { 2 } else { 3 }
);
let observed = observations.lock().unwrap();
assert_eq!(observed.close_calls, usize::from(failure == 2));
assert_eq!(observed.abort_calls, 1);
}
}
#[test]
fn test_validate_uri_valid() {
assert_eq!(validate_uri("s3://bucket/path").unwrap(), StorageScheme::S3);
src/common/datasource/src/packed_writer.rs
+25 -23
View File
@@ -410,7 +410,9 @@ mod tests {
use arrow::array::{ArrayRef, Int64Array, StringArray};
use arrow::record_batch::RecordBatch;
use futures::TryStreamExt;
use object_store::layers::mock::{Metadata, MockLayerBuilder, MockWriterFactory, oio};
use object_store::layers::mock::{
Metadata, MockLayerBuilder, MockWriterFactory, ObservedWriter, WriterObservations, oio,
};
use parquet::arrow::ParquetRecordBatchStreamBuilder;
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
@@ -424,10 +426,10 @@ mod tests {
let store = object_store::secure_fs::SecureFsRoot::open(directory.path())
.unwrap()
.build_operator();
let sizes = Arc::new(std::sync::Mutex::new(Vec::new()));
let observations = Arc::new(std::sync::Mutex::new(WriterObservations::default()));
let factory: MockWriterFactory = Arc::new({
let sizes = sizes.clone();
move |_, _, writer| Box::new(Observe(writer, sizes.clone()))
let observations = observations.clone();
move |_, _, writer| Box::new(ObservedWriter::new(writer, observations.clone()))
});
let store = store.layer(
MockLayerBuilder::default()
@@ -483,7 +485,14 @@ mod tests {
.validate_membership(["table0", "table1", "table2"])
.unwrap();
assert_eq!(inventory.len(), 3);
assert!(sizes.lock().unwrap().iter().all(|n| *n <= WRITE_BYTES));
assert!(
observations
.lock()
.unwrap()
.write_sizes
.iter()
.all(|n| *n <= WRITE_BYTES)
);
assert_eq!(
index
.objects
@@ -540,27 +549,13 @@ mod tests {
}
}
struct Observe(oio::Writer, Arc<std::sync::Mutex<Vec<usize>>>);
impl oio::Write for Observe {
async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> {
self.1.lock().unwrap().push(bytes.len());
self.0.write(bytes).await
}
async fn close(&mut self) -> object_store::Result<Metadata> {
self.0.close().await
}
async fn abort(&mut self) -> object_store::Result<()> {
self.0.abort().await
}
}
#[tokio::test]
async fn rolls_at_table_boundary_and_aborts_standalone_at_part_limit() {
let directory = common_test_util::temp_dir::create_temp_dir("packed-limit");
let sizes = Arc::new(std::sync::Mutex::new(Vec::new()));
let observations = Arc::new(std::sync::Mutex::new(WriterObservations::default()));
let factory: MockWriterFactory = Arc::new({
let sizes = sizes.clone();
move |_, _, writer| Box::new(Observe(writer, sizes.clone()))
let observations = observations.clone();
move |_, _, writer| Box::new(ObservedWriter::new(writer, observations.clone()))
});
let store = object_store::secure_fs::SecureFsRoot::open(directory.path())
.unwrap()
@@ -605,7 +600,14 @@ mod tests {
.iter()
.all(|t| t.offset == 0 && t.length == 600)
);
assert!(sizes.lock().unwrap().iter().all(|n| *n <= 512));
assert!(
observations
.lock()
.unwrap()
.write_sizes
.iter()
.all(|n| *n <= 512)
);
}
struct FailClose(oio::Writer);
src/common/datasource/src/parquet_writer.rs
+7 -18
View File
@@ -500,28 +500,17 @@ mod tests {
async fn large_footer_flush_uses_bounded_submissions_in_one_parquet_stream() {
use std::sync::Mutex;
use object_store::layers::mock::{Metadata, MockLayerBuilder, MockWriterFactory, oio};
struct ObservedWriter(oio::Writer, Arc<Mutex<Vec<usize>>>);
impl oio::Write for ObservedWriter {
async fn write(&mut self, bytes: object_store::Buffer) -> object_store::Result<()> {
self.1.lock().unwrap().push(bytes.len());
self.0.write(bytes).await
}
async fn close(&mut self) -> object_store::Result<Metadata> {
self.0.close().await
}
async fn abort(&mut self) -> object_store::Result<()> {
self.0.abort().await
}
}
use object_store::layers::mock::{
MockLayerBuilder, MockWriterFactory, ObservedWriter, WriterObservations,
};
let directory = common_test_util::temp_dir::create_temp_dir("bounded_parquet");
let store = object_store::secure_fs::SecureFsRoot::open(directory.path())
.unwrap()
.build_operator();
let sizes = Arc::new(Mutex::new(Vec::new()));
let observations = Arc::new(Mutex::new(WriterObservations::default()));
let factory: MockWriterFactory = Arc::new({
let sizes = sizes.clone();
move |_, _, writer| Box::new(ObservedWriter(writer, sizes.clone()))
let observations = observations.clone();
move |_, _, writer| Box::new(ObservedWriter::new(writer, observations.clone()))
});
let store = store.layer(
MockLayerBuilder::default()
@@ -548,7 +537,7 @@ mod tests {
writer.sink = ParquetSink::Object(store.writer("large.parquet").await.unwrap());
writer.write(batch.clone(), None).await.unwrap();
writer.finish(None).await.unwrap();
let sizes = sizes.lock().unwrap().clone();
let sizes = observations.lock().unwrap().write_sizes.clone();
let limit = DEFAULT_WRITE_BUFFER_SIZE.as_bytes() as usize;
assert!(sizes.iter().sum::<usize>() > limit);
assert!(sizes.iter().all(|size| *size <= limit), "{sizes:?}");
src/common/function/src/aggrs.rs
+1
View File
@@ -14,6 +14,7 @@
pub mod approximate;
pub mod count_hash;
pub mod extremum;
#[cfg(feature = "geo")]
pub mod geo;
pub mod vector;
src/common/function/src/aggrs/extremum.rs
+259
View File
@@ -0,0 +1,259 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! PromQL `min` and `max` aggregation operators.
//!
//! PromQL ignores NaN samples unless every sample of a group is NaN. DataFusion's
//! `min`/`max` order NaN differently depending on the code path (its grouped
//! accumulator lets NaN replace the current value and be replaced by the next one),
//! so the result depends on the order of the input.
use std::sync::Arc;
use datafusion::arrow::array::{ArrayRef, AsArray};
use datafusion::arrow::datatypes::{DataType, Field, FieldRef, Float64Type};
use datafusion::error::Result as DfResult;
use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs};
use datafusion::logical_expr::{
Accumulator, AggregateUDF, AggregateUDFImpl, GroupsAccumulator, Signature, Volatility,
};
use datafusion_common::ScalarValue;
use datafusion_functions_aggregate_common::aggregate::groups_accumulator::prim_op::PrimitiveGroupsAccumulator;
use crate::function_registry::FunctionRegistry;
/// `max` or `min` with PromQL NaN semantics.
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct Extremum {
name: &'static str,
is_max: bool,
signature: Signature,
}
impl Extremum {
pub const MAX_NAME: &'static str = "prom_max";
pub const MIN_NAME: &'static str = "prom_min";
/// Registers both functions; the registry also derives their state functions, which
/// distributed plans need to split the aggregation between datanodes and the frontend.
pub fn register(registry: &FunctionRegistry) {
registry.register_aggr(Self::max_udaf());
registry.register_aggr(Self::min_udaf());
}
pub fn max_udaf() -> AggregateUDF {
AggregateUDF::from(Self::new(Self::MAX_NAME, true))
}
pub fn min_udaf() -> AggregateUDF {
AggregateUDF::from(Self::new(Self::MIN_NAME, false))
}
fn new(name: &'static str, is_max: bool) -> Self {
Self {
name,
is_max,
signature: Signature::exact(vec![DataType::Float64], Volatility::Immutable),
}
}
}
/// Folds `value` into `current` like Prometheus' aggregation: a NaN is replaced by any
/// sample, and a NaN sample never replaces a number.
fn fold(is_max: bool, current: &mut f64, value: f64) {
let better = if is_max {
value > *current
} else {
value < *current
};
if current.is_nan() || better {
*current = value;
}
}
impl AggregateUDFImpl for Extremum {
fn name(&self) -> &str {
self.name
}
fn signature(&self) -> &Signature {
&self.signature
}
fn return_type(&self, _arg_types: &[DataType]) -> DfResult<DataType> {
Ok(DataType::Float64)
}
fn accumulator(&self, _acc_args: AccumulatorArgs) -> DfResult<Box<dyn Accumulator>> {
Ok(Box::new(ExtremumAccumulator {
is_max: self.is_max,
value: None,
}))
}
fn state_fields(&self, args: StateFieldsArgs) -> DfResult<Vec<FieldRef>> {
Ok(vec![Arc::new(Field::new(
format!("{}[{}]", args.name, self.name),
DataType::Float64,
true,
))])
}
fn groups_accumulator_supported(&self, _args: AccumulatorArgs) -> bool {
true
}
fn create_groups_accumulator(
&self,
_args: AccumulatorArgs,
) -> DfResult<Box<dyn GroupsAccumulator>> {
Ok(groups_accumulator(self.is_max))
}
}
fn groups_accumulator(is_max: bool) -> Box<dyn GroupsAccumulator> {
Box::new(
PrimitiveGroupsAccumulator::<Float64Type, _>::new(&DataType::Float64, move |cur, v| {
fold(is_max, cur, v)
})
.with_starting_value(f64::NAN),
)
}
#[derive(Debug)]
struct ExtremumAccumulator {
is_max: bool,
value: Option<f64>,
}
impl ExtremumAccumulator {
fn update(&mut self, values: &ArrayRef) {
for v in values.as_primitive::<Float64Type>().iter().flatten() {
match &mut self.value {
Some(current) => fold(self.is_max, current, v),
None => self.value = Some(v),
}
}
}
}
impl Accumulator for ExtremumAccumulator {
fn update_batch(&mut self, values: &[ArrayRef]) -> DfResult<()> {
self.update(&values[0]);
Ok(())
}
fn merge_batch(&mut self, states: &[ArrayRef]) -> DfResult<()> {
self.update(&states[0]);
Ok(())
}
fn state(&mut self) -> DfResult<Vec<ScalarValue>> {
Ok(vec![ScalarValue::Float64(self.value)])
}
fn evaluate(&mut self) -> DfResult<ScalarValue> {
Ok(ScalarValue::Float64(self.value))
}
fn size(&self) -> usize {
size_of_val(self)
}
}
#[cfg(test)]
mod tests {
use datafusion::arrow::array::Float64Array;
use datafusion::logical_expr::EmitTo;
use super::*;
fn groups_result(
is_max: bool,
values: Vec<Option<f64>>,
groups: Vec<usize>,
) -> Vec<Option<f64>> {
let num_groups = groups.iter().max().map_or(0, |g| g + 1);
let mut acc = groups_accumulator(is_max);
let values: ArrayRef = Arc::new(Float64Array::from(values));
acc.update_batch(&[values], &groups, None, num_groups)
.unwrap();
let out = acc.evaluate(EmitTo::All).unwrap();
out.as_primitive::<Float64Type>().iter().collect()
}
fn is_nan(v: Option<f64>) -> bool {
v.is_some_and(f64::is_nan)
}
#[test]
fn nan_is_ignored_unless_the_group_is_all_nan() {
// Group 0 has NaN on both sides of a number; group 1 is all NaN; group 2 has only NULL.
let values = vec![
Some(f64::NAN),
Some(3.0),
Some(f64::NAN),
Some(f64::NAN),
None,
];
let groups = vec![0, 0, 0, 1, 2];
for is_max in [true, false] {
let out = groups_result(is_max, values.clone(), groups.clone());
assert_eq!(out[0], Some(3.0));
assert!(is_nan(out[1]));
assert_eq!(out[2], None);
}
}
#[test]
fn result_does_not_depend_on_input_order() {
let forward = vec![Some(1.0), Some(f64::NAN), Some(4.0), Some(2.0)];
let mut backward = forward.clone();
backward.reverse();
assert_eq!(
groups_result(true, forward.clone(), vec![0; 4]),
vec![Some(4.0)]
);
assert_eq!(
groups_result(true, backward.clone(), vec![0; 4]),
vec![Some(4.0)]
);
assert_eq!(groups_result(false, forward, vec![0; 4]), vec![Some(1.0)]);
assert_eq!(groups_result(false, backward, vec![0; 4]), vec![Some(1.0)]);
}
#[test]
fn merged_partial_states_keep_nan_semantics() {
let mut partial_nan = ExtremumAccumulator {
is_max: true,
value: None,
};
partial_nan.update(&(Arc::new(Float64Array::from(vec![f64::NAN])) as ArrayRef));
let mut partial_num = ExtremumAccumulator {
is_max: true,
value: None,
};
partial_num.update(&(Arc::new(Float64Array::from(vec![2.0, 5.0])) as ArrayRef));
let mut merged = ExtremumAccumulator {
is_max: true,
value: None,
};
for partial in [&mut partial_nan, &mut partial_num] {
let state = partial.state().unwrap()[0].to_array().unwrap();
merged.merge_batch(&[state]).unwrap();
}
assert_eq!(merged.evaluate().unwrap(), ScalarValue::Float64(Some(5.0)));
}
}
src/common/function/src/function_registry.rs
+4
View File
@@ -25,6 +25,7 @@ use crate::admin::AdminFunction;
use crate::aggrs::aggr_wrapper::StateMergeHelper;
use crate::aggrs::approximate::ApproximateFunction;
use crate::aggrs::count_hash::CountHash;
use crate::aggrs::extremum::Extremum;
use crate::aggrs::vector::VectorFunction as VectorAggrFunction;
use crate::function::{Function, FunctionRef};
use crate::function_factory::ScalarFunctionFactory;
@@ -260,6 +261,9 @@ pub static FUNCTION_REGISTRY: LazyLock<Arc<FunctionRegistry>> = LazyLock::new(||
// CountHash function
CountHash::register(&function_registry);
// PromQL min/max; registered before the state functions are derived.
Extremum::register(&function_registry);
// state function of supported aggregate functions
StateMergeHelper::register(&function_registry);
src/frontend/src/instance.rs
+1 -1
View File
@@ -3477,7 +3477,7 @@ mod tests {
&instance,
ctx.clone(),
servers::http::jaeger::QueryTraceParams {
service_name: "service".to_string(),
service_name: Some("service".to_string()),
..Default::default()
},
)
src/frontend/src/instance/jaeger.rs
+35 -56
View File
@@ -202,8 +202,9 @@ impl JaegerQueryHandler for Instance {
let mut filters = vec![];
// `service_name` is already validated in `from_jaeger_query_params()`, so no additional check needed here.
filters.push(col(SERVICE_NAME_COLUMN).eq(lit(query_params.service_name)));
if let Some(service_name) = query_params.service_name {
filters.push(col(SERVICE_NAME_COLUMN).eq(lit(service_name)));
}
if let Some(operation_name) = query_params.operation_name {
filters.push(col(SPAN_NAME_COLUMN).eq(lit(operation_name)));
@@ -254,6 +255,7 @@ impl JaegerQueryHandler for Instance {
.await?;
// Get all traces that match the trace ids from the previous query.
// API v3 applies the time window only to the search, not to the returned spans.
// It's equivalent to the following SQL query:
//
// ```
@@ -276,11 +278,15 @@ impl JaegerQueryHandler for Instance {
),
];
if let Some(start_time) = query_params.start_time {
if !query_params.fetch_full_trace
&& let Some(start_time) = query_params.start_time
{
filters.push(col(TIMESTAMP_COLUMN).gt_eq(lit_timestamp_nano(start_time)));
}
if let Some(end_time) = query_params.end_time {
if !query_params.fetch_full_trace
&& let Some(end_time) = query_params.end_time
{
filters.push(col(TIMESTAMP_COLUMN).lt_eq(lit_timestamp_nano(end_time)));
}
@@ -540,64 +546,37 @@ fn json_tag_filters(
dataframe: &DataFrame,
tags: HashMap<String, JsonValue>,
) -> ServerResult<Vec<Expr>> {
let has_resource_attributes = dataframe
.schema()
.has_column_with_unqualified_name(RESOURCE_ATTRIBUTES_COLUMN);
let mut filters = vec![];
// NOTE: The key of the tags may contain `.`, for example: `http.status_code`, so we need to use `["http.status_code"]` in json path to access the value.
for (key, value) in tags.iter() {
if let JsonValue::String(value) = value {
filters.push(
dataframe
.registry()
.udf(JsonGetString::NAME)
.context(DataFusionSnafu)?
.call(vec![
col(SPAN_ATTRIBUTES_COLUMN),
lit(format!("[\"{}\"]", key)),
])
.eq(lit(value)),
);
}
if let JsonValue::Number(value) = value {
if value.is_i64() {
filters.push(
dataframe
.registry()
.udf(JsonGetInt::NAME)
.context(DataFusionSnafu)?
.call(vec![
col(SPAN_ATTRIBUTES_COLUMN),
lit(format!("[\"{}\"]", key)),
])
.eq(lit(value.as_i64().unwrap())),
);
let (udf, value) = match value {
JsonValue::String(value) => (JsonGetString::NAME, lit(value)),
JsonValue::Number(value) if value.is_i64() => {
(JsonGetInt::NAME, lit(value.as_i64().unwrap()))
}
if value.is_f64() {
filters.push(
dataframe
.registry()
.udf(JsonGetFloat::NAME)
.context(DataFusionSnafu)?
.call(vec![
col(SPAN_ATTRIBUTES_COLUMN),
lit(format!("[\"{}\"]", key)),
])
.eq(lit(value.as_f64().unwrap())),
);
JsonValue::Number(value) if value.is_f64() => {
(JsonGetFloat::NAME, lit(value.as_f64().unwrap()))
}
}
if let JsonValue::Bool(value) = value {
filters.push(
dataframe
.registry()
.udf(JsonGetBool::NAME)
.context(DataFusionSnafu)?
.call(vec![
col(SPAN_ATTRIBUTES_COLUMN),
lit(format!("[\"{}\"]", key)),
])
.eq(lit(*value)),
);
}
JsonValue::Bool(value) => (JsonGetBool::NAME, lit(*value)),
_ => continue,
};
let get = dataframe.registry().udf(udf).context(DataFusionSnafu)?;
let path = lit(format!("[\"{}\"]", key));
let span_value = get.call(vec![col(SPAN_ATTRIBUTES_COLUMN), path.clone()]);
// Same precedence as `json2_tag_filters`: a span attribute shadows a resource attribute.
let attribute = if has_resource_attributes {
coalesce(vec![
span_value,
get.call(vec![col(RESOURCE_ATTRIBUTES_COLUMN), path]),
])
} else {
span_value
};
filters.push(attribute.eq(value));
}
Ok(filters)
src/mito2/Cargo.toml
+1
View File
@@ -19,6 +19,7 @@ workspace = true
api.workspace = true
aquamarine.workspace = true
arrow-schema.workspace = true
arrow-select.workspace = true
async-channel = "1.9"
async-stream.workspace = true
async-trait.workspace = true
src/mito2/src/engine/scan_test.rs
+137
View File
@@ -4498,6 +4498,143 @@ async fn test_range_cache_key_separates_sequence_ranges() {
);
}
#[tokio::test]
async fn test_single_source_ranges_preserve_batches() {
for (file_count, row_group_size) in [(1, 64), (1, 16), (2, 16)] {
let mut env = TestEnv::new().await;
let engine = env
.create_engine(MitoConfig {
default_flat_format: true,
..Default::default()
})
.await;
let region_id = RegionId::new(1, 1);
let request = CreateRequestBuilder::new()
.insert_option("memtable.type", "bulk")
.insert_option("max_row_group_row_count", &row_group_size.to_string())
.build();
let schema = test_util::rows_schema(&request);
engine
.handle_request(region_id, RegionRequest::Create(request))
.await
.unwrap();
let rows_per_file = 64 / file_count;
for file in 0..file_count {
test_util::put_rows(
&engine,
region_id,
Rows {
schema: schema.clone(),
rows: test_util::build_rows(file * rows_per_file, (file + 1) * rows_per_file),
},
)
.await;
test_util::flush_region(&engine, region_id, None).await;
}
let Scanner::Seq(scanner) = engine
.scanner(region_id, ScanRequest::default())
.await
.unwrap()
else {
panic!("expected a sequential scan");
};
assert_eq!(file_count, scanner.input().files.len());
assert_eq!(
64 / row_group_size,
scanner.properties().partitions[0].len()
);
let stream = scanner
.scan_partition(&Default::default(), &ExecutionPlanMetricsSet::default(), 0)
.unwrap();
let batches = stream.try_collect::<Vec<_>>().await.unwrap();
assert_eq!(
vec![row_group_size; 64 / row_group_size],
batches
.iter()
.map(|batch| batch.num_rows())
.collect::<Vec<_>>()
);
let mut values = scan_field_values(&batches);
values.sort_by(f64::total_cmp);
assert_eq!((0..64).map(|i| i as f64).collect::<Vec<_>>(), values);
}
}
#[tokio::test]
async fn test_short_series_level_zero_merge_preserves_batches_and_latest_values() {
let mut env = TestEnv::new().await;
let engine = env
.create_engine(MitoConfig {
default_flat_format: true,
..Default::default()
})
.await;
let region_id = RegionId::new(1, 1);
let request = CreateRequestBuilder::new()
.insert_option("memtable.type", "bulk")
.insert_option("max_row_group_row_count", "65536")
.build();
let schema = test_util::rows_schema(&request);
engine
.handle_request(region_id, RegionRequest::Create(request))
.await
.unwrap();
for (start, end, offset) in [(0, 12_000, 0), (6_000, 18_000, 100_000)] {
let rows = (start..end)
.map(|i| {
row(vec![
ValueData::StringValue(format!("{i:05}")),
ValueData::F64Value((i + offset) as f64),
ValueData::TimestampMillisecondValue(0),
])
})
.collect();
test_util::put_rows(
&engine,
region_id,
Rows {
schema: schema.clone(),
rows,
},
)
.await;
test_util::flush_region(&engine, region_id, None).await;
}
let Scanner::Seq(scanner) = engine
.scanner(region_id, ScanRequest::default())
.await
.unwrap()
else {
panic!("expected a sequential scan");
};
assert_eq!(2, scanner.input().files.len());
for file in &scanner.input().files {
assert_eq!(0, file.meta_ref().level);
assert_eq!(12_000, file.meta_ref().num_series);
}
let batches = scanner
.scan_partition(&Default::default(), &ExecutionPlanMetricsSet::default(), 0)
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap();
assert!(
batches.len() < 64,
"short series became {} batches",
batches.len()
);
assert_eq!(
(0..18_000)
.map(|i| if i < 6_000 {
i as f64
} else {
(i + 100_000) as f64
})
.collect::<Vec<_>>(),
scan_field_values(&batches),
);
}
#[tokio::test]
async fn test_reset_state_discards_dynamic_field_pruning() {
for (append_mode, expected_scanner_name) in [(false, "SeqScan"), (true, "UnorderedScan")] {
src/mito2/src/read/flat_projection.rs
+68 -1
View File
@@ -22,7 +22,7 @@ use common_recordbatch::error::{
ArrowComputeSnafu, DataTypesSnafu, ExternalSnafu, NewDfRecordBatchSnafu,
};
use common_recordbatch::{DfRecordBatch, RecordBatch};
use datatypes::arrow::array::Array;
use datatypes::arrow::array::{Array, AsArray};
use datatypes::arrow::datatypes::{DataType as ArrowDataType, Field};
use datatypes::extension::json::is_json2_extension_type;
use datatypes::prelude::{ConcreteDataType, DataType};
@@ -320,6 +320,16 @@ impl FlatProjectionMapper {
let mut arrays = Vec::with_capacity(self.output_schema.num_columns());
for (output_idx, index) in self.batch_indices.iter().enumerate() {
let mut array = batch.column(*index).clone();
if matches!(
self.output_schema.arrow_schema().fields()[output_idx].data_type(),
ArrowDataType::Dictionary(_, _)
) && let Some(dictionary) = array.as_any_dictionary_opt()
&& dictionary.values().len() > dictionary.len()
{
// Small scan slices must not retain an entire SST's tag dictionary in merges.
array = arrow_select::dictionary::garbage_collect_any_dictionary(dictionary)
.context(ArrowComputeSnafu)?;
}
// Cast dictionary values to the target type.
if let ArrowDataType::Dictionary(_key_type, value_type) = array.data_type()
&& !matches!(
@@ -667,6 +677,63 @@ mod tests {
);
}
#[test]
fn test_tag_projection_compacts_dictionary_slices() {
use datatypes::arrow::array::{
DictionaryArray, StringArray, TimestampMillisecondArray, UInt32Array,
};
use datatypes::arrow::datatypes::UInt32Type;
let mut builder = RegionMetadataBuilder::new(RegionId::new(1024, 0));
builder
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new("tag", ConcreteDataType::string_datatype(), true),
semantic_type: SemanticType::Tag,
column_id: 0,
})
.push_column_metadata(ColumnMetadata {
column_schema: ColumnSchema::new(
"ts",
ConcreteDataType::timestamp_millisecond_datatype(),
false,
),
semantic_type: SemanticType::Timestamp,
column_id: 1,
})
.primary_key(vec![0]);
let metadata = Arc::new(builder.build().unwrap());
let mapper = FlatProjectionMapper::new(&metadata, [0, 1])
.unwrap()
.with_pk_dictionary_encoding();
let values = StringArray::from_iter(
(0..1024).map(|i| (i != 1).then(|| format!("{i}{}", "x".repeat(4096)))),
);
let dictionary = DictionaryArray::<UInt32Type>::try_new(
UInt32Array::from(vec![Some(1023), None, Some(1), Some(1023)]),
Arc::new(values),
)
.unwrap();
let expected = datatypes::arrow::compute::cast(&dictionary, &ArrowDataType::Utf8).unwrap();
let batch = DfRecordBatch::try_from_iter([
("tag", Arc::new(dictionary) as Arc<dyn Array>),
(
"ts",
Arc::new(TimestampMillisecondArray::from(vec![1, 2, 3, 4])),
),
])
.unwrap();
let output = mapper.convert(&batch, &CacheStrategy::Disabled).unwrap();
let actual = output.column(0);
assert!(actual.get_array_memory_size() < 64 * 1024);
assert_eq!(actual.data_type(), batch.column(0).data_type());
assert_eq!(
datatypes::arrow::compute::cast(actual.as_ref(), &ArrowDataType::Utf8)
.unwrap()
.as_ref(),
expected.as_ref(),
);
}
#[test]
fn test_single_value_dictionary_preserves_string_type() {
use datatypes::arrow::array::{DictionaryArray, TimestampMillisecondArray, UInt32Array};
src/mito2/src/read/scan_util.rs
+33 -1
View File
@@ -1234,7 +1234,15 @@ pub(crate) fn should_split_flat_batches_for_merge(
let file = &stream_ctx.input.files[file_index];
let file_meta = file.meta_ref();
if file_meta.level == 0 {
// Always split level 0 files.
// Short series can leave too many tiny batches retained by the merge.
if file_meta.num_series > 0 {
if !can_split_series(file_meta.num_rows, file_meta.num_series) {
return None;
}
total_rows += file_meta.num_rows;
total_series += file_meta.num_series;
}
// Older L0 files may lack series statistics; keep their fallback behavior.
num_files_to_split += 1;
continue;
} else if file_meta.num_rows < SPLIT_ROW_THRESHOLD || file_meta.num_series == 0 {
@@ -1913,6 +1921,30 @@ mod tests {
);
}
#[tokio::test]
async fn test_should_split_flat_batches_for_merge_uses_level_zero_series_stats() {
for (num_rows, num_series, expected) in [
(70_144, 35_083, None),
(100_000, 1_000, Some(100)),
(100_000, 0, Some(DEFAULT_READ_BATCH_SIZE / 4)),
] {
let mut meta = new_test_file(num_rows, num_series).meta_ref().clone();
meta.level = 0;
let file = FileHandle::new(meta, Arc::new(NoopFilePurger));
let stream_ctx = new_test_stream_ctx(vec![file], vec![]).await;
let range_meta = new_test_range_meta(smallvec![RowGroupIndex {
index: 0,
row_group_index: 0,
}]);
assert_eq!(
expected,
should_split_flat_batches_for_merge(&stream_ctx, &range_meta),
"rows={num_rows}, series={num_series}",
);
}
}
#[tokio::test]
async fn test_should_split_flat_batches_for_merge_falls_back_to_memtables() {
let stream_ctx = new_test_stream_ctx(vec![], vec![new_test_memtable(5_000, 100)]).await;
src/mito2/src/read/seq_scan.rs
+10 -2
View File
@@ -662,8 +662,6 @@ pub(crate) async fn build_flat_sources(
return Ok(None);
}
let split_batch_size = should_split_flat_batches_for_merge(stream_ctx, range_meta);
let should_split = split_batch_size.is_some();
sources.reserve(num_indices);
let mut ordered_sources = Vec::with_capacity(num_indices);
ordered_sources.resize_with(num_indices, || None);
@@ -737,6 +735,16 @@ pub(crate) async fn build_flat_sources(
}
}
// Queries merge one range at a time; compaction merges sources from all ranges.
let needs_merge =
ordered_sources.iter().flatten().count() > 1 || (compaction && stream_ctx.ranges.len() > 1);
let split_batch_size = if needs_merge {
should_split_flat_batches_for_merge(stream_ctx, range_meta)
} else {
None
};
let should_split = split_batch_size.is_some();
for stream in ordered_sources.into_iter().flatten() {
if should_split {
sources.push(Box::pin(SplitRecordBatchStream::new(stream)));
src/object-store/Cargo.toml
+2 -2
View File
@@ -24,7 +24,7 @@ common-telemetry.workspace = true
derive_builder = { workspace = true, optional = true }
hdfs-native = { workspace = true, optional = true }
humantime-serde.workspace = true
opendal = { version = "0.59.2", features = [
opendal = { version = "0.59.4", features = [
"layers-tracing",
"layers-prometheus",
"services-azblob",
@@ -34,7 +34,7 @@ opendal = { version = "0.59.2", features = [
"services-oss",
"services-s3",
] }
opendal-http-transport-reqwest = "0.59.2"
opendal-http-transport-reqwest = "0.59.4"
prometheus.workspace = true
reqwest.workspace = true
serde.workspace = true
src/object-store/src/layers/mock.rs
+45 -1
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use std::fmt::Debug;
use std::sync::Arc;
use std::sync::{Arc, Mutex};
use derive_builder::Builder;
pub use oio::*;
@@ -225,6 +225,50 @@ impl oio::Read for MockReader {
}
}
/// Writer calls observed before forwarding, including calls that fail.
#[derive(Debug, Default)]
pub struct WriterObservations {
pub write_sizes: Vec<usize>,
pub close_calls: usize,
pub abort_calls: usize,
}
/// Records calls without changing the inner writer's completion or abort semantics.
pub struct ObservedWriter {
inner: oio::Writer,
observations: Arc<Mutex<WriterObservations>>,
}
impl ObservedWriter {
pub fn new(inner: oio::Writer, observations: Arc<Mutex<WriterObservations>>) -> Self {
Self {
inner,
observations,
}
}
}
impl oio::Write for ObservedWriter {
async fn write(&mut self, bytes: Buffer) -> Result<()> {
self.observations
.lock()
.unwrap()
.write_sizes
.push(bytes.len());
self.inner.write(bytes).await
}
async fn close(&mut self) -> Result<Metadata> {
self.observations.lock().unwrap().close_calls += 1;
self.inner.close().await
}
async fn abort(&mut self) -> Result<()> {
self.observations.lock().unwrap().abort_calls += 1;
self.inner.abort().await
}
}
pub struct MockWriter {
inner: oio::Writer,
}
src/operator/src/statement/export_logical_tables.rs
+122 -23
View File
@@ -41,7 +41,7 @@ use common_recordbatch::SendableRecordBatchStream;
use common_time::range::TimestampRange;
use datafusion::datasource::DefaultTableSource;
use datafusion_common::TableReference as DfTableReference;
use datafusion_expr::{LogicalPlan, LogicalPlanBuilder, col};
use datafusion_expr::{LogicalPlan, LogicalPlanBuilder, col, lit};
use futures::StreamExt;
use object_store::ObjectStore;
use session::context::QueryContextRef;
@@ -53,6 +53,7 @@ use store_api::metric_engine_consts::{
use table::TableRef;
use table::metadata::TableId;
use table::table::adapter::DfTableProviderAdapter;
use tokio::sync::OwnedSemaphorePermit;
use tokio_util::sync::CancellationToken;
use crate::error::{self, InvalidLogicalTableExportSnafu, LogicalTableExportResourceSnafu, Result};
@@ -79,7 +80,7 @@ impl Default for LogicalTableExportLimits {
input_batch_bytes: 64 * 1024 * 1024,
conversion_bytes: 1024 * 1024,
writer: ParquetWriterLimits {
row_group_rows: 8192,
row_group_rows: 1024 * 1024,
flush_threshold_bytes: 8 * 1024 * 1024,
max_row_groups: 4096,
},
@@ -255,7 +256,7 @@ impl LogicalTableExport {
fn build_plan(&self, time_range: Option<&TimestampRange>) -> Result<LogicalPlan> {
let info = self.physical_table.table_info();
let filters = self
let mut filters = self
.physical_table
.schema()
.timestamp_column()
@@ -264,6 +265,11 @@ impl LogicalTableExport {
})
.into_iter()
.collect::<Vec<_>>();
if self.logical_tables.len() == 1
&& let Some((&id, _)) = self.logical_tables.first_key_value()
{
filters.push(col(TABLE_ID).eq(lit(id)));
}
let source = Arc::new(DefaultTableSource::new(Arc::new(
DfTableProviderAdapter::new(self.physical_table.clone()),
)));
@@ -283,11 +289,13 @@ impl LogicalTableExport {
.filter(filter)
.context(error::BuildDfLogicalPlanSnafu)?;
}
builder
.sort(vec![col(TABLE_ID).sort(true, false)])
.context(error::BuildDfLogicalPlanSnafu)?
.build()
.context(error::BuildDfLogicalPlanSnafu)
// A single selected table has a constant routing key after filtering.
if self.logical_tables.len() > 1 {
builder = builder
.sort(vec![col(TABLE_ID).sort(true, false)])
.context(error::BuildDfLogicalPlanSnafu)?;
}
builder.build().context(error::BuildDfLogicalPlanSnafu)
}
}
@@ -613,6 +621,46 @@ async fn expand_bounded_slice(
batch.num_columns(),
requested,
)?;
// Parquet accepts compatible string dictionaries without expanding their values.
let fields = schema
.fields()
.iter()
.zip(batch.columns())
.map(|(field, array)| {
if let DataType::Dictionary(_, value) = array.data_type()
&& value.as_ref() == field.data_type()
&& matches!(value.as_ref(), DataType::Utf8 | DataType::LargeUtf8)
{
Arc::new(
field
.as_ref()
.clone()
.with_data_type(array.data_type().clone()),
)
} else {
field.clone()
}
})
.collect::<Vec<_>>();
let schema = Arc::new(arrow::datatypes::Schema::new_with_metadata(
fields,
schema.metadata().clone(),
));
// Sharing compatible arrays needs no CPU conversion. Keep the size check on
// this task only when its dictionary inspection is bounded.
if start < end
&& batch
.columns()
.iter()
.zip(schema.fields())
.all(|(array, field)| array.data_type() == field.data_type())
&& let Some(estimated) = estimate_batch_bytes_upper_bound(&batch, end - start, &[])?
&& estimated <= conversion
{
let reservation = retained.saturating_add(estimated.saturating_mul(4));
let permit = budget.reserve(reservation, cancellation).await?;
return convert_slice_payload(batch, schema, start, end - start, reservation, permit);
}
let input = batch.clone();
let (len, estimated) = common_runtime::spawn_blocking_global(move || {
rows_within_budget(&input, start, end, conversion, &[])
@@ -622,26 +670,36 @@ async fn expand_bounded_slice(
let reservation = retained.saturating_add(estimated.saturating_mul(4));
let permit = budget.reserve(reservation, cancellation).await?;
common_runtime::spawn_blocking_global(move || {
let slice = batch.slice(start, len);
let expanded = expand_export_batch(&slice, schema)?;
ensure!(
expanded.get_array_memory_size() <= reservation,
LogicalTableExportResourceSnafu {
reason: "converted backing buffers exceed reservation"
}
);
Ok((
Payload {
batch: expanded,
permit,
},
len,
))
convert_slice_payload(batch, schema, start, len, reservation, permit)
})
.await
.context(error::JoinTaskSnafu)?
}
fn convert_slice_payload(
batch: RecordBatch,
schema: SchemaRef,
start: usize,
len: usize,
reservation: usize,
permit: OwnedSemaphorePermit,
) -> Result<(Payload, usize)> {
let expanded = expand_export_batch(&batch.slice(start, len), schema)?;
ensure!(
expanded.get_array_memory_size() <= reservation,
LogicalTableExportResourceSnafu {
reason: "converted backing buffers exceed reservation"
}
);
Ok((
Payload {
batch: expanded,
permit,
},
len,
))
}
/// Limits nested casts to selected children: Arrow's List cast otherwise expands
/// the entire values array, even when the parent has been sliced.
pub(crate) fn expand_export_batch(batch: &RecordBatch, schema: SchemaRef) -> Result<RecordBatch> {
@@ -788,6 +846,41 @@ fn estimate_value_size(array: &dyn Array, row: usize) -> Result<usize> {
Ok(bytes.saturating_add(16))
}
/// Estimates a conservative byte upper bound with bounded work on an async task.
/// `None` means this path cannot estimate the batch, not that it exceeds a budget.
fn estimate_batch_bytes_upper_bound(
batch: &RecordBatch,
rows: usize,
json_columns: &[usize],
) -> Result<Option<usize>> {
let mut dictionary_values = 1024usize;
let mut max_row_bytes = 0usize;
for (index, array) in batch.columns().iter().enumerate() {
let size = if let Some(width) = array.data_type().primitive_width() {
(width + 16).max(32)
} else if let Some(dictionary) = array.as_any_dictionary_opt()
&& dictionary.values().len() < rows
&& dictionary.values().len() <= dictionary_values
&& matches!(
dictionary.values().data_type(),
DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View
)
{
let values = dictionary.values();
dictionary_values -= values.len();
let max = (0..values.len()).try_fold(0usize, |max, row| {
Ok::<_, error::Error>(max.max(estimate_value_size(values.as_ref(), row)?))
})?;
max.saturating_add(16).max(32)
} else {
return Ok(None);
};
let expansion = if json_columns.contains(&index) { 8 } else { 1 };
max_row_bytes = max_row_bytes.saturating_add(size.saturating_mul(expansion));
}
Ok(max_row_bytes.checked_mul(rows))
}
pub(crate) fn rows_within_budget(
batch: &RecordBatch,
start: usize,
@@ -795,6 +888,12 @@ pub(crate) fn rows_within_budget(
budget: usize,
json_columns: &[usize],
) -> Result<(usize, usize)> {
if start < end
&& let Some(bytes) = estimate_batch_bytes_upper_bound(batch, end - start, json_columns)?
&& bytes <= budget
{
return Ok((end - start, bytes));
}
let mut bytes = 0usize;
let mut row = start;
while row < end {
src/operator/src/statement/export_logical_tables/tests.rs
+317 -42
View File
@@ -127,6 +127,188 @@ async fn read(store: &ObjectStore, path: &str) -> (SchemaRef, Vec<RecordBatch>)
)
}
#[tokio::test]
async fn single_table_plan_filters_other_routing_ids_before_export() {
let mut unit = unit();
unit.logical_tables.retain(|id, _| *id == 1025);
let schema = unit.physical_table.schema();
let input = expand_export_batch(
&batch(
vec![Some(1027), Some(1025), Some(1024), Some(1025), Some(1026)],
vec![
Some("other"),
Some("selected-a"),
None,
Some("selected-b"),
None,
],
),
schema.arrow_schema().clone(),
)
.unwrap();
unit.physical_table = table::test_util::MemTable::table(
"phy",
GreptimeRecordBatch::from_df_record_batch(schema, input),
);
let context = datafusion::prelude::SessionContext::new_with_config(
datafusion::prelude::SessionConfig::new().with_target_partitions(4),
);
let batches = context
.execute_logical_plan(unit.build_plan(None).unwrap())
.await
.unwrap()
.collect()
.await
.unwrap();
let store = ObjectStore::new(object_store::services::Memory::default()).unwrap();
let summary = export_stream(
&unit,
stream(batches),
&store,
LogicalTableExportLimits::default(),
&CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(summary.rows, 2);
assert_eq!(summary.files, 1);
let (_, output) = read(&store, "cpu.v1.parquet").await;
let hosts = output
.iter()
.flat_map(|batch| batch.column(0).as_string::<i32>().iter())
.collect::<Vec<_>>();
assert_eq!(hosts, vec![Some("selected-a"), Some("selected-b")]);
}
#[tokio::test]
async fn dictionary_null_values_write_the_logical_parquet_schema() {
let input = batch(vec![Some(1025); 4], vec![Some("placeholder"); 4]);
let mut columns = input.columns().to_vec();
columns[2] = Arc::new(
DictionaryArray::<UInt32Type>::try_new(
UInt32Array::from(vec![Some(0), Some(1), None, Some(0)]),
Arc::new(StringArray::from(vec![Some("host"), None])),
)
.unwrap(),
);
let input = RecordBatch::try_new(input.schema(), columns).unwrap();
let unit = unit();
let store = ObjectStore::new(object_store::services::Memory::default()).unwrap();
let summary = export_stream(
&unit,
stream(vec![input]),
&store,
LogicalTableExportLimits::default(),
&CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(summary.rows, 4);
let (schema, output) = read(&store, "cpu.v1.parquet").await;
assert_eq!(schema.fields(), unit.logical_tables[&1025].schema.fields());
let hosts = output
.iter()
.flat_map(|batch| batch.column(0).as_string::<i32>().iter())
.collect::<Vec<_>>();
assert_eq!(hosts, vec![Some("host"), None, None, Some("host")]);
}
#[tokio::test]
async fn dictionary_slices_share_backing_and_respect_budget() {
let input = batch(vec![Some(1025); 8192], vec![Some("host"); 8192])
.project(&[2, 3, 1])
.unwrap();
let schema = unit().logical_tables[&1025].schema.clone();
let budget = ExportWriteBudget::new(1);
let token = CancellationToken::new();
let conversion = expand_bounded_slice(
input.clone(),
schema.clone(),
7,
8192,
1024 * 1024,
&budget,
&token,
);
tokio::pin!(conversion);
// Sharing an admitted batch completes without waiting for a conversion worker.
let std::task::Poll::Ready(result) = futures::poll!(&mut conversion) else {
panic!("compatible slice dispatched a conversion worker");
};
let (payload, rows) = result.unwrap();
assert_eq!(rows, 8185);
let input_values = input.column(0).as_dictionary::<UInt32Type>().values();
let output_values = payload
.batch
.column(0)
.as_dictionary::<UInt32Type>()
.values();
assert!(Arc::ptr_eq(input_values, output_values));
assert!(64 * 1024 * 1024 - budget.available().1 >= input.get_array_memory_size());
drop(payload);
assert_eq!(budget.available(), (1, 64 * 1024 * 1024));
// A batch which does not fit still takes the bounded conversion path.
let (payload, rows) =
expand_bounded_slice(input.clone(), schema.clone(), 7, 8192, 128, &budget, &token)
.await
.unwrap();
assert_eq!(rows, 1);
drop(payload);
assert!(
expand_bounded_slice(input, schema, 7, 8192, 1, &budget, &token)
.await
.is_err()
);
assert_eq!(budget.available(), (1, 64 * 1024 * 1024));
}
#[test]
fn dictionary_size_bound_limits_inline_work() {
let values = (0..1025).map(|i| format!("host-{i}")).collect::<Vec<_>>();
let dictionary = DictionaryArray::<UInt32Type>::new(
UInt32Array::from(vec![0; 2048]),
Arc::new(StringArray::from_iter_values(values.iter())),
);
let input = RecordBatch::try_from_iter([("host", Arc::new(dictionary) as ArrayRef)]).unwrap();
assert!(
estimate_batch_bytes_upper_bound(&input, 2048, &[])
.unwrap()
.is_none()
);
assert_eq!(
rows_within_budget(&input, 0, 2048, 128, &[]).unwrap(),
(3, 114)
);
}
#[tokio::test]
async fn narrow_batches_share_row_groups() {
let store = ObjectStore::new(object_store::services::Memory::default()).unwrap();
let batches = (0..8)
.map(|_| batch(vec![Some(1025); 8192], vec![Some("host"); 8192]))
.collect();
let summary = export_stream(
&unit(),
stream(batches),
&store,
LogicalTableExportLimits::default(),
&CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(summary.rows, 65536);
let bytes = store.read("cpu.v1.parquet").await.unwrap().to_bytes();
let reader = ParquetRecordBatchReaderBuilder::try_new(bytes).unwrap();
assert_eq!(reader.metadata().num_row_groups(), 1);
let rows: usize = reader
.build()
.unwrap()
.map(|batch| batch.unwrap().num_rows())
.sum();
assert_eq!(rows, summary.rows);
}
#[tokio::test]
async fn routes_across_batches_and_writes_empty_files() {
let unit = unit();
@@ -249,6 +431,19 @@ async fn existing_outputs_are_not_overwritten() {
);
}
#[test]
fn repeated_dictionary_budget_preserves_null_and_offset_charges() {
let host = "x".repeat(64);
let input = batch(vec![Some(1025); 8], vec![Some(host.as_str()); 8])
.project(&[2, 4, 1])
.unwrap();
// Each row charges 96 bytes for the dictionary value, 32 for null, and 24 for time.
assert_eq!(rows_within_budget(&input, 0, 8, 1215, &[]).unwrap().0, 7);
let (rows, estimated) = rows_within_budget(&input, 0, 8, 1280, &[]).unwrap();
assert_eq!(rows, 8);
assert!((1216..=1280).contains(&estimated));
}
#[test]
fn dictionary_and_nested_histogram_values_are_bounded_before_expansion() {
let dictionary = DictionaryArray::<UInt32Type>::try_new(
@@ -291,6 +486,25 @@ fn validates_selected_schemas_and_projects_only_selected_columns() {
.unwrap();
assert_eq!(one.scan_projection, vec![0, 3]);
assert_eq!(one.logical_tables[&1025].projection, vec![1]);
let LogicalPlan::Filter(filter) = one.build_plan(None).unwrap() else {
panic!("single-table export must filter the routing key without sorting");
};
let datafusion_expr::Expr::BinaryExpr(predicate) = filter.predicate else {
panic!("expected table-id equality");
};
assert_eq!(predicate.op, datafusion_expr::Operator::Eq);
assert!(matches!(
predicate.left.as_ref(),
datafusion_expr::Expr::Column(column) if column.name == TABLE_ID
));
assert!(matches!(
predicate.right.as_ref(),
datafusion_expr::Expr::Literal(datafusion_common::ScalarValue::UInt32(Some(1025)), _)
));
assert!(matches!(
unit.build_plan(None).unwrap(),
LogicalPlan::Sort(_)
));
assert!(
LogicalTableExport::try_new(unit.physical_table.clone(), &[selected.clone(), selected])
.is_err()
@@ -333,7 +547,7 @@ async fn cancellation_drops_input_and_aborts_active_upload() {
}
#[tokio::test]
async fn native_histogram_parquet_roundtrip() {
async fn supported_metric_types_parquet_roundtrip() {
fn sample(data_type: &DataType) -> ArrayRef {
match data_type {
DataType::Struct(fields) => Arc::new(StructArray::new(
@@ -359,44 +573,102 @@ async fn native_histogram_parquet_roundtrip() {
other => panic!("unexpected histogram field {other}"),
}
}
let data_type = common_query::native_histogram::native_histogram_arrow_type();
let field = Field::new("histogram", data_type.clone(), true);
let unit = LogicalTableExport::try_new(
table(
1024,
"phy",
vec![Field::new(TABLE_ID, DataType::UInt32, false), field.clone()],
true,
),
&[table(1025, "histogram", vec![field], false)],
)
.unwrap();
let batch = RecordBatch::try_from_iter_with_nullable([
(
TABLE_ID,
Arc::new(UInt32Array::from(vec![1025; 2])) as ArrayRef,
false,
),
("histogram", sample(&data_type), true),
])
.unwrap();
let store = ObjectStore::new(object_store::services::Memory::default()).unwrap();
let summary = export_stream(
&unit,
stream(vec![batch.clone()]),
&store,
export_limits(),
&CancellationToken::new(),
)
.await
.unwrap();
assert_eq!(summary.rows, 2);
let (schema, batches) = read(&store, "histogram.parquet").await;
assert_eq!(schema.fields(), unit.logical_tables[&1025].schema.fields());
assert_eq!(
arrow::compute::concat_batches(&schema, &batches).unwrap(),
batch.project(&[1]).unwrap()
);
use common_datasource::file_format::parquet::packed_reader::{
PackReadWindows, PackedParquetReader,
};
use common_datasource::packed_snapshot::{PACK_INDEX_FILE, PackIndex};
use common_datasource::packed_writer::PackedWriter;
use datafusion::parquet::arrow::ParquetRecordBatchStreamBuilder;
use futures::TryStreamExt;
let timestamps: [ArrayRef; 4] = [
Arc::new(arrow::array::TimestampSecondArray::from(vec![4, 5])),
Arc::new(arrow::array::TimestampMillisecondArray::from(vec![4, 5])),
Arc::new(arrow::array::TimestampMicrosecondArray::from(vec![4, 5])),
Arc::new(arrow::array::TimestampNanosecondArray::from(vec![4, 5])),
];
let values = [
Arc::new(Float64Array::from(vec![Some(3.5), None])) as ArrayRef,
sample(&common_query::native_histogram::native_histogram_arrow_type()),
];
for timestamp in timestamps {
for value in &values {
let logical = RecordBatch::try_from_iter_with_nullable([
("ts", timestamp.clone(), false),
(
"host",
Arc::new(StringArray::from(vec![Some(""), None])) as ArrayRef,
true,
),
("value", value.clone(), true),
])
.unwrap();
let mut fields = vec![Field::new(TABLE_ID, DataType::UInt32, false)];
fields.extend(logical.schema().fields().iter().map(|f| f.as_ref().clone()));
let unit = LogicalTableExport::try_new(
table(1024, "phy", fields.clone(), true),
&[table(1025, "metric", fields[1..].to_vec(), false)],
)
.unwrap();
let mut columns = vec![Arc::new(UInt32Array::from(vec![1025; 2])) as ArrayRef];
columns.extend_from_slice(logical.columns());
let input = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).unwrap();
for packed in [false, true] {
let store = ObjectStore::new(object_store::services::Memory::default()).unwrap();
let writer = packed.then(|| PackedWriter::new(store.clone()).unwrap());
let summary = export_stream_managed(
&unit,
stream(vec![input.clone()]),
&store,
export_limits(),
&CancellationToken::new(),
ExportWriteBudget::new(1),
writer.clone(),
)
.await
.unwrap();
assert_eq!(summary.rows, 2);
let batches = if let Some(writer) = writer {
writer
.lock()
.await
.finish(&CancellationToken::new())
.await
.unwrap();
let index: PackIndex = serde_json::from_slice(
&store.read(PACK_INDEX_FILE).await.unwrap().to_bytes(),
)
.unwrap();
index.validate_membership(["metric"]).unwrap();
let entry = &index.tables[0];
let object = &index.objects[0];
let reader = PackedParquetReader::new(
PackReadWindows::new(store.clone()),
object.path.clone(),
object.length,
entry.offset,
entry.length,
)
.unwrap();
ParquetRecordBatchStreamBuilder::new(reader)
.await
.unwrap()
.build()
.unwrap()
.try_collect::<Vec<_>>()
.await
.unwrap()
} else {
read(&store, "metric.parquet").await.1
};
assert_eq!(batches[0].schema().fields(), logical.schema().fields());
assert_eq!(
arrow::compute::concat_batches(&logical.schema(), &batches).unwrap(),
logical
);
}
}
}
}
struct PausedFileWriter {
@@ -748,9 +1020,12 @@ async fn retained_backing_is_reserved_before_conversion_and_until_payload_drop()
drop(blocker);
let (payload, rows) = convert.await.unwrap();
assert_eq!(rows, 1);
assert!(
64 * 1024 * 1024 - budget.available().1 >= full + payload.batch.get_array_memory_size()
);
assert!(matches!(
payload.batch.column(0).data_type(),
DataType::Dictionary(_, _)
));
// The retained dictionary shares its input buffers, so they are charged once.
assert!(64 * 1024 * 1024 - budget.available().1 >= full);
drop(payload);
assert_eq!(budget.available(), (1, 64 * 1024 * 1024));
assert!(budget.reserve(64 * 1024 * 1024 + 1, &token).await.is_err());
src/promql/src/functions.rs
+2
View File
@@ -19,6 +19,7 @@ mod double_exponential_smoothing;
mod edge_count;
mod extrapolate_rate;
mod idelta;
mod ieee_float;
mod native_histogram;
mod predict_linear;
mod quantile;
@@ -45,6 +46,7 @@ pub use deriv::Deriv;
pub use double_exponential_smoothing::DoubleExponentialSmoothing;
pub use extrapolate_rate::{Delta, Increase, Rate};
pub use idelta::IDelta;
pub use ieee_float::{IeeeComparison, IeeeSqrt};
pub use native_histogram::{
MixedRange, NativeHistogramAbsentOverTime, NativeHistogramAdd, NativeHistogramAggAvg,
NativeHistogramAggSum, NativeHistogramAvg, NativeHistogramAvgOverTime, NativeHistogramChanges,
src/promql/src/functions/ieee_float.rs
+265
View File
@@ -0,0 +1,265 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! PromQL comparison operators and `sqrt()`, which follow IEEE 754.
//!
//! Arrow compares floats by total order, where NaN equals NaN and is greater than `+Inf`, and
//! DataFusion's `sqrt` rejects negative input. In PromQL every comparison involving NaN is false
//! except `!=`, and the square root of a negative number is NaN. NULL input gives NULL output.
//!
//! The functions report the column name of the expression they replace (`a < b`, `sqrt(a)`), so
//! query output columns are unchanged.
use std::sync::Arc;
use datafusion::arrow::array::{AsArray, BooleanArray, Float64Array};
use datafusion::arrow::datatypes::{DataType, Float64Type};
use datafusion::error::Result as DfResult;
use datafusion::logical_expr::{
ColumnarValue, Operator, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature, Volatility,
};
use datafusion_expr::Expr;
/// A PromQL comparison operator on two float operands.
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct IeeeComparison {
op: Operator,
signature: Signature,
}
impl IeeeComparison {
/// The comparison UDF for `op`, which must be a comparison operator.
pub fn scalar_udf(op: Operator) -> ScalarUDF {
ScalarUDF::new_from_impl(Self {
op,
signature: Signature::uniform(2, vec![DataType::Float64], Volatility::Immutable),
})
}
/// All comparison UDFs, for registering them where plans are decoded.
pub fn all() -> Vec<ScalarUDF> {
[
Operator::Eq,
Operator::NotEq,
Operator::Lt,
Operator::LtEq,
Operator::Gt,
Operator::GtEq,
]
.into_iter()
.map(Self::scalar_udf)
.collect()
}
fn compare(&self, lhs: f64, rhs: f64) -> bool {
match self.op {
Operator::Eq => lhs == rhs,
Operator::NotEq => lhs != rhs,
Operator::Lt => lhs < rhs,
Operator::LtEq => lhs <= rhs,
Operator::Gt => lhs > rhs,
Operator::GtEq => lhs >= rhs,
_ => unreachable!("not a comparison operator: {}", self.op),
}
}
}
impl ScalarUDFImpl for IeeeComparison {
fn name(&self) -> &str {
match self.op {
Operator::Eq => "prom_eq",
Operator::NotEq => "prom_ne",
Operator::Lt => "prom_lt",
Operator::LtEq => "prom_le",
Operator::Gt => "prom_gt",
Operator::GtEq => "prom_ge",
_ => unreachable!("not a comparison operator: {}", self.op),
}
}
fn signature(&self) -> &Signature {
&self.signature
}
fn return_type(&self, _arg_types: &[DataType]) -> DfResult<DataType> {
Ok(DataType::Boolean)
}
fn schema_name(&self, args: &[Expr]) -> DfResult<String> {
Ok(format!(
"{} {} {}",
args[0].schema_name(),
self.op,
args[1].schema_name()
))
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DfResult<ColumnarValue> {
let arrays = ColumnarValue::values_to_arrays(&args.args)?;
let lhs = arrays[0].as_primitive::<Float64Type>();
let rhs = arrays[1].as_primitive::<Float64Type>();
let result: BooleanArray = lhs
.iter()
.zip(rhs.iter())
.map(|(lhs, rhs)| Some(self.compare(lhs?, rhs?)))
.collect();
Ok(ColumnarValue::Array(Arc::new(result)))
}
}
/// PromQL `sqrt()`.
#[derive(Debug, PartialEq, Eq, Hash)]
pub struct IeeeSqrt {
signature: Signature,
}
impl IeeeSqrt {
pub fn scalar_udf() -> ScalarUDF {
ScalarUDF::new_from_impl(Self {
signature: Signature::uniform(1, vec![DataType::Float64], Volatility::Immutable),
})
}
}
impl ScalarUDFImpl for IeeeSqrt {
fn name(&self) -> &str {
"prom_sqrt"
}
fn signature(&self) -> &Signature {
&self.signature
}
fn return_type(&self, _arg_types: &[DataType]) -> DfResult<DataType> {
Ok(DataType::Float64)
}
fn schema_name(&self, args: &[Expr]) -> DfResult<String> {
Ok(format!("sqrt({})", args[0].schema_name()))
}
fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DfResult<ColumnarValue> {
let arrays = ColumnarValue::values_to_arrays(&args.args)?;
let input = arrays[0].as_primitive::<Float64Type>();
let result: Float64Array = input.unary(f64::sqrt);
Ok(ColumnarValue::Array(Arc::new(result)))
}
}
#[cfg(test)]
mod tests {
use datafusion::arrow::array::{Array, ArrayRef};
use datafusion::arrow::datatypes::Field;
use datafusion_common::config::ConfigOptions;
use super::*;
fn invoke(udf: &ScalarUDF, args: Vec<ArrayRef>, return_type: DataType) -> ArrayRef {
let number_rows = args[0].len();
let arg_fields = args
.iter()
.map(|array| Arc::new(Field::new("a", array.data_type().clone(), true)))
.collect();
let result = udf
.invoke_with_args(ScalarFunctionArgs {
args: args.into_iter().map(ColumnarValue::Array).collect(),
arg_fields,
number_rows,
return_field: Arc::new(Field::new("r", return_type, true)),
config_options: Arc::new(ConfigOptions::default()),
})
.unwrap();
match result {
ColumnarValue::Array(array) => array,
ColumnarValue::Scalar(_) => unreachable!(),
}
}
#[test]
fn comparisons_follow_ieee_and_propagate_null() {
let lhs: ArrayRef = Arc::new(Float64Array::from(vec![
Some(1.0),
Some(f64::NAN),
Some(f64::NAN),
None,
Some(f64::INFINITY),
]));
let rhs: ArrayRef = Arc::new(Float64Array::from(vec![
Some(1.0),
Some(f64::NAN),
Some(1.0),
Some(f64::NAN),
Some(f64::NAN),
]));
let expected = [
(
Operator::Eq,
[Some(true), Some(false), Some(false), None, Some(false)],
),
(
Operator::NotEq,
[Some(false), Some(true), Some(true), None, Some(true)],
),
(
Operator::Lt,
[Some(false), Some(false), Some(false), None, Some(false)],
),
(
Operator::LtEq,
[Some(true), Some(false), Some(false), None, Some(false)],
),
(
Operator::Gt,
[Some(false), Some(false), Some(false), None, Some(false)],
),
(
Operator::GtEq,
[Some(true), Some(false), Some(false), None, Some(false)],
),
];
for (op, expected) in expected {
let result = invoke(
&IeeeComparison::scalar_udf(op),
vec![lhs.clone(), rhs.clone()],
DataType::Boolean,
);
let result: Vec<_> = result.as_boolean().iter().collect();
assert_eq!(result, expected, "{op}");
}
}
#[test]
fn sqrt_of_negative_is_nan() {
let input: ArrayRef = Arc::new(Float64Array::from(vec![Some(4.0), Some(-1.0), None]));
let result = invoke(&IeeeSqrt::scalar_udf(), vec![input], DataType::Float64);
let result = result.as_primitive::<Float64Type>();
assert_eq!(result.value(0), 2.0);
assert!(result.value(1).is_nan());
assert!(result.is_null(2));
}
#[test]
fn column_names_match_the_replaced_expressions() {
let a = datafusion_expr::col("a");
let b = datafusion_expr::lit(1.5_f64);
let comparison =
IeeeComparison::scalar_udf(Operator::LtEq).call(vec![a.clone(), b.clone()]);
assert_eq!(
comparison.schema_name().to_string(),
a.clone().lt_eq(b).schema_name().to_string()
);
let sqrt = IeeeSqrt::scalar_udf().call(vec![a.clone()]);
assert_eq!(sqrt.schema_name().to_string(), "sqrt(a)");
}
}
src/promql/src/functions/round.rs
+44 -18
View File
@@ -66,13 +66,7 @@ impl Round {
let null_array = Float64Array::new_null(value_array.len());
return Ok(ColumnarValue::Array(Arc::new(null_array)));
};
let op = |a: f64| {
if nearest == 0.0 {
a.round()
} else {
(a / nearest).round() * nearest
}
};
let op = |a: f64| round_to_nearest(a, nearest);
let result: PrimitiveArray<Float64Type> =
value_array.as_primitive::<Float64Type>().unary(op);
Ok(ColumnarValue::Array(Arc::new(result) as _))
@@ -90,14 +84,10 @@ impl Round {
)?;
let result: PrimitiveArray<Float64Type> =
datatypes::arrow::compute::binary(value_array, nearest_array, |a, nearest| {
if nearest == 0.0 {
a.round()
} else {
(a / nearest).round() * nearest
}
})
.map_err(|err: ArrowError| DataFusionError::ArrowError(Box::new(err), None))?;
datatypes::arrow::compute::binary(value_array, nearest_array, round_to_nearest)
.map_err(|err: ArrowError| {
DataFusionError::ArrowError(Box::new(err), None)
})?;
Ok(ColumnarValue::Array(Arc::new(result) as _))
}
@@ -105,6 +95,13 @@ impl Round {
}
}
/// Rounds like Prometheus: ties round up (towards +Inf), and multiplying by the inverse
/// of `nearest` follows its float behavior, including NaN for `nearest == 0`.
fn round_to_nearest(value: f64, nearest: f64) -> f64 {
let inverse = 1.0 / nearest;
(value * inverse + 0.5).floor() / inverse
}
#[cfg(test)]
mod tests {
use datafusion_common::config::ConfigOptions;
@@ -143,14 +140,43 @@ mod tests {
#[test]
fn test_round() {
// Expected values are Prometheus 3.15 `round()` results.
test_round_f64(vec![123.456], 0.001, vec![123.456]);
test_round_f64(vec![123.456], 0.01, vec![123.46000000000001]);
test_round_f64(vec![123.456], 0.01, vec![123.46]);
test_round_f64(vec![123.456], 0.1, vec![123.5]);
test_round_f64(vec![123.456], 0.0, vec![123.0]);
test_round_f64(vec![123.456], 1.0, vec![123.0]);
test_round_f64(vec![123.456], 10.0, vec![120.0]);
test_round_f64(vec![123.456], 100.0, vec![100.0]);
test_round_f64(vec![123.456], 105.0, vec![105.0]);
test_round_f64(vec![123.456], 105.0, vec![104.99999999999999]);
test_round_f64(vec![123.456], 1000.0, vec![0.0]);
}
#[test]
fn test_round_ties_towards_positive_infinity() {
test_round_f64(vec![-0.5], 1.0, vec![0.0]);
test_round_f64(vec![-2.5], 1.0, vec![-2.0]);
test_round_f64(vec![2.5], 1.0, vec![3.0]);
}
#[test]
fn test_round_to_zero_is_nan() {
let round_udf = Round::scalar_udf();
let input = vec![
ColumnarValue::Array(Arc::new(Float64Array::from(vec![123.456]))),
ColumnarValue::Scalar(ScalarValue::Float64(Some(0.0))),
];
let arg_fields = vec![
Arc::new(Field::new("a", input[0].data_type(), false)),
Arc::new(Field::new("b", input[1].data_type(), false)),
];
let args = ScalarFunctionArgs {
args: input,
arg_fields,
number_rows: 1,
return_field: Arc::new(Field::new("c", DataType::Float64, false)),
config_options: Arc::new(ConfigOptions::default()),
};
let result = extract_array(&round_udf.invoke_with_args(args).unwrap()).unwrap();
assert!(result.as_primitive::<Float64Type>().value(0).is_nan());
}
}
src/query/src/optimizer/constant_term.rs
+8 -4
View File
@@ -201,10 +201,14 @@ impl PhysicalOptimizerRule for MatchesConstantTermOptimizer {
})?;
if new_pred.transformed {
let exec = FilterExecBuilder::new(new_pred.data, filter.input().clone())
.with_default_selectivity(filter.default_selectivity())
.apply_projection_by_ref(filter.projection().as_ref())
.and_then(|x| x.build())?;
// `FilterExecBuilder::from(&FilterExec)` already carries over the
// input, projection, default selectivity, batch size and fetch, so
// only the predicate has to be replaced. Re-applying the projection
// is not only pointless but wrong: `apply_projection` composes with
// the projection copied from the builder.
let exec = FilterExecBuilder::from(filter)
.with_predicate(new_pred.data)
.build()?;
return Ok(Transformed::yes(Arc::new(exec) as _));
}
}
src/query/src/optimizer/count_nest_aggr.rs
+15 -4
View File
@@ -15,6 +15,8 @@
use std::collections::HashSet;
use std::sync::Arc;
use common_function::aggrs::aggr_wrapper::get_aggr_func;
use common_function::aggrs::extremum::Extremum;
use datafusion::config::ConfigOptions;
use datafusion::functions_aggregate::count::count_udaf;
use datafusion::logical_expr::{Extension, LogicalPlan, LogicalPlanBuilder, Sort};
@@ -243,9 +245,8 @@ impl CountNestAggrRule {
where
F: FnOnce(&str) -> bool,
{
let Expr::AggregateFunction(func) = expr else {
return None;
};
// The PromQL planner aliases `prom_min`/`prom_max` to the builtin names.
let func = get_aggr_func(expr)?;
let name = func.func.name();
if !accept_name(name)
|| func.params.filter.is_some()
@@ -260,9 +261,19 @@ impl CountNestAggrRule {
}
fn is_supported_inner_aggregate(name: &str) -> bool {
// PromQL `min`/`max` plan to `prom_min`/`prom_max`, which return NULL for the same
// groups as the builtin ones.
matches!(
name,
"count" | "sum" | "avg" | "min" | "max" | "stddev_pop" | "var_pop"
"count"
| "sum"
| "avg"
| "min"
| "max"
| Extremum::MIN_NAME
| Extremum::MAX_NAME
| "stddev_pop"
| "var_pop"
)
}
src/query/src/promql/planner.rs
+89 -33
View File
@@ -27,6 +27,7 @@ use async_recursion::async_recursion;
use catalog::table_source::DfTableSourceProvider;
use common_error::ext::ErrorExt;
use common_error::status_code::StatusCode;
use common_function::aggrs::extremum::Extremum;
use common_function::function::FunctionContext;
use common_query::native_histogram::native_histogram_value_type;
use common_query::prelude::{
@@ -70,18 +71,18 @@ use promql::extension_plan::{
};
use promql::functions::{
AbsentOverTime, AvgOverTime, Changes, CountOverTime, Delta, Deriv, DoubleExponentialSmoothing,
IDelta, Increase, LastOverTime, MatchGroupViolation, MaxOverTime, MinOverTime, MixedRange,
NativeHistogramAbsentOverTime, NativeHistogramAdd, NativeHistogramAggAvg,
NativeHistogramAggSum, NativeHistogramAvg, NativeHistogramAvgOverTime, NativeHistogramChanges,
NativeHistogramCount, NativeHistogramCountOverTime, NativeHistogramDelta,
NativeHistogramDivScalar, NativeHistogramDrop, NativeHistogramEq, NativeHistogramIDelta,
NativeHistogramIRate, NativeHistogramIncrease, NativeHistogramLastOverTime,
NativeHistogramMulScalar, NativeHistogramNeg, NativeHistogramNotEq,
NativeHistogramPresentOverTime, NativeHistogramRate, NativeHistogramResets,
NativeHistogramScalarMul, NativeHistogramStddev, NativeHistogramStdvar, NativeHistogramSub,
NativeHistogramSum, NativeHistogramSumOverTime, NativeHistogramToString, PredictLinear,
PresentOverTime, PromqlFloatToString, QuantileOverTime, Rate, Resets, Round, StddevOverTime,
StdvarOverTime, SumOverTime, UniqueMatchGroup, quantile_udaf,
IDelta, IeeeComparison, IeeeSqrt, Increase, LastOverTime, MatchGroupViolation, MaxOverTime,
MinOverTime, MixedRange, NativeHistogramAbsentOverTime, NativeHistogramAdd,
NativeHistogramAggAvg, NativeHistogramAggSum, NativeHistogramAvg, NativeHistogramAvgOverTime,
NativeHistogramChanges, NativeHistogramCount, NativeHistogramCountOverTime,
NativeHistogramDelta, NativeHistogramDivScalar, NativeHistogramDrop, NativeHistogramEq,
NativeHistogramIDelta, NativeHistogramIRate, NativeHistogramIncrease,
NativeHistogramLastOverTime, NativeHistogramMulScalar, NativeHistogramNeg,
NativeHistogramNotEq, NativeHistogramPresentOverTime, NativeHistogramRate,
NativeHistogramResets, NativeHistogramScalarMul, NativeHistogramStddev, NativeHistogramStdvar,
NativeHistogramSub, NativeHistogramSum, NativeHistogramSumOverTime, NativeHistogramToString,
PredictLinear, PresentOverTime, PromqlFloatToString, QuantileOverTime, Rate, Resets, Round,
StddevOverTime, StdvarOverTime, SumOverTime, UniqueMatchGroup, quantile_udaf,
};
use promql_parser::label::{METRIC_NAME, MatchOp, Matcher, Matchers};
use promql_parser::parser::token::TokenType;
@@ -595,9 +596,10 @@ impl PromPlanner {
};
let current_interval = self.ctx.interval;
if let Some(step) = step {
self.ctx.interval = step.as_millis() as _;
}
self.ctx.interval = match step {
Some(step) => step.as_millis() as _,
None => DEFAULT_SUBQUERY_STEP_MS,
};
ensure!(self.ctx.interval > 0, ZeroRangeSelectorSnafu);
let current_start = self.ctx.start;
let current_end = self.ctx.end;
@@ -4830,10 +4832,14 @@ impl PromPlanner {
}
"round" => {
if other_input_exprs.is_empty() {
other_input_exprs.push_front(0.0f64.lit());
other_input_exprs.push_front(1.0f64.lit());
}
ScalarFunc::DataFusionUdf(Arc::new(Round::scalar_udf()))
}
// DataFusion's `sqrt` rejects negative input; PromQL returns NaN for it.
"sqrt" if !self.all_field_columns_are_native_histograms(input_schema) => {
ScalarFunc::DataFusionUdf(Arc::new(IeeeSqrt::scalar_udf()))
}
"rad" | "deg" | "sgn" if self.all_field_columns_are_native_histograms(input_schema) => {
ScalarFunc::DataFusionUdf(native_histogram_drop_udf(func.name))
}
@@ -5683,19 +5689,41 @@ impl PromPlanner {
vec![]
};
// update value column name according to the aggregators,
let mut new_field_columns = Vec::with_capacity(self.ctx.field_columns.len());
// The alias is built from normalized columns so it matches the builtin's
// qualified name.
let exprs = normalize_cols(exprs, input_plan)
.context(DataFusionPlanningSnafu)?
.into_iter()
.map(Self::alias_extremum_as_builtin)
.collect::<Vec<_>>();
let normalized_exprs =
normalize_cols(exprs.iter().cloned(), input_plan).context(DataFusionPlanningSnafu)?;
for expr in normalized_exprs {
new_field_columns.push(expr.schema_name().to_string());
}
self.ctx.field_columns = new_field_columns;
// update value column name according to the aggregators,
self.ctx.field_columns = exprs
.iter()
.map(|expr| expr.schema_name().to_string())
.collect();
Ok((exprs, prev_field_exprs))
}
/// `prom_min`/`prom_max` keep their own names so SQL can use them next to `min`/`max`
/// in one aggregate. PromQL output columns keep the builtin names.
fn alias_extremum_as_builtin(expr: DfExpr) -> DfExpr {
let DfExpr::AggregateFunction(aggr) = &expr else {
return expr;
};
let builtin = match aggr.func.name() {
Extremum::MIN_NAME => min_udaf(),
Extremum::MAX_NAME => max_udaf(),
_ => return expr,
};
let name = builtin
.call(aggr.params.args.clone())
.schema_name()
.to_string();
expr.alias(name)
}
fn create_numeric_aggregate_expr(
op: TokenType,
param: &Option<Box<PromExpr>>,
@@ -5713,8 +5741,8 @@ impl PromPlanner {
}
token::T_AVG => avg_udaf().call(vec![input]),
token::T_COUNT_VALUES | token::T_COUNT => count_udaf().call(vec![input]),
token::T_MIN => min_udaf().call(vec![input]),
token::T_MAX => max_udaf().call(vec![input]),
token::T_MIN => Extremum::min_udaf().call(vec![input]),
token::T_MAX => Extremum::max_udaf().call(vec![input]),
// PromQL's `group()` aggregator produces 1 for each group.
// Use `max(1.0)` (per-group) to match semantics and output type (Float64).
token::T_GROUP => max_udaf().call(vec![lit(1_f64)]),
@@ -6058,7 +6086,15 @@ impl PromPlanner {
.field_columns
.iter()
.map(|col| {
let mut sort_exprs = Vec::with_capacity(self.ctx.tag_columns.len() + 1);
let mut sort_exprs = Vec::with_capacity(self.ctx.tag_columns.len() + 2);
// PromQL ranks NaN last for both `topk` and `bottomk`, while DataFusion
// orders NaN above every number.
sort_exprs.push(
datafusion_functions::math::expr_fn::isnan(DfExpr::Column(Column::from_name(
col,
)))
.sort(true, true),
);
// Order by value in the specific order
sort_exprs.push(DfExpr::Column(Column::from_name(col)).sort(asc, true));
// Then tags if the values are equal,
@@ -6249,12 +6285,24 @@ impl PromPlanner {
token::T_MOD => Ok(Box::new(move |lhs: DfExpr, rhs| {
Ok(cast_float(lhs) % cast_float(rhs))
})),
token::T_EQLC => Ok(Box::new(|lhs, rhs| Ok(lhs.eq(rhs)))),
token::T_NEQ => Ok(Box::new(|lhs, rhs| Ok(lhs.not_eq(rhs)))),
token::T_GTR => Ok(Box::new(|lhs, rhs| Ok(lhs.gt(rhs)))),
token::T_LSS => Ok(Box::new(|lhs, rhs| Ok(lhs.lt(rhs)))),
token::T_GTE => Ok(Box::new(|lhs, rhs| Ok(lhs.gt_eq(rhs)))),
token::T_LTE => Ok(Box::new(|lhs, rhs| Ok(lhs.lt_eq(rhs)))),
token::T_EQLC => Ok(Box::new(|lhs, rhs| {
Ok(Self::ieee_comparison(lhs, Operator::Eq, rhs))
})),
token::T_NEQ => Ok(Box::new(|lhs, rhs| {
Ok(Self::ieee_comparison(lhs, Operator::NotEq, rhs))
})),
token::T_GTR => Ok(Box::new(|lhs, rhs| {
Ok(Self::ieee_comparison(lhs, Operator::Gt, rhs))
})),
token::T_LSS => Ok(Box::new(|lhs, rhs| {
Ok(Self::ieee_comparison(lhs, Operator::Lt, rhs))
})),
token::T_GTE => Ok(Box::new(|lhs, rhs| {
Ok(Self::ieee_comparison(lhs, Operator::GtEq, rhs))
})),
token::T_LTE => Ok(Box::new(|lhs, rhs| {
Ok(Self::ieee_comparison(lhs, Operator::LtEq, rhs))
})),
token::T_POW => Ok(Box::new(move |lhs, rhs| {
Ok(DfExpr::ScalarFunction(ScalarFunction {
func: datafusion_functions::math::power(),
@@ -6271,6 +6319,14 @@ impl PromPlanner {
}
}
/// Compares two expressions with IEEE 754 semantics, as PromQL does; see [`IeeeComparison`].
fn ieee_comparison(lhs: DfExpr, op: Operator, rhs: DfExpr) -> DfExpr {
DfExpr::ScalarFunction(ScalarFunction {
func: Arc::new(IeeeComparison::scalar_udf(op)),
args: vec![lhs, rhs],
})
}
/// Check if the given op is a [comparison operator](https://prometheus.io/docs/prometheus/latest/querying/operators/#comparison-binary-operators).
fn is_token_a_comparison_op(token: TokenType) -> bool {
matches!(
src/query/src/promql/planner/test.rs
+44 -21
View File
@@ -1827,7 +1827,20 @@ async fn single_sort_desc() {
#[tokio::test]
async fn single_sqrt() {
do_single_instant_function_call("sqrt", "sqrt").await;
// Negative input is mapped to NaN before DataFusion's `sqrt`, which rejects it.
indie_query_plan_compare(
r#"sqrt(some_metric{tag_0!="bar"})"#,
"Projection: some_metric.timestamp, sqrt(field_0), some_metric.tag_0 [timestamp:Timestamp(ms), sqrt(field_0):Float64;N, tag_0:Utf8]\
\n Filter: sqrt(field_0) IS NOT NULL [timestamp:Timestamp(ms), sqrt(field_0):Float64;N, tag_0:Utf8, __promql_metric_name:Utf8]\
\n Projection: some_metric.timestamp, prom_sqrt(some_metric.field_0) AS sqrt(field_0), some_metric.tag_0, __promql_metric_name [timestamp:Timestamp(ms), sqrt(field_0):Float64;N, tag_0:Utf8, __promql_metric_name:Utf8]\
\n Projection: some_metric.tag_0, some_metric.timestamp, some_metric.field_0, Utf8(\"some_metric\") AS __promql_metric_name [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_metric_name:Utf8]\
\n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\
\n PromSeriesDivide: tags=[\"tag_0\"] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\
\n Sort: some_metric.tag_0 ASC NULLS FIRST, some_metric.timestamp ASC NULLS FIRST [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\
\n Filter: some_metric.tag_0 != Utf8(\"bar\") AND some_metric.timestamp >= TimestampMillisecond(-999, None) AND some_metric.timestamp <= TimestampMillisecond(100000000, None) [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\
\n TableScan: some_metric [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]",
)
.await;
}
#[tokio::test]
@@ -1959,6 +1972,19 @@ async fn single_rad() {
// },
// },
async fn do_aggregate_expr_plan(fn_name: &str, plan_name: &str) {
do_aggregate_expr_plan_with_display(fn_name, plan_name, plan_name).await;
}
/// `display_name` is the aggregate function as the plan shows it, `plan_name` the name of the
/// output column; they differ for PromQL `min`/`max`.
async fn do_aggregate_expr_plan_with_display(fn_name: &str, plan_name: &str, display_name: &str) {
let display = |field: &str| {
if display_name == plan_name {
format!("{plan_name}({field})")
} else {
format!("{display_name}({field}) AS {plan_name}({field})")
}
};
let prom_expr = parser::parse(&format!(
"{fn_name} by (tag_1)(some_metric{{tag_0!=\"bar\"}})",
))
@@ -1985,14 +2011,17 @@ async fn do_aggregate_expr_plan(fn_name: &str, plan_name: &str) {
.unwrap();
let expected_no_without = String::from(
"Sort: some_metric.tag_1 ASC NULLS LAST, some_metric.timestamp ASC NULLS LAST [tag_1:Utf8, timestamp:Timestamp(ms), TEMPLATE(some_metric.field_0):Float64;N, TEMPLATE(some_metric.field_1):Float64;N]\
\n Aggregate: groupBy=[[some_metric.tag_1, some_metric.timestamp]], aggr=[[TEMPLATE(some_metric.field_0), TEMPLATE(some_metric.field_1)]] [tag_1:Utf8, timestamp:Timestamp(ms), TEMPLATE(some_metric.field_0):Float64;N, TEMPLATE(some_metric.field_1):Float64;N]\
\n Aggregate: groupBy=[[some_metric.tag_1, some_metric.timestamp]], aggr=[[DISPLAY_0, DISPLAY_1]] [tag_1:Utf8, timestamp:Timestamp(ms), TEMPLATE(some_metric.field_0):Float64;N, TEMPLATE(some_metric.field_1):Float64;N]\
\n Projection: some_metric.tag_0, some_metric.tag_1, some_metric.timestamp, some_metric.field_0, some_metric.field_1, Utf8(\"some_metric\") AS __promql_metric_name [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N, __promql_metric_name:Utf8]\
\n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]\
\n PromSeriesDivide: tags=[\"tag_0\", \"tag_1\"] [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]\
\n Sort: some_metric.tag_0 ASC NULLS FIRST, some_metric.tag_1 ASC NULLS FIRST, some_metric.timestamp ASC NULLS FIRST [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]\
\n Filter: some_metric.tag_0 != Utf8(\"bar\") AND some_metric.timestamp >= TimestampMillisecond(-999, None) AND some_metric.timestamp <= TimestampMillisecond(100000000, None) [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]\
\n TableScan: some_metric [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]"
).replace("TEMPLATE", plan_name);
)
.replace("TEMPLATE", plan_name)
.replace("DISPLAY_0", &display("some_metric.field_0"))
.replace("DISPLAY_1", &display("some_metric.field_1"));
assert_eq!(
plan.display_indent_schema().to_string(),
expected_no_without
@@ -2015,14 +2044,16 @@ async fn do_aggregate_expr_plan(fn_name: &str, plan_name: &str) {
.unwrap();
let expected_without = String::from(
"Sort: some_metric.tag_0 ASC NULLS LAST, some_metric.timestamp ASC NULLS LAST [tag_0:Utf8, timestamp:Timestamp(ms), TEMPLATE(some_metric.field_0):Float64;N, TEMPLATE(some_metric.field_1):Float64;N]\
\n Aggregate: groupBy=[[some_metric.tag_0, some_metric.timestamp]], aggr=[[TEMPLATE(some_metric.field_0), TEMPLATE(some_metric.field_1)]] [tag_0:Utf8, timestamp:Timestamp(ms), TEMPLATE(some_metric.field_0):Float64;N, TEMPLATE(some_metric.field_1):Float64;N]\
\n Aggregate: groupBy=[[some_metric.tag_0, some_metric.timestamp]], aggr=[[DISPLAY_0, DISPLAY_1]] [tag_0:Utf8, timestamp:Timestamp(ms), TEMPLATE(some_metric.field_0):Float64;N, TEMPLATE(some_metric.field_1):Float64;N]\
\n Projection: some_metric.tag_0, some_metric.tag_1, some_metric.timestamp, some_metric.field_0, some_metric.field_1, Utf8(\"some_metric\") AS __promql_metric_name [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N, __promql_metric_name:Utf8]\
\n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]\
\n PromSeriesDivide: tags=[\"tag_0\", \"tag_1\"] [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]\
\n Sort: some_metric.tag_0 ASC NULLS FIRST, some_metric.tag_1 ASC NULLS FIRST, some_metric.timestamp ASC NULLS FIRST [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]\
\n Filter: some_metric.tag_0 != Utf8(\"bar\") AND some_metric.timestamp >= TimestampMillisecond(-999, None) AND some_metric.timestamp <= TimestampMillisecond(100000000, None) [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]\
\n TableScan: some_metric [tag_0:Utf8, tag_1:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, field_1:Float64;N]"
).replace("TEMPLATE", plan_name);
).replace("TEMPLATE", plan_name)
.replace("DISPLAY_0", &display("some_metric.field_0"))
.replace("DISPLAY_1", &display("some_metric.field_1"));
assert_eq!(plan.display_indent_schema().to_string(), expected_without);
}
@@ -4048,12 +4079,12 @@ async fn aggregate_count() {
#[tokio::test]
async fn aggregate_min() {
do_aggregate_expr_plan("min", "min").await;
do_aggregate_expr_plan_with_display("min", "min", "prom_min").await;
}
#[tokio::test]
async fn aggregate_max() {
do_aggregate_expr_plan("max", "max").await;
do_aggregate_expr_plan_with_display("max", "max", "prom_max").await;
}
#[tokio::test]
@@ -4240,7 +4271,7 @@ async fn simple_bool_grammar() {
let query = "some_metric != bool 1.2345";
let expected = String::from(
"Projection: some_metric.tag_0, some_metric.timestamp, field_0 != Float64(1.2345) [tag_0:Utf8, timestamp:Timestamp(ms), field_0 != Float64(1.2345):Float64;N]\
\n Projection: some_metric.tag_0, __promql_metric_name, some_metric.timestamp, CAST(some_metric.field_0 != Float64(1.2345) AS Float64) AS field_0 != Float64(1.2345) [tag_0:Utf8, __promql_metric_name:Utf8, timestamp:Timestamp(ms), field_0 != Float64(1.2345):Float64;N]\
\n Projection: some_metric.tag_0, __promql_metric_name, some_metric.timestamp, CAST(prom_ne(some_metric.field_0, Float64(1.2345)) AS Float64) AS field_0 != Float64(1.2345) [tag_0:Utf8, __promql_metric_name:Utf8, timestamp:Timestamp(ms), field_0 != Float64(1.2345):Float64;N]\
\n Projection: some_metric.tag_0, some_metric.timestamp, some_metric.field_0, Utf8(\"some_metric\") AS __promql_metric_name [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_metric_name:Utf8]\
\n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\
\n PromSeriesDivide: tags=[\"tag_0\"] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\
@@ -4257,7 +4288,7 @@ async fn bool_with_additional_arithmetic() {
let query = "some_metric + (1 == bool 2)";
let expected = String::from(
"Projection: some_metric.tag_0, some_metric.timestamp, field_0 + Float64(1) = Float64(2) [tag_0:Utf8, timestamp:Timestamp(ms), field_0 + Float64(1) = Float64(2):Float64;N]\
\n Projection: some_metric.tag_0, __promql_metric_name, some_metric.timestamp, CAST(some_metric.field_0 AS Float64) + CAST(Float64(1) = Float64(2) AS Float64) AS field_0 + Float64(1) = Float64(2) [tag_0:Utf8, __promql_metric_name:Utf8, timestamp:Timestamp(ms), field_0 + Float64(1) = Float64(2):Float64;N]\
\n Projection: some_metric.tag_0, __promql_metric_name, some_metric.timestamp, CAST(some_metric.field_0 AS Float64) + CAST(prom_eq(Float64(1), Float64(2)) AS Float64) AS field_0 + Float64(1) = Float64(2) [tag_0:Utf8, __promql_metric_name:Utf8, timestamp:Timestamp(ms), field_0 + Float64(1) = Float64(2):Float64;N]\
\n Projection: some_metric.tag_0, some_metric.timestamp, some_metric.field_0, Utf8(\"some_metric\") AS __promql_metric_name [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_metric_name:Utf8]\
\n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\
\n PromSeriesDivide: tags=[\"tag_0\"] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\
@@ -5239,7 +5270,7 @@ async fn native_histogram_mixed_field_table_behaves() {
async fn less_filter_on_value() {
let query = "some_metric < 1.2345";
let expected = String::from(
"Filter: some_metric.field_0 < Float64(1.2345) [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_metric_name:Utf8]\
"Filter: prom_lt(some_metric.field_0, Float64(1.2345)) [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_metric_name:Utf8]\
\n Projection: some_metric.tag_0, some_metric.timestamp, some_metric.field_0, Utf8(\"some_metric\") AS __promql_metric_name [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N, __promql_metric_name:Utf8]\
\n PromInstantManipulate: range=[0..100000000], lookback=[1000], interval=[5000], time index=[timestamp] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\
\n PromSeriesDivide: tags=[\"tag_0\"] [tag_0:Utf8, timestamp:Timestamp(ms), field_0:Float64;N]\
@@ -5864,14 +5895,6 @@ async fn subquery_rejects_invalid_range_and_zero_step() {
matches!(&err, crate::promql::error::Error::ZeroRangeSelector { .. }),
"{err}"
);
// An omitted child step inherits the zero parent step.
let err = plan_subquery_offset_probe("sum_over_time(subquery_offset_offgrid[5s:])", 60, 60, 0)
.await
.unwrap_err();
assert!(
matches!(&err, crate::promql::error::Error::ZeroRangeSelector { .. }),
"{err}"
);
}
/// Rejects nonempty child windows whose derived millisecond bounds are unrepresentable.
@@ -6882,9 +6905,9 @@ async fn test_topk_expr() {
.await
.unwrap();
let expected = "Projection: sum(prometheus_tsdb_head_series.greptime_value), prometheus_tsdb_head_series.ip, prometheus_tsdb_head_series.greptime_timestamp [sum(prometheus_tsdb_head_series.greptime_value):Float64;N, ip:Utf8, greptime_timestamp:Timestamp(ms)]\
\n Sort: prometheus_tsdb_head_series.greptime_timestamp ASC NULLS LAST, row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ASC NULLS LAST [ip:Utf8, greptime_timestamp:Timestamp(ms), sum(prometheus_tsdb_head_series.greptime_value):Float64;N, row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW:UInt64]\
\n Filter: row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW <= Float64(10) [ip:Utf8, greptime_timestamp:Timestamp(ms), sum(prometheus_tsdb_head_series.greptime_value):Float64;N, row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW:UInt64]\
\n WindowAggr: windowExpr=[[row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]] [ip:Utf8, greptime_timestamp:Timestamp(ms), sum(prometheus_tsdb_head_series.greptime_value):Float64;N, row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW:UInt64]\
\n Sort: prometheus_tsdb_head_series.greptime_timestamp ASC NULLS LAST, row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [isnan(sum(prometheus_tsdb_head_series.greptime_value)) ASC NULLS FIRST, sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ASC NULLS LAST [ip:Utf8, greptime_timestamp:Timestamp(ms), sum(prometheus_tsdb_head_series.greptime_value):Float64;N, row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [isnan(sum(prometheus_tsdb_head_series.greptime_value)) ASC NULLS FIRST, sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW:UInt64]\
\n Filter: row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [isnan(sum(prometheus_tsdb_head_series.greptime_value)) ASC NULLS FIRST, sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW <= Float64(10) [ip:Utf8, greptime_timestamp:Timestamp(ms), sum(prometheus_tsdb_head_series.greptime_value):Float64;N, row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [isnan(sum(prometheus_tsdb_head_series.greptime_value)) ASC NULLS FIRST, sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW:UInt64]\
\n WindowAggr: windowExpr=[[row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [isnan(sum(prometheus_tsdb_head_series.greptime_value)) ASC NULLS FIRST, sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]] [ip:Utf8, greptime_timestamp:Timestamp(ms), sum(prometheus_tsdb_head_series.greptime_value):Float64;N, row_number() PARTITION BY [prometheus_tsdb_head_series.greptime_timestamp] ORDER BY [isnan(sum(prometheus_tsdb_head_series.greptime_value)) ASC NULLS FIRST, sum(prometheus_tsdb_head_series.greptime_value) DESC NULLS FIRST, prometheus_tsdb_head_series.ip DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW:UInt64]\
\n Sort: prometheus_tsdb_head_series.ip ASC NULLS LAST, prometheus_tsdb_head_series.greptime_timestamp ASC NULLS LAST [ip:Utf8, greptime_timestamp:Timestamp(ms), sum(prometheus_tsdb_head_series.greptime_value):Float64;N]\
\n Aggregate: groupBy=[[prometheus_tsdb_head_series.ip, prometheus_tsdb_head_series.greptime_timestamp]], aggr=[[sum(prometheus_tsdb_head_series.greptime_value)]] [ip:Utf8, greptime_timestamp:Timestamp(ms), sum(prometheus_tsdb_head_series.greptime_value):Float64;N]\
\n Projection: prometheus_tsdb_head_series.ip, prometheus_tsdb_head_series.greptime_timestamp, prometheus_tsdb_head_series.greptime_value, Utf8(\"prometheus_tsdb_head_series\") AS __promql_metric_name [ip:Utf8, greptime_timestamp:Timestamp(ms), greptime_value:Float64;N, __promql_metric_name:Utf8]\
src/query/src/query_engine/default_serializer.rs
+5 -1
View File
@@ -32,7 +32,7 @@ use datafusion_expr::{Expr, UserDefinedLogicalNode};
use greptime_proto::substrait_extension::MergeScan as PbMergeScan;
use promql::functions::{
AbsentOverTime, AvgOverTime, Changes, CountOverTime, Delta, Deriv, DoubleExponentialSmoothing,
IDelta, Increase, LastOverTime, MaxOverTime, MinOverTime, MixedRange,
IDelta, IeeeComparison, IeeeSqrt, Increase, LastOverTime, MaxOverTime, MinOverTime, MixedRange,
NativeHistogramAbsentOverTime, NativeHistogramAdd, NativeHistogramAggAvg,
NativeHistogramAggSum, NativeHistogramAvg, NativeHistogramAvgOverTime, NativeHistogramChanges,
NativeHistogramCount, NativeHistogramCountOverTime, NativeHistogramDelta,
@@ -334,6 +334,10 @@ fn register_greptime_functions(
}
let _ = session_state.register_udaf(quantile_udaf());
for udf in IeeeComparison::all() {
let _ = session_state.register_udf(Arc::new(udf));
}
let _ = session_state.register_udf(Arc::new(IeeeSqrt::scalar_udf()));
let _ = session_state.register_udf(Arc::new(IDelta::<false>::scalar_udf()));
let _ = session_state.register_udf(Arc::new(IDelta::<true>::scalar_udf()));
src/servers/src/http.rs
+20
View File
@@ -1576,6 +1576,26 @@ impl HttpServer {
"/api/traces/{trace_id}",
routing::get(jaeger::handle_get_trace),
)
.route(
"/api/v3/services",
routing::get(jaeger::v3::handle_get_services),
)
.route(
"/api/v3/operations",
routing::get(jaeger::v3::handle_get_operations),
)
.route(
"/api/v3/traces",
routing::get(jaeger::v3::handle_find_traces),
)
.route(
"/api/v3/trace-summaries",
routing::get(jaeger::v3::handle_find_trace_summaries),
)
.route(
"/api/v3/traces/{trace_id}",
routing::get(jaeger::v3::handle_get_trace),
)
.with_state(handler)
}
src/servers/src/http/jaeger.rs
+11 -5
View File
@@ -12,6 +12,8 @@
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod v3;
use std::collections::{BTreeMap, HashMap};
use std::fmt;
use std::str::FromStr;
@@ -289,9 +291,9 @@ fn update_query_context(query_ctx: &mut QueryContext, table_name: Option<String>
impl QueryTraceParams {
fn from_jaeger_query_params(query_params: JaegerQueryParams) -> Result<Self> {
let mut internal_query_params: QueryTraceParams = QueryTraceParams {
service_name: query_params.service_name.context(InvalidJaegerQuerySnafu {
service_name: Some(query_params.service_name.context(InvalidJaegerQuerySnafu {
reason: "service_name is required".to_string(),
})?,
})?),
operation_name: query_params.operation_name,
// Convert start time from microseconds to nanoseconds.
start_time: query_params.start.map(|start| start * 1000),
@@ -347,7 +349,7 @@ impl QueryTraceParams {
#[derive(Debug, Default, PartialEq)]
pub struct QueryTraceParams {
pub service_name: String,
pub service_name: Option<String>,
pub operation_name: Option<String>,
// The limit of the number of traces to return.
@@ -362,6 +364,9 @@ pub struct QueryTraceParams {
pub min_duration: Option<u64>,
pub max_duration: Option<u64>,
/// Fetch all spans of matching traces, including spans outside the search window.
pub fetch_full_trace: bool,
// The user agent of the trace query, mainly find traces
pub user_agent: TraceUserAgent,
}
@@ -1647,7 +1652,7 @@ mod tests {
..Default::default()
},
QueryTraceParams {
service_name: "test-service-0".to_string(),
service_name: Some("test-service-0".to_string()),
..Default::default()
},
),
@@ -1664,7 +1669,7 @@ mod tests {
..Default::default()
},
QueryTraceParams {
service_name: "test-service-0".to_string(),
service_name: Some("test-service-0".to_string()),
operation_name: Some("access-mysql".to_string()),
start_time: Some(1738726754492422000),
end_time: Some(1738726754642422000),
@@ -1679,6 +1684,7 @@ mod tests {
("http.path".to_string(), JsonValue::String("/api/v1/users".to_string())),
])),
user_agent: TraceUserAgent::Jaeger,
fetch_full_trace: false,
},
),
];
src/servers/src/http/jaeger/v3.rs
+542
View File
@@ -0,0 +1,542 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
mod otlp;
mod summary;
use std::collections::HashMap;
use std::sync::Arc;
use axum::extract::rejection::QueryRejection;
use axum::extract::{Path, Query, State};
use axum::http::StatusCode;
use axum::response::{IntoResponse, Response};
use axum::{Extension, Json};
use common_error::ext::ErrorExt;
use common_error::status_code::StatusCode as GreptimeStatusCode;
use common_query::Output;
use common_telemetry::error;
use rust_decimal::Decimal;
use rust_decimal::prelude::ToPrimitive;
use serde::Deserialize;
use serde_json::{Value, json};
use session::context::QueryContext;
use crate::error::{Error, status_code_to_http_status};
use crate::http::extractor::TraceTableName;
use crate::http::jaeger::{
QueryTraceParams, convert_string_to_boolean, convert_string_to_number, covert_to_records,
empty_string_as_none, operations_from_records, services_from_records, update_query_context,
};
use crate::metrics::METRIC_JAEGER_QUERY_ELAPSED;
use crate::query_handler::JaegerQueryHandlerRef;
/// Jaeger API v3 uses an error envelope rather than the legacy `errors` array.
#[derive(Debug)]
pub struct ApiError {
status: StatusCode,
message: String,
}
impl ApiError {
fn invalid(message: impl Into<String>) -> Self {
Self {
status: StatusCode::BAD_REQUEST,
message: message.into(),
}
}
fn not_found() -> Self {
Self {
status: StatusCode::NOT_FOUND,
message: "No traces found".into(),
}
}
}
impl From<Error> for ApiError {
fn from(err: Error) -> Self {
error!(err; "Jaeger v3 query failed");
Self {
status: status_code_to_http_status(&err.status_code()),
message: err.output_msg(),
}
}
}
impl From<QueryRejection> for ApiError {
fn from(err: QueryRejection) -> Self {
Self::invalid(err.body_text())
}
}
impl IntoResponse for ApiError {
fn into_response(self) -> Response {
(
self.status,
Json(json!({"error": {"httpCode": self.status.as_u16(), "message": self.message}})),
)
.into_response()
}
}
#[derive(Debug, Default, Deserialize)]
pub struct OperationsParams {
service: String,
#[serde(rename = "spanKind", alias = "span_kind")]
span_kind: Option<String>,
}
#[derive(Debug, Default, Deserialize)]
pub struct GetTraceParams {
#[serde(rename = "startTime", alias = "start_time")]
#[serde(default, deserialize_with = "empty_string_as_none")]
start_time: Option<String>,
#[serde(rename = "endTime", alias = "end_time")]
#[serde(default, deserialize_with = "empty_string_as_none")]
end_time: Option<String>,
// GreptimeDB returns stored spans without Jaeger's clock-skew enrichment.
#[serde(rename = "rawTraces", alias = "raw_traces")]
#[serde(default, deserialize_with = "empty_string_as_none")]
_raw_traces: Option<bool>,
}
#[derive(Debug, Default, Deserialize)]
pub struct FindTracesParams {
#[serde(rename = "query.serviceName", alias = "query.service_name")]
service_name: Option<String>,
#[serde(rename = "query.operationName", alias = "query.operation_name")]
operation_name: Option<String>,
#[serde(rename = "query.startTimeMin", alias = "query.start_time_min")]
start_time_min: String,
#[serde(rename = "query.startTimeMax", alias = "query.start_time_max")]
start_time_max: String,
#[serde(rename = "query.durationMin", alias = "query.duration_min")]
#[serde(default, deserialize_with = "empty_string_as_none")]
duration_min: Option<String>,
#[serde(rename = "query.durationMax", alias = "query.duration_max")]
#[serde(default, deserialize_with = "empty_string_as_none")]
duration_max: Option<String>,
#[serde(
rename = "query.searchDepth",
alias = "query.search_depth",
alias = "query.num_traces"
)]
#[serde(default, deserialize_with = "empty_string_as_none")]
search_depth: Option<i32>,
#[serde(rename = "query.attributes")]
#[serde(default, deserialize_with = "empty_string_as_none")]
attributes: Option<String>,
#[serde(rename = "query.rawTraces", alias = "query.raw_traces")]
#[serde(default, deserialize_with = "empty_string_as_none")]
_raw_traces: Option<bool>,
#[serde(rename = "query.filter")]
filter: Option<String>,
}
impl FindTracesParams {
fn into_query(self) -> Result<QueryTraceParams, ApiError> {
if self.filter.is_some() {
return Err(ApiError::invalid("query.filter is not supported"));
}
let start = parse_timestamp(&self.start_time_min)?;
let end = parse_timestamp(&self.start_time_max)?;
if start >= end {
return Err(ApiError::invalid(
"query.startTimeMin must be before query.startTimeMax",
));
}
let limit = match self.search_depth.unwrap_or(0) {
0 => 100,
limit @ 1..=10_000 => limit,
_ => {
return Err(ApiError::invalid(
"query.searchDepth must be between 0 and 10000",
));
}
};
let tags = self
.attributes
.map(|attributes| {
let values = serde_json::from_str::<HashMap<String, String>>(&attributes)
.map_err(|err| ApiError::invalid(format!("Invalid query.attributes: {err}")))?;
Ok::<_, ApiError>(
values
.into_iter()
.map(|(key, value)| {
let value = Value::String(value);
let value = convert_string_to_number(&value)
.or_else(|| convert_string_to_boolean(&value))
.unwrap_or(value);
(key, value)
})
.collect(),
)
})
.transpose()?;
let query = QueryTraceParams {
service_name: self.service_name.filter(|name| !name.is_empty()),
operation_name: self.operation_name.filter(|name| !name.is_empty()),
start_time: Some(start),
// The backend uses an inclusive bound; API v3 specifies an exclusive bound.
end_time: Some(end - 1),
min_duration: self
.duration_min
.as_deref()
.map(parse_duration)
.transpose()?,
max_duration: self
.duration_max
.as_deref()
.map(parse_duration)
.transpose()?,
limit: Some(limit as usize),
tags,
fetch_full_trace: true,
..Default::default()
};
if let (Some(min), Some(max)) = (query.min_duration, query.max_duration)
&& min > max
{
return Err(ApiError::invalid(
"query.durationMin must not exceed query.durationMax",
));
}
Ok(query)
}
}
fn parse_timestamp(value: &str) -> Result<i64, ApiError> {
chrono::DateTime::parse_from_rfc3339(value)
.map_err(|err| ApiError::invalid(format!("Invalid RFC3339 timestamp '{value}': {err}")))?
.timestamp_nanos_opt()
.ok_or_else(|| ApiError::invalid("Timestamp is outside the supported nanosecond range"))
}
fn parse_duration(value: &str) -> Result<u64, ApiError> {
let invalid = || ApiError::invalid(format!("Invalid Go duration '{value}'"));
let mut rest = value.strip_prefix('+').unwrap_or(value);
if rest == "0" {
return Ok(0);
}
if rest.is_empty() {
return Err(invalid());
}
let mut total = 0_u64;
while !rest.is_empty() {
let number_end = rest
.find(|ch: char| !ch.is_ascii_digit() && ch != '.')
.ok_or_else(invalid)?;
let amount: Decimal = rest[..number_end].parse().map_err(|_| invalid())?;
rest = &rest[number_end..];
let unit_end = rest
.find(|ch: char| ch.is_ascii_digit() || ch == '.')
.unwrap_or(rest.len());
let unit = match &rest[..unit_end] {
"ns" => 1_u64,
"us" | "µs" | "μs" => 1_000,
"ms" => 1_000_000,
"s" => 1_000_000_000,
"m" => 60_000_000_000,
"h" => 3_600_000_000_000,
_ => return Err(invalid()),
};
let nanos = amount
.checked_mul(Decimal::from(unit))
.and_then(|amount| amount.trunc().to_u64())
.ok_or_else(invalid)?;
total = total
.checked_add(nanos)
.filter(|total| *total <= i64::MAX as u64)
.ok_or_else(invalid)?;
rest = &rest[unit_end..];
}
Ok(total)
}
pub async fn handle_get_services(
State(handler): State<JaegerQueryHandlerRef>,
Extension(mut ctx): Extension<QueryContext>,
TraceTableName(table): TraceTableName,
) -> Result<Json<Value>, ApiError> {
update_query_context(&mut ctx, table);
let _timer = METRIC_JAEGER_QUERY_ELAPSED
.with_label_values(&[&ctx.get_db_string(), "/api/v3/services"])
.start_timer();
let services = match query_records(handler.get_services(Arc::new(ctx)).await).await? {
Some(records) => services_from_records(records)?,
None => vec![],
};
Ok(Json(json!({"services": services})))
}
pub async fn handle_get_operations(
State(handler): State<JaegerQueryHandlerRef>,
params: Result<Query<OperationsParams>, QueryRejection>,
Extension(mut ctx): Extension<QueryContext>,
TraceTableName(table): TraceTableName,
) -> Result<Json<Value>, ApiError> {
let Query(params) = params?;
if params.service.is_empty() {
return Err(ApiError::invalid("service is required"));
}
update_query_context(&mut ctx, table);
let _timer = METRIC_JAEGER_QUERY_ELAPSED
.with_label_values(&[&ctx.get_db_string(), "/api/v3/operations"])
.start_timer();
let operations = match query_records(
handler
.get_operations(
Arc::new(ctx),
&params.service,
params.span_kind.as_deref().filter(|kind| !kind.is_empty()),
)
.await,
)
.await?
{
Some(records) => operations_from_records(records, true)?,
None => vec![],
};
let operations: Vec<_> = operations.into_iter().map(|operation| json!({
"name": operation.name,
"spanKind": operation.span_kind.filter(|kind| !kind.is_empty()).unwrap_or_else(|| "internal".into()),
})).collect();
Ok(Json(json!({"operations": operations})))
}
pub async fn handle_get_trace(
State(handler): State<JaegerQueryHandlerRef>,
Path(trace_id): Path<String>,
params: Result<Query<GetTraceParams>, QueryRejection>,
Extension(mut ctx): Extension<QueryContext>,
TraceTableName(table): TraceTableName,
) -> Result<Json<Value>, ApiError> {
let Query(params) = params?;
if !matches!(trace_id.len(), 16 | 32) || !trace_id.bytes().all(|ch| ch.is_ascii_hexdigit()) {
return Err(ApiError::invalid(
"trace_id must be a 64-bit or 128-bit hexadecimal ID",
));
}
let trace_id = format!("{:0>32}", trace_id.to_ascii_lowercase());
let start = params
.start_time
.as_deref()
.map(parse_timestamp)
.transpose()?;
let end = params
.end_time
.as_deref()
.map(parse_timestamp)
.transpose()?;
if let (Some(start), Some(end)) = (start, end)
&& start > end
{
return Err(ApiError::invalid("startTime must not exceed endTime"));
}
update_query_context(&mut ctx, table);
let _timer = METRIC_JAEGER_QUERY_ELAPSED
.with_label_values(&[&ctx.get_db_string(), "/api/v3/traces/{trace_id}"])
.start_timer();
// API v3 time bounds are lookup hints. Applying them as span filters would truncate the trace,
// so they are only validated.
trace_response(
handler
.get_trace(Arc::new(ctx), &trace_id, None, None, None)
.await,
)
.await
}
pub async fn handle_find_traces(
State(handler): State<JaegerQueryHandlerRef>,
params: Result<Query<FindTracesParams>, QueryRejection>,
Extension(mut ctx): Extension<QueryContext>,
TraceTableName(table): TraceTableName,
) -> Result<Json<Value>, ApiError> {
let Query(params) = params?;
let query = params.into_query()?;
update_query_context(&mut ctx, table);
let _timer = METRIC_JAEGER_QUERY_ELAPSED
.with_label_values(&[&ctx.get_db_string(), "/api/v3/traces"])
.start_timer();
trace_response(handler.find_traces(Arc::new(ctx), query).await).await
}
pub async fn handle_find_trace_summaries(
State(handler): State<JaegerQueryHandlerRef>,
params: Result<Query<FindTracesParams>, QueryRejection>,
Extension(mut ctx): Extension<QueryContext>,
TraceTableName(table): TraceTableName,
) -> Result<Json<Value>, ApiError> {
let Query(params) = params?;
let query = params.into_query()?;
update_query_context(&mut ctx, table);
let _timer = METRIC_JAEGER_QUERY_ELAPSED
.with_label_values(&[&ctx.get_db_string(), "/api/v3/trace-summaries"])
.start_timer();
let summaries = match query_records(handler.find_traces(Arc::new(ctx), query).await).await? {
Some(records) => summary::summarize(otlp::traces_from_records(records)?),
None => vec![],
};
Ok(Json(json!({"summaries": summaries})))
}
async fn query_records(
output: crate::error::Result<Output>,
) -> Result<Option<crate::http::HttpRecordsOutput>, ApiError> {
match output {
Ok(output) => covert_to_records(output)
.await?
.map(Some)
.ok_or_else(|| ApiError {
status: StatusCode::INTERNAL_SERVER_ERROR,
message: "Unexpected Jaeger query output".into(),
}),
Err(err) if err.status_code() == GreptimeStatusCode::TableNotFound => Ok(None),
Err(err) => Err(err.into()),
}
}
async fn trace_response(output: crate::error::Result<Output>) -> Result<Json<Value>, ApiError> {
let records = query_records(output)
.await?
.ok_or_else(ApiError::not_found)?;
if records.rows.is_empty() {
return Err(ApiError::not_found());
}
Ok(Json(json!({"result": otlp::traces_from_records(records)?})))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_find_traces_parameters() {
for (start, end, service, operation, depth, duration) in [
(
"start_time_min",
"start_time_max",
"service_name",
"operation_name",
"num_traces",
"duration_min",
),
(
"startTimeMin",
"startTimeMax",
"serviceName",
"operationName",
"searchDepth",
"durationMin",
),
] {
let uri = format!("/api/v3/traces?query.{start}=2026-01-01T00:00:00.000000001Z&query.{end}=2026-01-01T00:00:01Z&query.{service}=checkout&query.{operation}=GET&query.{depth}=7&query.{duration}=1.5ms&query.attributes=%7B%22error%22%3A%22true%22%7D").parse().unwrap();
let Query(params) = Query::<FindTracesParams>::try_from_uri(&uri).unwrap();
let query = params.into_query().unwrap();
assert_eq!(query.service_name.as_deref(), Some("checkout"));
assert_eq!(query.operation_name.as_deref(), Some("GET"));
assert_eq!(query.limit, Some(7));
assert_eq!(query.min_duration, Some(1_500_000));
assert_eq!(query.start_time, Some(1767225600000000001));
assert_eq!(query.end_time, Some(1767225600999999999));
assert_eq!(query.tags.unwrap()["error"], json!(true));
assert!(query.fetch_full_trace);
}
let uri = "/api/v3/traces?query.startTimeMin=2026-01-01T00:00:00Z&query.startTimeMax=2026-01-01T01:00:00Z&query.durationMin=&query.durationMax=&query.searchDepth=&query.rawTraces=&query.attributes=".parse().unwrap();
let Query(params) = Query::<FindTracesParams>::try_from_uri(&uri).unwrap();
let query = params.into_query().unwrap();
assert_eq!(query.service_name, None);
assert_eq!(query.limit, Some(100));
for name in [
"query.searchDepth",
"query.search_depth",
"query.num_traces",
] {
for (depth, expected) in [(0, 100), (1, 1), (10_000, 10_000)] {
let uri = format!("/api/v3/traces?query.startTimeMin=2026-01-01T00:00:00Z&query.startTimeMax=2026-01-01T01:00:00Z&{name}={depth}").parse().unwrap();
let Query(params) = Query::<FindTracesParams>::try_from_uri(&uri).unwrap();
assert_eq!(params.into_query().unwrap().limit, Some(expected));
}
}
}
#[test]
fn test_go_durations() {
for (value, expected) in [
("0", 0),
("1.5ms", 1_500_000),
("1h2m3.5s", 3_723_500_000_000),
("+.5us", 500),
("1.µs", 1_000),
("1μs", 1_000),
("0.1ns", 0),
("9223372036854775807ns", i64::MAX as u64),
] {
assert_eq!(parse_duration(value).unwrap(), expected, "{value}");
}
for value in [
"",
"1",
"-1s",
"1.2.3s",
"s",
"1 s",
"1d",
"1s-1s",
"9223372036854775808ns",
] {
assert!(parse_duration(value).is_err(), "{value}");
}
}
#[test]
fn test_invalid_find_traces_parameters() {
for extra in [
"query.searchDepth=-1",
"query.searchDepth=10001",
"query.searchDepth=2147483647",
"query.durationMin=-1s",
"query.durationMin=2s&query.durationMax=1s",
"query.durationMax=999999999999h",
"query.attributes=%7B%22error%22%3Atrue%7D",
"query.attributes=invalid",
"query.filter=%7B%7D",
] {
let uri = format!("/api/v3/traces?query.startTimeMin=2026-01-01T00:00:00Z&query.startTimeMax=2026-01-01T01:00:00Z&{extra}").parse().unwrap();
let Query(params) = Query::<FindTracesParams>::try_from_uri(&uri).unwrap();
assert_eq!(
params.into_query().unwrap_err().status,
StatusCode::BAD_REQUEST,
"{extra}"
);
}
for (start, end) in [
("invalid", "2026-01-01T00:00:00Z"),
("2500-01-01T00:00:00Z", "2500-01-02T00:00:00Z"),
("2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z"),
] {
assert!(
FindTracesParams {
start_time_min: start.into(),
start_time_max: end.into(),
..Default::default()
}
.into_query()
.is_err()
);
}
}
}
src/servers/src/http/jaeger/v3/otlp.rs
+294
View File
@@ -0,0 +1,294 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::{HashMap, HashSet};
use axum::http::StatusCode;
use opentelemetry_proto::tonic::common::v1::{
AnyValue, ArrayValue, InstrumentationScope, KeyValue, KeyValueList, any_value,
};
use opentelemetry_proto::tonic::resource::v1::Resource;
use opentelemetry_proto::tonic::trace::v1::span::{Event, Link, SpanKind};
use opentelemetry_proto::tonic::trace::v1::status::StatusCode as SpanStatusCode;
use opentelemetry_proto::tonic::trace::v1::{ResourceSpans, ScopeSpans, Span, Status, TracesData};
use serde_json::Value;
use crate::http::HttpRecordsOutput;
use crate::http::jaeger::v3::ApiError;
use crate::otlp::trace::{
KEY_SERVICE_NAME, PARENT_SPAN_ID_COLUMN, RESOURCE_ATTRIBUTES_COLUMN, SCOPE_ATTRIBUTES_COLUMN,
SCOPE_NAME_COLUMN, SCOPE_VERSION_COLUMN, SERVICE_NAME_COLUMN, SPAN_ATTRIBUTES_COLUMN,
SPAN_EVENTS_COLUMN, SPAN_ID_COLUMN, SPAN_KIND_COLUMN, SPAN_LINKS_COLUMN, SPAN_NAME_COLUMN,
SPAN_STATUS_CODE, SPAN_STATUS_MESSAGE_COLUMN, TIMESTAMP_COLUMN, TIMESTAMP_END_COLUMN,
TRACE_ID_COLUMN, TRACE_STATE_COLUMN,
};
/// Converts stored trace rows directly to OTLP, without the legacy microsecond conversion.
pub fn traces_from_records(records: HttpRecordsOutput) -> Result<TracesData, ApiError> {
let binary_columns: HashSet<_> = records
.schema
.column_schemas
.iter()
.filter(|column| column.data_type == "Binary")
.map(|column| column.name.as_str())
.collect();
let mut resource_spans = Vec::with_capacity(records.rows.len());
for row in &records.rows {
let cells: HashMap<_, _> = records
.schema
.column_schemas
.iter()
.zip(row)
.map(|(column, cell)| (column.name.as_str(), cell))
.collect();
let string = |name| cells.get(name).and_then(|v| v.as_str()).unwrap_or_default();
let timestamp = |name| {
cells
.get(name)
.and_then(|v| v.as_u64())
.ok_or_else(|| invalid_data(format!("Invalid timestamp column: {name}")))
};
let span = Span {
trace_id: decode_id(string(TRACE_ID_COLUMN), 16)?,
span_id: decode_id(string(SPAN_ID_COLUMN), 8)?,
parent_span_id: if string(PARENT_SPAN_ID_COLUMN).is_empty() {
vec![]
} else {
decode_id(string(PARENT_SPAN_ID_COLUMN), 8)?
},
trace_state: string(TRACE_STATE_COLUMN).into(),
name: string(SPAN_NAME_COLUMN).into(),
kind: SpanKind::from_str_name(string(SPAN_KIND_COLUMN)).unwrap_or(SpanKind::Unspecified)
as i32,
start_time_unix_nano: timestamp(TIMESTAMP_COLUMN)?,
end_time_unix_nano: timestamp(TIMESTAMP_END_COLUMN)?,
attributes: attributes(&cells, SPAN_ATTRIBUTES_COLUMN, &binary_columns)?,
events: events(cells.get(SPAN_EVENTS_COLUMN).copied())?,
links: links(cells.get(SPAN_LINKS_COLUMN).copied())?,
status: Some(Status {
code: SpanStatusCode::from_str_name(string(SPAN_STATUS_CODE))
.unwrap_or(SpanStatusCode::Unset) as i32,
message: string(SPAN_STATUS_MESSAGE_COLUMN).into(),
}),
..Default::default()
};
let mut resource_attributes =
attributes(&cells, RESOURCE_ATTRIBUTES_COLUMN, &binary_columns)?;
if !resource_attributes
.iter()
.any(|attr| attr.key == KEY_SERVICE_NAME)
&& !string(SERVICE_NAME_COLUMN).is_empty()
{
resource_attributes.push(KeyValue {
key: KEY_SERVICE_NAME.into(),
value: Some(AnyValue {
value: Some(any_value::Value::StringValue(
string(SERVICE_NAME_COLUMN).into(),
)),
}),
..Default::default()
});
}
// Each row retains its own resource and scope. Service name alone does not identify
// a resource: different instances of one service can have different attributes.
resource_spans.push(ResourceSpans {
resource: Some(Resource {
attributes: resource_attributes,
..Default::default()
}),
scope_spans: vec![ScopeSpans {
scope: Some(InstrumentationScope {
name: string(SCOPE_NAME_COLUMN).into(),
version: string(SCOPE_VERSION_COLUMN).into(),
attributes: attributes(&cells, SCOPE_ATTRIBUTES_COLUMN, &binary_columns)?,
..Default::default()
}),
spans: vec![span],
..Default::default()
}],
..Default::default()
});
}
Ok(TracesData { resource_spans })
}
fn invalid_data(message: impl Into<String>) -> ApiError {
ApiError {
status: StatusCode::INTERNAL_SERVER_ERROR,
message: message.into(),
}
}
fn decode_id(value: &str, length: usize) -> Result<Vec<u8>, ApiError> {
let bytes = hex::decode(value).map_err(|_| invalid_data("Invalid stored span or trace ID"))?;
if bytes.len() != length {
return Err(invalid_data("Invalid stored span or trace ID length"));
}
Ok(bytes)
}
fn attributes(
cells: &HashMap<&str, &Value>,
name: &str,
binary_columns: &HashSet<&str>,
) -> Result<Vec<KeyValue>, ApiError> {
if let Some(value) = cells.get(name) {
return object_attributes(value);
}
let prefix = format!("{name}.");
let mut attrs = cells
.iter()
.filter_map(|(column, value)| {
column
.strip_prefix(&prefix)
.filter(|_| !value.is_null())
.map(|key| (key, *value, binary_columns.contains(column)))
})
.map(|(key, value, is_binary)| {
let value = if is_binary {
AnyValue {
value: Some(any_value::Value::BytesValue(
serde_json::from_value(value.clone())
.map_err(|_| invalid_data("Invalid stored binary attribute"))?,
)),
}
} else {
any_value(value)?
};
Ok(KeyValue {
key: key.into(),
value: Some(value),
..Default::default()
})
})
.collect::<Result<Vec<_>, ApiError>>()?;
attrs.sort_unstable_by(|a, b| a.key.cmp(&b.key));
Ok(attrs)
}
fn object_attributes(value: &Value) -> Result<Vec<KeyValue>, ApiError> {
match value {
Value::Null => Ok(vec![]),
Value::Object(values) => values
.iter()
.map(|(key, value)| {
Ok(KeyValue {
key: key.clone(),
value: Some(any_value(value)?),
..Default::default()
})
})
.collect(),
_ => Err(invalid_data("Stored attributes must be a JSON object")),
}
}
fn any_value(value: &Value) -> Result<AnyValue, ApiError> {
use any_value::Value as OtlpValue;
let value = match value {
Value::Null => None,
Value::String(value) => Some(OtlpValue::StringValue(value.clone())),
Value::Bool(value) => Some(OtlpValue::BoolValue(*value)),
Value::Number(value) => Some(if let Some(value) = value.as_i64() {
OtlpValue::IntValue(value)
} else {
OtlpValue::DoubleValue(
value
.as_f64()
.ok_or_else(|| invalid_data("Invalid numeric attribute"))?,
)
}),
Value::Array(values) => Some(OtlpValue::ArrayValue(ArrayValue {
values: values.iter().map(any_value).collect::<Result<_, _>>()?,
})),
Value::Object(_) => Some(OtlpValue::KvlistValue(KeyValueList {
values: object_attributes(value)?,
})),
};
Ok(AnyValue { value })
}
fn events(value: Option<&Value>) -> Result<Vec<Event>, ApiError> {
let Some(value) = value.filter(|value| !value.is_null()) else {
return Ok(vec![]);
};
let values = value
.as_array()
.ok_or_else(|| invalid_data("Invalid stored span events"))?;
values
.iter()
.map(|event| {
let time = event["time"]
.as_str()
.ok_or_else(|| invalid_data("Missing span event time"))?;
let time = chrono::DateTime::parse_from_str(time, "%Y-%m-%d %H:%M:%S%.f%z")
.ok()
.and_then(|time| time.timestamp_nanos_opt())
.and_then(|time| u64::try_from(time).ok())
.ok_or_else(|| invalid_data("Invalid stored span event time"))?;
Ok(Event {
time_unix_nano: time,
name: event["name"]
.as_str()
.ok_or_else(|| invalid_data("Missing span event name"))?
.into(),
attributes: object_attributes(&event["attributes"])?,
..Default::default()
})
})
.collect()
}
fn links(value: Option<&Value>) -> Result<Vec<Link>, ApiError> {
let Some(value) = value.filter(|value| !value.is_null()) else {
return Ok(vec![]);
};
let values = value
.as_array()
.ok_or_else(|| invalid_data("Invalid stored span links"))?;
values
.iter()
.map(|link| {
Ok(Link {
trace_id: decode_id(link["trace_id"].as_str().unwrap_or_default(), 16)?,
span_id: decode_id(link["span_id"].as_str().unwrap_or_default(), 8)?,
trace_state: link["trace_state"].as_str().unwrap_or_default().into(),
attributes: object_attributes(&link["attributes"])?,
..Default::default()
})
})
.collect()
}
#[cfg(test)]
mod tests {
use serde_json::json;
use super::*;
#[test]
fn test_event_timestamp_precision() {
for (time, expected) in [
("1970-01-01 00:00:01+0000", 1_000_000_000),
("1970-01-01 00:00:01.123+0000", 1_123_000_000),
("1970-01-01 00:00:01.123456+0000", 1_123_456_000),
("1970-01-01 00:00:01.123456789+0000", 1_123_456_789),
] {
let value = json!([{"name":"event", "time":time, "attributes":{}}]);
let events = events(Some(&value)).unwrap();
assert_eq!(events[0].time_unix_nano, expected);
}
}
}
src/servers/src/http/jaeger/v3/summary.rs
+142
View File
@@ -0,0 +1,142 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::collections::{BTreeMap, HashSet};
use opentelemetry_proto::tonic::common::v1::any_value;
use opentelemetry_proto::tonic::trace::v1::status::StatusCode;
use opentelemetry_proto::tonic::trace::v1::{Span, TracesData};
use serde_json::{Value, json};
use crate::otlp::trace::KEY_SERVICE_NAME;
pub fn summarize(traces: TracesData) -> Vec<Value> {
let mut by_trace: BTreeMap<&[u8], Vec<(&str, &Span)>> = BTreeMap::new();
for resource in &traces.resource_spans {
let service = resource
.resource
.as_ref()
.and_then(|resource| {
resource
.attributes
.iter()
.find(|attr| attr.key == KEY_SERVICE_NAME)
.and_then(|attr| attr.value.as_ref())
.and_then(|value| match &value.value {
Some(any_value::Value::StringValue(name)) => Some(name.as_str()),
_ => None,
})
})
.unwrap_or_default();
for scope in &resource.scope_spans {
for span in &scope.spans {
by_trace
.entry(&span.trace_id)
.or_default()
.push((service, span));
}
}
}
by_trace
.into_iter()
.map(|(trace_id, spans)| {
let ids: HashSet<_> = spans
.iter()
.map(|(_, span)| span.span_id.as_slice())
.collect();
let mut services: BTreeMap<&str, (usize, usize)> = BTreeMap::new();
let mut min_start = u64::MAX;
let mut max_end = 0;
let mut errors = 0;
let mut orphans = 0;
let mut root: Option<(&str, &Span)> = None;
for &(service, span) in &spans {
let stats = services.entry(service).or_default();
stats.0 += 1;
if span
.status
.as_ref()
.is_some_and(|status| status.code == StatusCode::Error as i32)
{
stats.1 += 1;
errors += 1;
}
min_start = min_start.min(span.start_time_unix_nano);
max_end = max_end.max(span.end_time_unix_nano);
if span.parent_span_id.iter().all(|byte| *byte == 0) {
if root.is_none_or(|(_, root)| {
span.start_time_unix_nano < root.start_time_unix_nano
}) {
root = Some((service, span));
}
} else if !ids.contains(span.parent_span_id.as_slice()) {
orphans += 1;
}
}
let (root_service, root_operation) =
root.map_or(("", ""), |(service, span)| (service, span.name.as_str()));
let services: Vec<_> = services
.into_iter()
.map(|(name, (spans, errors))| {
json!({
"name": name, "spanCount": spans, "errorSpanCount": errors,
})
})
.collect();
json!({
"traceId": hex::encode(trace_id),
"rootServiceName": root_service,
"rootOperationName": root_operation,
"minStartTimeUnixNano": min_start.to_string(),
"maxEndTimeUnixNano": max_end.to_string(),
"spanCount": spans.len(),
"errorSpanCount": errors,
"orphanSpanCount": orphans,
"services": services,
})
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_summary_roots_and_orphans() {
let traces: TracesData = serde_json::from_value(json!({"resourceSpans":[
{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"b"}}]},"scopeSpans":[{"spans":[
{"traceId":"00000000000000000000000000000001","spanId":"0000000000000001","name":"late-root","startTimeUnixNano":"20","endTimeUnixNano":"50"},
{"traceId":"00000000000000000000000000000001","spanId":"0000000000000002","parentSpanId":"0000000000000099","name":"orphan","startTimeUnixNano":"1","endTimeUnixNano":"60","status":{"code":2}},
{"traceId":"00000000000000000000000000000002","spanId":"0000000000000003","parentSpanId":"0000000000000001","startTimeUnixNano":"1","endTimeUnixNano":"2"}
]}]},
{"resource":{"attributes":[{"key":"service.name","value":{"stringValue":"a"}}]},"scopeSpans":[{"spans":[
{"traceId":"00000000000000000000000000000001","spanId":"0000000000000004","parentSpanId":"0000000000000000","name":"early-root","startTimeUnixNano":"10","endTimeUnixNano":"20"},
{"traceId":"00000000000000000000000000000001","spanId":"0000000000000005","parentSpanId":"0000000000000004","startTimeUnixNano":"11","endTimeUnixNano":"15","status":{"code":2}}
]}]}
]})).unwrap();
let summaries = summarize(traces);
assert_eq!(
summaries[0],
json!({
"traceId":"00000000000000000000000000000001", "rootServiceName":"a", "rootOperationName":"early-root",
"minStartTimeUnixNano":"1", "maxEndTimeUnixNano":"60", "spanCount":4, "errorSpanCount":2, "orphanSpanCount":1,
"services":[{"name":"a","spanCount":2,"errorSpanCount":1},{"name":"b","spanCount":2,"errorSpanCount":1}]
})
);
assert_eq!(summaries[1]["orphanSpanCount"], 1);
assert_eq!(summaries[1]["rootServiceName"], "");
assert_eq!(summaries[1]["rootOperationName"], "");
}
}
tests-integration/src/tests/instance_test.rs
+119
View File
@@ -415,6 +415,125 @@ LEFT JOIN (SELECT k, count(*) n FROM join_repro WHERE kind = 'child' AND k < '8'
}
}
/// Regression test for `MatchesConstantTermOptimizer` dropping `FilterExec`'s `fetch`.
///
/// On a single-partition plan the `LIMIT` is enforced by the `fetch` that the physical
/// `LimitPushdown` rule leaves on the `FilterExec`, so a rule rebuilding that `FilterExec`
/// has to carry the `fetch` over. While it does not, the limit is silently lifted for
/// `matches_term` filters and
/// `SELECT count(*) FROM (SELECT ts, msg FROM t WHERE matches_term(msg, 'foo') LIMIT 1) s`
/// returns the number of matching rows (2) instead of 1.
///
/// `query_parallelism=4` is expected to stay correct, because the limit is then materialized
/// above the scan instead of being delegated to the `FilterExec`; the test pins down that
/// contrast. The metric-engine logical table covers the same filter on a shared physical
/// region.
///
/// Both matching rows share `ts` and `msg`, so the limited output does not depend on the
/// order in which the scan returns them.
#[apply(both_instances_cases)]
async fn test_matches_term_subquery_limit_count(instance: Arc<dyn MockInstance>) {
let frontend = instance.frontend();
execute_sql(
&frontend,
r#"
CREATE TABLE f07_mito_msgs (
ts TIMESTAMP TIME INDEX,
host STRING PRIMARY KEY,
msg STRING,
val DOUBLE
)
ENGINE = mito
"#,
)
.await;
execute_sql(
&frontend,
r#"INSERT INTO f07_mito_msgs (ts, host, msg, val) VALUES
(0, 'host1', 'foo bar', 1.0),
(0, 'host2', 'foo bar', 2.0),
(0, 'host3', 'other baz', 3.0)"#,
)
.await;
execute_sql(
&frontend,
r#"
CREATE TABLE f07_metric_phy (
ts TIMESTAMP TIME INDEX,
val DOUBLE,
host STRING,
msg STRING,
PRIMARY KEY (host, msg)
)
ENGINE = metric
WITH ('physical_metric_table' = '')
"#,
)
.await;
execute_sql(
&frontend,
r#"
CREATE TABLE f07_metric_msgs (
ts TIMESTAMP TIME INDEX,
val DOUBLE,
host STRING,
msg STRING,
PRIMARY KEY (host, msg)
)
ENGINE = metric
WITH ('on_physical_table' = 'f07_metric_phy')
"#,
)
.await;
execute_sql(
&frontend,
r#"INSERT INTO f07_metric_msgs (ts, host, msg, val) VALUES
(0, 'host1', 'foo bar', 1.0),
(0, 'host2', 'foo bar', 2.0),
(0, 'host3', 'other baz', 3.0)"#,
)
.await;
for table in ["f07_mito_msgs", "f07_metric_msgs"] {
let limited = format!("SELECT ts, msg FROM {table} WHERE matches_term(msg, 'foo') LIMIT 1");
let count = format!("SELECT count(*) FROM ({limited}) s");
let limit_rows = execute_sql_with_query_parallelism(&frontend, &limited, 1)
.await
.data
.pretty_print()
.await;
assert_eq!(
limit_rows,
r#"+---------------------+---------+
| ts | msg |
+---------------------+---------+
| 1970-01-01T00:00:00 | foo bar |
+---------------------+---------+"#,
"query_parallelism=1: `{limited}` must return exactly the single limited row"
);
for parallelism in [1, 4] {
let count_rows = execute_sql_with_query_parallelism(&frontend, &count, parallelism)
.await
.data
.pretty_print()
.await;
assert_eq!(
count_rows,
r#"+----------+
| count(*) |
+----------+
| 1 |
+----------+"#,
"query_parallelism={parallelism}: the LIMIT 1 of the subquery must cap `{count}` to one row"
);
}
}
}
#[apply(both_instances_cases)]
async fn test_partitioned_merge_scan_semi_join_repartitions(instance: Arc<dyn MockInstance>) {
let frontend = instance.frontend();
tests-integration/tests/export_logical_tables.rs
+23 -7
View File
@@ -1426,6 +1426,7 @@ struct ImportRequests {
async fn import_http(
instance: Arc<Instance>,
older: bool,
packed: bool,
) -> (String, tokio::task::JoinHandle<()>, Arc<ImportRequests>) {
use axum::response::IntoResponse;
use std::sync::atomic::Ordering;
@@ -1446,6 +1447,12 @@ async fn import_http(
if request.uri().path() == "/v1/capabilities" {
observed.capabilities.fetch_add(1, Ordering::SeqCst);
if older {
if packed {
return axum::Json(serde_json::json!({
"metric_packed_import": 1
}))
.into_response();
}
return axum::http::StatusCode::NOT_FOUND.into_response();
}
}
@@ -1528,6 +1535,11 @@ async fn packed_export_v2_cli_local_roundtrip() {
metric_export_v2_cli_roundtrip(false, true, false).await;
}
#[tokio::test(flavor = "multi_thread")]
async fn packed_import_v2_without_batch_ddl_roundtrip() {
metric_export_v2_cli_roundtrip(false, true, true).await;
}
#[tokio::test(flavor = "multi_thread")]
async fn packed_export_v2_cli_s3_roundtrip() {
if std::env::var("GT_S3_ENDPOINT_URL").is_ok_and(|e| !e.is_empty())
@@ -1606,7 +1618,7 @@ async fn metric_export_v2_cli_roundtrip(s3: bool, packed: bool, older: bool) {
&format!("CREATE VIEW \"{quoted_special}\" AS SELECT * FROM audit"),
)
.await;
if packed && !s3 {
if packed && !s3 && !older {
for id in 0..128 {
let name = format!("batch_{id:03}");
sql(instance, &format!("CREATE TABLE {name} (host STRING PRIMARY KEY, val DOUBLE, ts TIMESTAMP TIME INDEX) ENGINE=metric WITH (on_physical_table='v2_a')")).await;
@@ -1630,7 +1642,7 @@ async fn metric_export_v2_cli_roundtrip(s3: bool, packed: bool, older: bool) {
"CREATE TABLE bulk (host STRING PRIMARY KEY, payload STRING, ts TIMESTAMP TIME INDEX)",
)
.await;
for batch in 0..64 {
for batch in 0..96 {
let rows = (0..64)
.map(|row| {
let payload = (0..128)
@@ -1813,7 +1825,7 @@ async fn metric_export_v2_cli_roundtrip(s3: bool, packed: bool, older: bool) {
.await
.unwrap()
.content_length()
> 5 * 1024 * 1024
> common_datasource::packed_writer::WRITE_BYTES as u64
);
}
let mut preserved = Vec::new();
@@ -1923,7 +1935,7 @@ async fn metric_export_v2_cli_roundtrip(s3: bool, packed: bool, older: bool) {
.await;
}
let (target_addr, target_server, requests) =
import_http(target.fe_instance().clone(), older).await;
import_http(target.fe_instance().clone(), older, packed).await;
let state = destination.path().join("restore-state.json");
let mut import = vec![
"import-v2",
@@ -1959,7 +1971,11 @@ async fn metric_export_v2_cli_roundtrip(s3: bool, packed: bool, older: bool) {
requests.capabilities.load(Ordering::SeqCst),
if older { 1 } else { 2 }
);
assert_eq!(requests.batches.load(Ordering::SeqCst) > 1, !older);
if older {
assert_eq!(requests.batches.load(Ordering::SeqCst), 0);
} else {
assert!(requests.batches.load(Ordering::SeqCst) > 1);
}
assert!(!state.exists());
for name in names {
let source_table = table(instance, &name).await;
@@ -2003,7 +2019,7 @@ async fn metric_export_v2_cli_roundtrip(s3: bool, packed: bool, older: bool) {
.column_schemas()
);
}
if packed {
if packed && !older {
let schema_uri = format!("{uri}/schema-only");
let mut schema_args = vec![
"export-v2",
@@ -2039,7 +2055,7 @@ async fn metric_export_v2_cli_roundtrip(s3: bool, packed: bool, older: bool) {
.build()
.await;
let (schema_addr, schema_server, schema_requests) =
import_http(schema_target.fe_instance().clone(), false).await;
import_http(schema_target.fe_instance().clone(), false, false).await;
schema_requests.fail_batch.store(false, Ordering::SeqCst);
let schema_state = destination.path().join("schema-only-state.json");
let mut schema_import = vec![
tests-integration/tests/http.rs
+319
View File
@@ -227,6 +227,7 @@ macro_rules! http_tests {
test_splunk_raw,
test_log_query,
test_jaeger_query_api,
test_jaeger_v3_query_api,
test_jaeger_query_api_for_trace_v1,
test_jaeger_query_api_for_trace_v2,
@@ -11494,6 +11495,324 @@ pub async fn test_log_query(store_type: StorageType) {
guard.remove_all().await;
}
pub async fn test_jaeger_v3_query_api(
store_type: StorageType,
) -> Result<(), Box<dyn std::error::Error>> {
let (app, mut guard) =
setup_test_http_app_with_frontend(store_type, "test_jaeger_v3_query_api").await;
let client = TestClient::new(app).await;
for (path, expected_status, expected) in [
("services", StatusCode::OK, json!({"services":[]})),
(
"trace-summaries?query.startTimeMin=2026-01-01T00:00:00Z&query.startTimeMax=2026-01-02T00:00:00Z",
StatusCode::OK,
json!({"summaries":[]}),
),
(
"operations?service=checkout",
StatusCode::OK,
json!({"operations":[]}),
),
(
"traces/00000000000000000000000000000001",
StatusCode::NOT_FOUND,
json!({"error":{"httpCode":404,"message":"No traces found"}}),
),
] {
let response = client
.get(&format!("/v1/jaeger/api/v3/{path}"))
.send()
.await;
assert_eq!(response.status(), expected_status, "{path}");
assert_eq!(
serde_json::from_str::<Value>(&response.text().await)?,
expected
);
}
for path in [
"operations",
"traces",
"traces/not-hex",
"traces/0000000000000001?startTime=invalid",
"traces/0000000000000001?rawTraces=invalid",
"traces?query.startTimeMin=2026-01-01T00:00:00Z&query.startTimeMax=2026-01-02T00:00:00Z&query.searchDepth=10001",
"trace-summaries?query.startTimeMin=2026-01-01T00:00:00Z&query.startTimeMax=2026-01-02T00:00:00Z&query.searchDepth=10001",
] {
let response = client
.get(&format!("/v1/jaeger/api/v3/{path}"))
.send()
.await;
assert_eq!(response.status(), StatusCode::BAD_REQUEST, "{path}");
let body: Value = serde_json::from_str(&response.text().await)?;
assert_eq!(body["error"]["httpCode"], 400);
}
// The parent starts before the search window. A matching child must return the whole trace.
let start = chrono::Utc::now()
.timestamp_nanos_opt()
.ok_or("Timestamp out of range")? as u64;
let start = start / 1_000_000_000 * 1_000_000_000 + 123;
let timestamp = |ns: u64| {
chrono::DateTime::from_timestamp_nanos(ns as i64)
.to_rfc3339_opts(chrono::SecondsFormat::Nanos, true)
};
let request: ExportTraceServiceRequest = serde_json::from_value(json!({"resourceSpans":[
{"resource":{"attributes":[make_string_attr("service.name","checkout"), make_string_attr("service.instance.id","one")]},
"scopeSpans":[{"scope":{"name":"sdk","version":"1.0","attributes":[make_string_attr("scope.key","value"), {"key":"scope.bytes","value":{"bytesValue":"AQID/w=="}}]},"spans":[{
"traceId":"000000000000000000000000000000ab", "spanId":"0000000000000001", "name":"parent", "kind":2,
"startTimeUnixNano":start.to_string(), "endTimeUnixNano":(start+2000).to_string()
}]}]},
{"resource":{"attributes":[make_string_attr("service.name","checkout"), make_string_attr("service.instance.id","two"), {"key":"resource.bytes","value":{"bytesValue":"AQID/w=="}}]},
"scopeSpans":[{"scope":{"name":"sdk","version":"1.0","attributes":[make_string_attr("scope.key","value"), {"key":"scope.bytes","value":{"bytesValue":"AQID/w=="}}]},"spans":[{
"traceId":"000000000000000000000000000000ab", "spanId":"0000000000000002", "parentSpanId":"0000000000000001", "name":"child", "kind":3,
"traceState":"vendor=value", "startTimeUnixNano":(start+1000).to_string(), "endTimeUnixNano":(start+1999).to_string(),
"attributes":[make_int_attr("http.status_code",500),make_bool_attr("failed",true),
{"key":"bytes","value":{"bytesValue":"AQID/w=="}},
{"key":"integers","value":{"arrayValue":{"values":[{"intValue":"1"},{"intValue":"2"},{"intValue":"3"},{"intValue":"255"}]}}},
{"key":"nested","value":{"arrayValue":{"values":[{"intValue":"9223372036854775807"},{"kvlistValue":{"values":[make_string_attr("key","value")]}}]}}}],
"events":[{"name":"exception","timeUnixNano":(start+1011).to_string(),"attributes":[make_string_attr("message","failure")]}],
"links":[{"traceId":"000000000000000000000000000000cd","spanId":"0000000000000003","traceState":"link=value","attributes":[make_bool_attr("remote",true)]}],
"status":{"code":2,"message":"failure"}
}, {
"traceId":"000000000000000000000000000000cd", "spanId":"0000000000000003", "name":"boundary", "kind":2,
"startTimeUnixNano":(start+2000).to_string(), "endTimeUnixNano":(start+3000).to_string()
}]}]}
]}))?;
for pipeline in [
"greptime_trace_v0",
GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME,
GREPTIME_INTERNAL_TRACE_PIPELINE_V2_NAME,
] {
let table = format!("jaeger_api_v3_{pipeline}");
let response = send_req(
&client,
vec![
(
HeaderName::from_static("content-type"),
HeaderValue::from_static("application/x-protobuf"),
),
(
HeaderName::from_static("x-greptime-pipeline-name"),
HeaderValue::from_str(pipeline)?,
),
(
HeaderName::from_static("x-greptime-trace-table-name"),
HeaderValue::from_str(&table)?,
),
],
"/v1/otlp/v1/traces",
request.encode_to_vec(),
false,
)
.await;
assert_eq!(response.status(), StatusCode::OK);
for (path, expected) in [
("services", json!({"services":["checkout"]})),
(
"operations?service=checkout&span_kind=client",
json!({"operations":[{"name":"child","spanKind":"client"}]}),
),
(
"operations?service=checkout&spanKind=client",
json!({"operations":[{"name":"child","spanKind":"client"}]}),
),
] {
let response = client
.get(&format!("/v1/jaeger/api/v3/{path}"))
.header("x-greptime-trace-table-name", &table)
.send()
.await;
assert_eq!(response.status(), StatusCode::OK);
assert_eq!(
serde_json::from_str::<Value>(&response.text().await)?,
expected
);
}
for flushed in [false, true] {
if flushed {
let response = client
.post("/v1/sql")
.form(&[("sql", format!("ADMIN FLUSH_TABLE('{table}')"))])
.send()
.await;
assert_eq!(response.status(), StatusCode::OK);
}
let response = client.get(&format!(
"/v1/jaeger/api/v3/trace-summaries?query.serviceName=checkout&query.startTimeMin={}&query.startTimeMax={}",
timestamp(start+1000), timestamp(start+2000),
)).header("x-greptime-trace-table-name", &table).send().await;
assert_eq!(response.status(), StatusCode::OK);
let body: Value = serde_json::from_str(&response.text().await)?;
assert_eq!(
body,
json!({"summaries":[{
"traceId":"000000000000000000000000000000ab",
"rootServiceName":"checkout", "rootOperationName":"parent",
"minStartTimeUnixNano":start.to_string(), "maxEndTimeUnixNano":(start+2000).to_string(),
"spanCount":2, "errorSpanCount":1, "orphanSpanCount":0,
"services":[{"name":"checkout", "spanCount":2, "errorSpanCount":1}],
}]})
);
for path in [
"traces/00000000000000AB".to_string(),
format!(
"traces/00000000000000AB?startTime={}",
timestamp(start + 1000)
),
format!("traces/00000000000000AB?endTime={}", timestamp(start + 500)),
format!(
"traces/00000000000000AB?start_time={}&end_time={}",
timestamp(start + 500),
timestamp(start + 999)
),
format!(
"traces?query.startTimeMin={}&query.startTimeMax={}&query.searchDepth=0",
timestamp(start + 1000),
timestamp(start + 2000)
),
format!(
"traces?query.service_name=checkout&query.operation_name=child&query.start_time_min={}&query.start_time_max={}&query.num_traces=1&query.duration_min=999ns&query.duration_max=999ns&query.attributes={}",
timestamp(start + 1000),
timestamp(start + 2000),
encode(r#"{"http.status_code":"500","failed":"true"}"#)
),
format!(
"traces?query.startTimeMin={}&query.startTimeMax={}&query.attributes={}",
timestamp(start + 1000),
timestamp(start + 2000),
encode(r#"{"service.instance.id":"two"}"#)
),
] {
let response = client
.get(&format!("/v1/jaeger/api/v3/{path}"))
.header("x-greptime-trace-table-name", &table)
.send()
.await;
let status = response.status();
let body: Value = serde_json::from_str(&response.text().await)?;
assert_eq!(status, StatusCode::OK, "{pipeline}: {path}: {body}");
let resources = body["result"]["resourceSpans"]
.as_array()
.ok_or("Missing resource spans")?;
let spans: Vec<_> = resources
.iter()
.flat_map(|r| r["scopeSpans"][0]["spans"].as_array().unwrap())
.collect();
assert_eq!(spans.len(), 2, "{body}");
assert!(
spans
.iter()
.all(|span| span["traceId"] == "000000000000000000000000000000ab")
);
let child = spans
.iter()
.find(|span| span["name"] == "child")
.ok_or("Missing child span")?;
assert_eq!(child["startTimeUnixNano"], (start + 1000).to_string());
assert_eq!(child["endTimeUnixNano"], (start + 1999).to_string());
assert_eq!(child["parentSpanId"], "0000000000000001");
assert_eq!(child["kind"], 3);
assert_eq!(child["status"]["code"], 2);
assert_eq!(child["traceState"], "vendor=value");
assert_eq!(
child["events"][0]["timeUnixNano"],
(start + 1011).to_string()
);
assert_eq!(child["links"][0]["traceState"], "link=value");
assert_eq!(
child["links"][0]["attributes"][0]["value"]["boolValue"],
true
);
let nested = child["attributes"]
.as_array()
.unwrap()
.iter()
.find(|attr| attr["key"] == "nested")
.unwrap();
assert_eq!(
nested["value"]["arrayValue"]["values"][0]["intValue"],
"9223372036854775807"
);
let attribute = |values: &Value, key: &str| {
values
.as_array()
.unwrap()
.iter()
.find(|attr| attr["key"] == key)
.unwrap()["value"]
.clone()
};
assert_eq!(
attribute(&child["attributes"], "integers"),
json!({"arrayValue":{"values":[{"intValue":"1"},{"intValue":"2"},{"intValue":"3"},{"intValue":"255"}]}})
);
if pipeline == GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME {
assert_eq!(
attribute(&child["attributes"], "bytes"),
json!({"bytesValue":"AQID/w=="})
);
}
for resource in resources {
let span = &resource["scopeSpans"][0]["spans"][0];
let attrs = resource["resource"]["attributes"].as_array().unwrap();
let instance = attrs
.iter()
.find(|attr| attr["key"] == "service.instance.id")
.unwrap();
assert_eq!(
instance["value"]["stringValue"],
if span["name"] == "parent" {
"one"
} else {
"two"
}
);
let scope_attrs = &resource["scopeSpans"][0]["scope"]["attributes"];
assert_eq!(
attribute(scope_attrs, "scope.key"),
json!({"stringValue":"value"})
);
if pipeline == GREPTIME_INTERNAL_TRACE_PIPELINE_V1_NAME {
assert_eq!(
attribute(scope_attrs, "scope.bytes"),
json!({"bytesValue":"AQID/w=="})
);
if span["name"] == "child" {
assert_eq!(
attribute(&resource["resource"]["attributes"], "resource.bytes"),
json!({"bytesValue":"AQID/w=="})
);
}
}
}
}
// Instance `one` is on the parent, which starts before the search window.
let response = client
.get(&format!(
"/v1/jaeger/api/v3/traces?query.startTimeMin={}&query.startTimeMax={}&query.attributes={}",
timestamp(start + 1000),
timestamp(start + 2000),
encode(r#"{"service.instance.id":"one"}"#)
))
.header("x-greptime-trace-table-name", &table)
.send()
.await;
assert_eq!(response.status(), StatusCode::NOT_FOUND, "{pipeline}");
}
// Legacy clients continue receiving Jaeger JSON at the old URL.
let response = client
.get("/v1/jaeger/api/traces/000000000000000000000000000000ab")
.header("x-greptime-trace-table-name", &table)
.send()
.await;
assert_eq!(response.status(), StatusCode::OK);
let body: Value = serde_json::from_str(&response.text().await)?;
assert_eq!(body["data"][0]["spans"].as_array().unwrap().len(), 2);
}
guard.remove_all().await;
Ok(())
}
pub async fn test_jaeger_query_api(store_type: StorageType) {
common_telemetry::init_default_ut_logging();
let (app, mut guard) =
tests/cases/distributed/explain/step_aggr_advance.result
+55 -53
View File
@@ -21,26 +21,26 @@ Affected Rows: 0
-- SQLNESS REPLACE (Hash.*) REDACTED
tql explain (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr_optimize_not [2m]));
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| plan_type | plan |
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| logical_plan | MergeSort: aggr_optimize_not.a ASC NULLS LAST, aggr_optimize_not.b ASC NULLS LAST, aggr_optimize_not.c ASC NULLS LAST, aggr_optimize_not.greptime_timestamp ASC NULLS LAST |
| | MergeScan [is_placeholder=false, remote_input=[ |
| | Sort: aggr_optimize_not.a ASC NULLS LAST, aggr_optimize_not.b ASC NULLS LAST, aggr_optimize_not.c ASC NULLS LAST, aggr_optimize_not.greptime_timestamp ASC NULLS LAST |
| | Aggregate: groupBy=[[aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.greptime_timestamp]], aggr=[[max(prom_max_over_time(greptime_timestamp_range,greptime_value))]] |
| | Filter: prom_max_over_time(greptime_timestamp_range,greptime_value) IS NOT NULL |
| | Projection: aggr_optimize_not.greptime_timestamp, prom_max_over_time(greptime_timestamp_range, greptime_value) AS prom_max_over_time(greptime_timestamp_range,greptime_value), aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d |
| | PromRangeManipulate: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp], values=["greptime_value"] |
| | PromSeriesNormalize: offset=[0], time index=[greptime_timestamp], filter NaN: [true] |
| | PromSeriesDivide: tags=["a", "b", "c", "d"] |
| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST |
| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) |
| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] |
| | ]] |
| physical_plan | MergeSortExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] |
+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| plan_type | plan |
+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| logical_plan | MergeSort: aggr_optimize_not.a ASC NULLS LAST, aggr_optimize_not.b ASC NULLS LAST, aggr_optimize_not.c ASC NULLS LAST, aggr_optimize_not.greptime_timestamp ASC NULLS LAST |
| | MergeScan [is_placeholder=false, remote_input=[ |
| | Sort: aggr_optimize_not.a ASC NULLS LAST, aggr_optimize_not.b ASC NULLS LAST, aggr_optimize_not.c ASC NULLS LAST, aggr_optimize_not.greptime_timestamp ASC NULLS LAST |
| | Aggregate: groupBy=[[aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.greptime_timestamp]], aggr=[[prom_max(prom_max_over_time(greptime_timestamp_range,greptime_value)) AS max(prom_max_over_time(greptime_timestamp_range,greptime_value))]] |
| | Filter: prom_max_over_time(greptime_timestamp_range,greptime_value) IS NOT NULL |
| | Projection: aggr_optimize_not.greptime_timestamp, prom_max_over_time(greptime_timestamp_range, greptime_value) AS prom_max_over_time(greptime_timestamp_range,greptime_value), aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d |
| | PromRangeManipulate: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp], values=["greptime_value"] |
| | PromSeriesNormalize: offset=[0], time index=[greptime_timestamp], filter NaN: [true] |
| | PromSeriesDivide: tags=["a", "b", "c", "d"] |
| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST |
| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) |
| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] |
| | ]] |
| physical_plan | MergeSortExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] |
| | MergeScanExec: REDACTED
| | |
+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| | |
+---------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-- SQLNESS REPLACE (metrics.*) REDACTED
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
@@ -57,11 +57,12 @@ tql analyze (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr
| 0_| 0_|_MergeSortExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED
|_|_|_MergeScanExec: REDACTED
|_|_|_|
| 1_| 0_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED
| 1_| 0_|_ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp, prom_max(prom_max_over_time(greptime_timestamp_range,greptime_value))@4 as max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED
|_|_|_SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] REDACTED
|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp], aggr=[max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp], aggr=[prom_max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_RepartitionExec: partitioning=REDACTED
|_|_|_AggregateExec: mode=Partial, gby=[a@2 as a, b@3 as b, c@4 as c, greptime_timestamp@0 as greptime_timestamp], aggr=[max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_AggregateExec: mode=Partial, gby=[a@2 as a, b@3 as b, c@4 as c, greptime_timestamp@0 as greptime_timestamp], aggr=[prom_max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_FilterExec: prom_max_over_time(greptime_timestamp_range,greptime_value)@1 IS NOT NULL REDACTED
|_|_|_ProjectionExec: expr=[greptime_timestamp@4 as greptime_timestamp, prom_max_over_time(greptime_timestamp_range@6, greptime_value@5) as prom_max_over_time(greptime_timestamp_range,greptime_value), a@0 as a, b@1 as b, c@2 as c] REDACTED
|_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED
@@ -69,11 +70,12 @@ tql analyze (1752591864, 1752592164, '30s') max by (a, b, c) (max_over_time(aggr
|_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED
|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED
|_|_|_|
| 1_| 1_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED
| 1_| 1_|_ProjectionExec: expr=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp, prom_max(prom_max_over_time(greptime_timestamp_range,greptime_value))@4 as max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_SortPreservingMergeExec: [a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED
|_|_|_SortExec: expr=[a@0 ASC NULLS LAST, b@1 ASC NULLS LAST, c@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] REDACTED
|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp], aggr=[max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_AggregateExec: mode=FinalPartitioned, gby=[a@0 as a, b@1 as b, c@2 as c, greptime_timestamp@3 as greptime_timestamp], aggr=[prom_max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_RepartitionExec: partitioning=REDACTED
|_|_|_AggregateExec: mode=Partial, gby=[a@2 as a, b@3 as b, c@4 as c, greptime_timestamp@0 as greptime_timestamp], aggr=[max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_AggregateExec: mode=Partial, gby=[a@2 as a, b@3 as b, c@4 as c, greptime_timestamp@0 as greptime_timestamp], aggr=[prom_max(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_FilterExec: prom_max_over_time(greptime_timestamp_range,greptime_value)@1 IS NOT NULL REDACTED
|_|_|_ProjectionExec: expr=[greptime_timestamp@4 as greptime_timestamp, prom_max_over_time(greptime_timestamp_range@6, greptime_value@5) as prom_max_over_time(greptime_timestamp_range,greptime_value), a@0 as a, b@1 as b, c@2 as c] REDACTED
|_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED
@@ -305,31 +307,31 @@ tql analyze (1752591864, 1752592164, '30s') count by (a, b, c, d) (max_over_time
-- SQLNESS REPLACE (Hash.*) REDACTED
tql explain (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr_optimize_not [2m]));
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| plan_type | plan |
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| logical_plan | Sort: aggr_optimize_not.b ASC NULLS LAST, aggr_optimize_not.c ASC NULLS LAST, aggr_optimize_not.d ASC NULLS LAST, aggr_optimize_not.greptime_timestamp ASC NULLS LAST |
| | Aggregate: groupBy=[[aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d, aggr_optimize_not.greptime_timestamp]], aggr=[[__min_merge(__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) AS min(prom_max_over_time(greptime_timestamp_range,greptime_value))]] |
| | MergeScan [is_placeholder=false, remote_input=[ |
| | Aggregate: groupBy=[[aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d, aggr_optimize_not.greptime_timestamp]], aggr=[[__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))]] |
| | Filter: prom_max_over_time(greptime_timestamp_range,greptime_value) IS NOT NULL |
| | Projection: aggr_optimize_not.greptime_timestamp, prom_max_over_time(greptime_timestamp_range, greptime_value) AS prom_max_over_time(greptime_timestamp_range,greptime_value), aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d |
| | PromRangeManipulate: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp], values=["greptime_value"] |
| | PromSeriesNormalize: offset=[0], time index=[greptime_timestamp], filter NaN: [true] |
| | PromSeriesDivide: tags=["a", "b", "c", "d"] |
| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST |
| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) |
| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] |
| | ]] |
| physical_plan | SortPreservingMergeExec: [b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] |
| | SortExec: expr=[b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] |
| | AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__min_merge(__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] |
+---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| plan_type | plan |
+---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| logical_plan | Sort: aggr_optimize_not.b ASC NULLS LAST, aggr_optimize_not.c ASC NULLS LAST, aggr_optimize_not.d ASC NULLS LAST, aggr_optimize_not.greptime_timestamp ASC NULLS LAST |
| | Aggregate: groupBy=[[aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d, aggr_optimize_not.greptime_timestamp]], aggr=[[__prom_min_merge(__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) AS min(prom_max_over_time(greptime_timestamp_range,greptime_value))]] |
| | MergeScan [is_placeholder=false, remote_input=[ |
| | Aggregate: groupBy=[[aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d, aggr_optimize_not.greptime_timestamp]], aggr=[[__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))]] |
| | Filter: prom_max_over_time(greptime_timestamp_range,greptime_value) IS NOT NULL |
| | Projection: aggr_optimize_not.greptime_timestamp, prom_max_over_time(greptime_timestamp_range, greptime_value) AS prom_max_over_time(greptime_timestamp_range,greptime_value), aggr_optimize_not.a, aggr_optimize_not.b, aggr_optimize_not.c, aggr_optimize_not.d |
| | PromRangeManipulate: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp], values=["greptime_value"] |
| | PromSeriesNormalize: offset=[0], time index=[greptime_timestamp], filter NaN: [true] |
| | PromSeriesDivide: tags=["a", "b", "c", "d"] |
| | Sort: aggr_optimize_not.a ASC NULLS FIRST, aggr_optimize_not.b ASC NULLS FIRST, aggr_optimize_not.c ASC NULLS FIRST, aggr_optimize_not.d ASC NULLS FIRST, aggr_optimize_not.greptime_timestamp ASC NULLS FIRST |
| | Filter: aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None) AND aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None) |
| | TableScan: aggr_optimize_not, partial_filters=[aggr_optimize_not.greptime_timestamp >= TimestampMillisecond(1752591744001, None), aggr_optimize_not.greptime_timestamp <= TimestampMillisecond(1752592164000, None)] |
| | ]] |
| physical_plan | SortPreservingMergeExec: [b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] |
| | SortExec: expr=[b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] |
| | AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__prom_min_merge(__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] |
| | RepartitionExec: partitioning=REDACTED
| | AggregateExec: mode=Partial, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__min_merge(__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] |
| | AggregateExec: mode=Partial, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__prom_min_merge(__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] |
| | RepartitionExec: partitioning=REDACTED
| | MergeScanExec: REDACTED
| | |
+---------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| | |
+---------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
-- SQLNESS REPLACE (metrics.*) REDACTED
-- SQLNESS REPLACE (RoundRobinBatch.*) REDACTED
@@ -345,15 +347,15 @@ tql analyze (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr
+-+-+-+
| 0_| 0_|_SortPreservingMergeExec: [b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST] REDACTED
|_|_|_SortExec: expr=[b@0 ASC NULLS LAST, c@1 ASC NULLS LAST, d@2 ASC NULLS LAST, greptime_timestamp@3 ASC NULLS LAST], preserve_partitioning=[true] REDACTED
|_|_|_AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__min_merge(__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__prom_min_merge(__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_RepartitionExec: partitioning=REDACTED
|_|_|_AggregateExec: mode=Partial, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__min_merge(__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_AggregateExec: mode=Partial, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__prom_min_merge(__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))) as min(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_RepartitionExec: partitioning=REDACTED
|_|_|_MergeScanExec: REDACTED
|_|_|_|
| 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
| 1_| 0_|_AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_RepartitionExec: partitioning=REDACTED
|_|_|_AggregateExec: mode=Partial, gby=[b@2 as b, c@3 as c, d@4 as d, greptime_timestamp@0 as greptime_timestamp], aggr=[__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_AggregateExec: mode=Partial, gby=[b@2 as b, c@3 as c, d@4 as d, greptime_timestamp@0 as greptime_timestamp], aggr=[__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_FilterExec: prom_max_over_time(greptime_timestamp_range,greptime_value)@1 IS NOT NULL REDACTED
|_|_|_ProjectionExec: expr=[greptime_timestamp@4 as greptime_timestamp, prom_max_over_time(greptime_timestamp_range@6, greptime_value@5) as prom_max_over_time(greptime_timestamp_range,greptime_value), b@1 as b, c@2 as c, d@3 as d] REDACTED
|_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED
@@ -361,9 +363,9 @@ tql analyze (1752591864, 1752592164, '30s') min by (b, c, d) (max_over_time(aggr
|_|_|_PromSeriesDivideExec: tags=["a", "b", "c", "d"] REDACTED
|_|_|_SeriesScan: region=REDACTED, "partition_count":{"count":0, "mem_ranges":0, "files":0, "file_ranges":0}, "distribution":"PerSeries", "mode":"legacy" REDACTED
|_|_|_|
| 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
| 1_| 1_|_AggregateExec: mode=FinalPartitioned, gby=[b@0 as b, c@1 as c, d@2 as d, greptime_timestamp@3 as greptime_timestamp], aggr=[__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_RepartitionExec: partitioning=REDACTED
|_|_|_AggregateExec: mode=Partial, gby=[b@2 as b, c@3 as c, d@4 as d, greptime_timestamp@0 as greptime_timestamp], aggr=[__min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_AggregateExec: mode=Partial, gby=[b@2 as b, c@3 as c, d@4 as d, greptime_timestamp@0 as greptime_timestamp], aggr=[__prom_min_state(prom_max_over_time(greptime_timestamp_range,greptime_value))] REDACTED
|_|_|_FilterExec: prom_max_over_time(greptime_timestamp_range,greptime_value)@1 IS NOT NULL REDACTED
|_|_|_ProjectionExec: expr=[greptime_timestamp@4 as greptime_timestamp, prom_max_over_time(greptime_timestamp_range@6, greptime_value@5) as prom_max_over_time(greptime_timestamp_range,greptime_value), b@1 as b, c@2 as c, d@3 as d] REDACTED
|_|_|_PromRangeManipulateExec: req range=[1752591864000..1752592164000], interval=[30000], eval range=[120000], time index=[greptime_timestamp] REDACTED
tests/cases/standalone/common/promql/count_values.result
+1 -1
View File
@@ -84,7 +84,7 @@ TQL EVAL (0, 15, '5s') abs(count_values("v", http_requests));
TQL EVAL (0, 15, '5s') round(count_values("v", http_requests));
+---------------------+-------------------------------------------------+-----+
| ts | prom_round(count(http_requests.val),Float64(0)) | v |
| ts | prom_round(count(http_requests.val),Float64(1)) | v |
+---------------------+-------------------------------------------------+-----+
| 1970-01-01T00:00:00 | 1.0 | 401 |
| 1970-01-01T00:00:00 | 3.0 | 200 |
tests/cases/standalone/common/promql/delta_temporality.result
+1 -1
View File
@@ -162,7 +162,7 @@ TQL EVAL (180, 180, '1m') round(sum(rate(delta_temporality[3m])), 0.000001);
+---------------------+----------------------------------------------------------------------------------------+
| ts | prom_round(sum(prom_rate(ts_range,greptime_value,ts,Int64(180000))),Float64(0.000001)) |
+---------------------+----------------------------------------------------------------------------------------+
| 1970-01-01T00:03:00 | 0.45555599999999996 |
| 1970-01-01T00:03:00 | 0.455556 |
+---------------------+----------------------------------------------------------------------------------------+
CREATE TABLE delta_marker_only (
tests/cases/standalone/common/promql/dotted_column_names.result
+7 -7
View File
@@ -424,9 +424,9 @@ TQL EXPLAIN (0, 10, '5s') topk(1, {"otel.m"});
| plan_type_| plan_|
+-+-+
| logical_plan_| Projection: otel.m.v.val, otel.m.service.name, otel.m.Host, otel.m.host, otel.m.ts.time_|
|_|_Sort: otel.m.ts.time ASC NULLS LAST, row_number() PARTITION BY [otel.m.ts.time] ORDER BY [otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ASC NULLS LAST_|
|_|_Filter: CAST(row_number() PARTITION BY [otel.m.ts.time] ORDER BY [otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS Float64) <= Float64(1)_|
|_|_WindowAggr: windowExpr=[[row_number() PARTITION BY [otel.m.ts.time] ORDER BY [otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]_|
|_|_Sort: otel.m.ts.time ASC NULLS LAST, row_number() PARTITION BY [otel.m.ts.time] ORDER BY [isnan(otel.m.v.val) ASC NULLS FIRST, otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW ASC NULLS LAST_|
|_|_Filter: CAST(row_number() PARTITION BY [otel.m.ts.time] ORDER BY [isnan(otel.m.v.val) ASC NULLS FIRST, otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW AS Float64) <= Float64(1)_|
|_|_WindowAggr: windowExpr=[[row_number() PARTITION BY [otel.m.ts.time] ORDER BY [isnan(otel.m.v.val) ASC NULLS FIRST, otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW]]_|
|_|_MergeScan [is_placeholder=false, remote_input=[_|
|_| PromInstantManipulate: range=[0..10000], lookback=[300000], interval=[5000], time index=[ts.time]_|
|_|_PromSeriesDivide: tags=["service.name", "Host", "host"]_|
@@ -435,10 +435,10 @@ TQL EXPLAIN (0, 10, '5s') topk(1, {"otel.m"});
|_|_TableScan: otel.m, partial_filters=[otel.m.ts.time >= TimestampMillisecond(-299999, None), otel.m.ts.time <= TimestampMillisecond(10000, None)]_|
|_| ]]_|
| physical_plan | ProjectionExec: expr=[v.val@4 as v.val, service.name@1 as service.name, Host@2 as Host, host@3 as host, ts.time@0 as ts.time]_|
|_|_SortPreservingMergeExec: [ts.time@0 ASC NULLS LAST, row_number() PARTITION BY [otel.m.ts.time] ORDER BY [otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 ASC NULLS LAST]_|
|_|_FilterExec: CAST(row_number() PARTITION BY [otel.m.ts.time] ORDER BY [otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 AS Float64) <= 1_|
|_|_BoundedWindowAggExec: wdw=[row_number() PARTITION BY [otel.m.ts.time] ORDER BY [otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [otel.m.ts.time] ORDER BY [otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] |
|_|_SortExec: expr=[ts.time@0 ASC NULLS LAST, v.val@4 DESC, service.name@1 DESC, Host@2 DESC, host@3 DESC], preserve_REDACTED
|_|_SortPreservingMergeExec: [ts.time@0 ASC NULLS LAST, row_number() PARTITION BY [otel.m.ts.time] ORDER BY [isnan(otel.m.v.val) ASC NULLS FIRST, otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 ASC NULLS LAST]_|
|_|_FilterExec: CAST(row_number() PARTITION BY [otel.m.ts.time] ORDER BY [isnan(otel.m.v.val) ASC NULLS FIRST, otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW@5 AS Float64) <= 1_|
|_|_BoundedWindowAggExec: wdw=[row_number() PARTITION BY [otel.m.ts.time] ORDER BY [isnan(otel.m.v.val) ASC NULLS FIRST, otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW: Field { "row_number() PARTITION BY [otel.m.ts.time] ORDER BY [isnan(otel.m.v.val) ASC NULLS FIRST, otel.m.v.val DESC NULLS FIRST, otel.m.service.name DESC NULLS FIRST, otel.m.Host DESC NULLS FIRST, otel.m.host DESC NULLS FIRST] ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW": UInt64 }, frame: ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW], mode=[Sorted] |
|_|_SortExec: expr=[ts.time@0 ASC NULLS LAST, isnan(v.val@4) ASC, v.val@4 DESC, service.name@1 DESC, Host@2 DESC, host@3 DESC], preserve_REDACTED
|_|_RepartitionExec: REDACTED
|_|_MergeScanExec: REDACTED
|_|_|
tests/cases/standalone/common/promql/float_semantics.result
+115
View File
@@ -0,0 +1,115 @@
-- NaN and rounding follow Prometheus (IEEE 754) rather than SQL ordering.
-- Expected results are from Prometheus 3.
create table float_semantics (
ts timestamp time index,
val double,
host string primary key
);
Affected Rows: 0
insert into float_semantics values
(0, 1.0, 'a'),
(0, 2.0, 'b'),
(0, 'NaN'::double, 'c'),
(0, -2.5, 'd');
Affected Rows: 4
tql eval (0, 0, '1s') max(float_semantics);
+---------------------+--------------------------+
| ts | max(float_semantics.val) |
+---------------------+--------------------------+
| 1970-01-01T00:00:00 | 2.0 |
+---------------------+--------------------------+
tql eval (0, 0, '1s') min(float_semantics);
+---------------------+--------------------------+
| ts | min(float_semantics.val) |
+---------------------+--------------------------+
| 1970-01-01T00:00:00 | -2.5 |
+---------------------+--------------------------+
tql eval (0, 0, '1s') topk(1, float_semantics);
+-----+------+---------------------+
| val | host | ts |
+-----+------+---------------------+
| 2.0 | b | 1970-01-01T00:00:00 |
+-----+------+---------------------+
-- SQLNESS SORT_RESULT 3 1
tql eval (0, 0, '1s') float_semantics <= NaN;
+----+-----+------+
| ts | val | host |
+----+-----+------+
+----+-----+------+
-- SQLNESS SORT_RESULT 3 1
tql eval (0, 0, '1s') float_semantics != bool NaN;
+------+---------------------+---------------------+
| host | ts | val != Float64(NaN) |
+------+---------------------+---------------------+
| a | 1970-01-01T00:00:00 | 1.0 |
| b | 1970-01-01T00:00:00 | 1.0 |
| c | 1970-01-01T00:00:00 | 1.0 |
| d | 1970-01-01T00:00:00 | 1.0 |
+------+---------------------+---------------------+
-- SQLNESS SORT_RESULT 3 1
tql eval (0, 0, '1s') round(float_semantics);
+---------------------+----------------------------+------+
| ts | prom_round(val,Float64(1)) | host |
+---------------------+----------------------------+------+
| 1970-01-01T00:00:00 | -2.0 | d |
| 1970-01-01T00:00:00 | 1.0 | a |
| 1970-01-01T00:00:00 | 2.0 | b |
| 1970-01-01T00:00:00 | NaN | c |
+---------------------+----------------------------+------+
-- SQLNESS SORT_RESULT 3 1
tql eval (0, 0, '1s') sqrt(float_semantics);
+---------------------+--------------------+------+
| ts | sqrt(val) | host |
+---------------------+--------------------+------+
| 1970-01-01T00:00:00 | 1.0 | a |
| 1970-01-01T00:00:00 | 1.4142135623730951 | b |
| 1970-01-01T00:00:00 | NaN | c |
| 1970-01-01T00:00:00 | NaN | d |
+---------------------+--------------------+------+
-- prom_min/prom_max have their own column names, so SQL can use them next to min/max.
select min(val) as sql_lo, prom_min(val) as prom_lo from float_semantics;
+--------+---------+
| sql_lo | prom_lo |
+--------+---------+
| -2.5 | -2.5 |
+--------+---------+
select max(val) as sql_hi, prom_max(val) as prom_hi from float_semantics;
+--------+---------+
| sql_hi | prom_hi |
+--------+---------+
| NaN | 2.0 |
+--------+---------+
select prom_min(val) as lo from float_semantics having min(val) < 0;
+------+
| lo |
+------+
| -2.5 |
+------+
drop table float_semantics;
Affected Rows: 0
tests/cases/standalone/common/promql/float_semantics.sql
+41
View File
@@ -0,0 +1,41 @@
-- NaN and rounding follow Prometheus (IEEE 754) rather than SQL ordering.
-- Expected results are from Prometheus 3.
create table float_semantics (
ts timestamp time index,
val double,
host string primary key
);
insert into float_semantics values
(0, 1.0, 'a'),
(0, 2.0, 'b'),
(0, 'NaN'::double, 'c'),
(0, -2.5, 'd');
tql eval (0, 0, '1s') max(float_semantics);
tql eval (0, 0, '1s') min(float_semantics);
tql eval (0, 0, '1s') topk(1, float_semantics);
-- SQLNESS SORT_RESULT 3 1
tql eval (0, 0, '1s') float_semantics <= NaN;
-- SQLNESS SORT_RESULT 3 1
tql eval (0, 0, '1s') float_semantics != bool NaN;
-- SQLNESS SORT_RESULT 3 1
tql eval (0, 0, '1s') round(float_semantics);
-- SQLNESS SORT_RESULT 3 1
tql eval (0, 0, '1s') sqrt(float_semantics);
-- prom_min/prom_max have their own column names, so SQL can use them next to min/max.
select min(val) as sql_lo, prom_min(val) as prom_lo from float_semantics;
select max(val) as sql_hi, prom_max(val) as prom_hi from float_semantics;
select prom_min(val) as lo from float_semantics having min(val) < 0;
drop table float_semantics;
tests/cases/standalone/common/promql/functions.result
+6 -6
View File
@@ -109,7 +109,7 @@ tql eval (4, 4, '1s') quantile_over_time(0.4 + 0.1, prom_series_q[4s]);
tql eval (3, 3, '1s') round(prom_series_q);
+---------------------+----------------------------+------+
| ts | prom_round(val,Float64(0)) | host |
| ts | prom_round(val,Float64(1)) | host |
+---------------------+----------------------------+------+
| 1970-01-01T00:00:03 | 346.0 | q |
+---------------------+----------------------------+------+
@@ -133,8 +133,8 @@ tql eval (1, 4, '1s') round(prom_series_q, 0.05 + 0.05);
| ts | prom_round(val,Float64(0.05) + Float64(0.05)) | host |
+---------------------+-----------------------------------------------+------+
| 1970-01-01T00:00:01 | 123.5 | q |
| 1970-01-01T00:00:02 | 234.60000000000002 | q |
| 1970-01-01T00:00:03 | 345.70000000000005 | q |
| 1970-01-01T00:00:02 | 234.6 | q |
| 1970-01-01T00:00:03 | 345.7 | q |
| 1970-01-01T00:00:04 | 456.8 | q |
+---------------------+-----------------------------------------------+------+
@@ -156,10 +156,10 @@ tql eval (1, 4, '1s') round(prom_series_q, 100.0 + 3.0);
+---------------------+-------------------------------------------+------+
| ts | prom_round(val,Float64(100) + Float64(3)) | host |
+---------------------+-------------------------------------------+------+
| 1970-01-01T00:00:01 | 103.0 | q |
| 1970-01-01T00:00:02 | 206.0 | q |
| 1970-01-01T00:00:01 | 103.00000000000001 | q |
| 1970-01-01T00:00:02 | 206.00000000000003 | q |
| 1970-01-01T00:00:03 | 309.0 | q |
| 1970-01-01T00:00:04 | 412.0 | q |
| 1970-01-01T00:00:04 | 412.00000000000006 | q |
+---------------------+-------------------------------------------+------+
-- SQLNESS SORT_RESULT 3 1
tests/cases/standalone/common/promql/subquery_default_step.result
+53
View File
@@ -0,0 +1,53 @@
-- A subquery without a step uses the evaluation interval (one minute by
-- default in Prometheus), not the query step. Expected results are from Prometheus 3.
create table subquery_default_step (
ts timestamp time index,
val double,
host string primary key
);
Affected Rows: 0
insert into subquery_default_step values
(0, 1.0, 'a'),
(60000, 1.0, 'a'),
(120000, 1.0, 'a'),
(180000, 1.0, 'a'),
(240000, 1.0, 'a'),
(300000, 1.0, 'a');
Affected Rows: 6
tql eval (300, 300, '1s') count_over_time(subquery_default_step[5m:]);
+---------------------+------------------------------------+------+
| ts | prom_count_over_time(ts_range,val) | host |
+---------------------+------------------------------------+------+
| 1970-01-01T00:05:00 | 5.0 | a |
+---------------------+------------------------------------+------+
tql eval (300, 300, '1s') count_over_time(subquery_default_step[5m:30s]);
+---------------------+------------------------------------+------+
| ts | prom_count_over_time(ts_range,val) | host |
+---------------------+------------------------------------+------+
| 1970-01-01T00:05:00 | 10.0 | a |
+---------------------+------------------------------------+------+
-- The outer range query keeps its own step around the subquery's default step.
tql eval (300, 360, '15s') count_over_time(subquery_default_step[5m:]);
+---------------------+------------------------------------+------+
| ts | prom_count_over_time(ts_range,val) | host |
+---------------------+------------------------------------+------+
| 1970-01-01T00:05:00 | 5.0 | a |
| 1970-01-01T00:05:15 | 5.0 | a |
| 1970-01-01T00:05:30 | 5.0 | a |
| 1970-01-01T00:05:45 | 5.0 | a |
| 1970-01-01T00:06:00 | 5.0 | a |
+---------------------+------------------------------------+------+
drop table subquery_default_step;
Affected Rows: 0
tests/cases/standalone/common/promql/subquery_default_step.sql
+25
View File
@@ -0,0 +1,25 @@
-- A subquery without a step uses the evaluation interval (one minute by
-- default in Prometheus), not the query step. Expected results are from Prometheus 3.
create table subquery_default_step (
ts timestamp time index,
val double,
host string primary key
);
insert into subquery_default_step values
(0, 1.0, 'a'),
(60000, 1.0, 'a'),
(120000, 1.0, 'a'),
(180000, 1.0, 'a'),
(240000, 1.0, 'a'),
(300000, 1.0, 'a');
tql eval (300, 300, '1s') count_over_time(subquery_default_step[5m:]);
tql eval (300, 300, '1s') count_over_time(subquery_default_step[5m:30s]);
-- The outer range query keeps its own step around the subquery's default step.
tql eval (300, 360, '15s') count_over_time(subquery_default_step[5m:]);
drop table subquery_default_step;
tests/cases/standalone/common/promql/tsid_binary_join_regression.result
+6 -6
View File
@@ -346,7 +346,7 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > tsid_binary_join_right;
+-+-+-+
| stage | node | plan_|
+-+-+-+
| 0_| 0_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@3, __tsid@1), (ts@4, ts@2)], filter=greptime_value@1 < greptime_value@0, projection=[ts@4, greptime_value@0, host@1, job@2, __tsid@3], NullsEqual: true REDACTED
| 0_| 0_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@3, __tsid@1), (ts@4, ts@2)], filter=prom_gt(greptime_value@0, greptime_value@1), projection=[ts@4, greptime_value@0, host@1, job@2, __tsid@3], NullsEqual: true REDACTED
|_|_|_RepartitionExec: partitioning=Hash([__tsid@3, ts@4],REDACTED
|_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, host@1 as host, job@2 as job, __tsid@3 as __tsid, ts@4 as ts] REDACTED
|_|_|_MergeScanExec: REDACTED
@@ -387,8 +387,8 @@ TQL ANALYZE (0, 5, '5s') tsid_binary_join_left > bool tsid_binary_join_right;
+-+-+-+
| stage | node | plan_|
+-+-+-+
| 0_| 0_|_ProjectionExec: expr=[host@0 as host, job@1 as job, ts@2 as ts, __tsid@3 as __tsid, CAST(greptime_value@4 < greptime_value@5 AS Float64) as tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value] REDACTED
|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@2, ts@4)], projection=[host@4, job@5, ts@7, __tsid@6, greptime_value@3, greptime_value@0], NullsEqual: true REDACTED
| 0_| 0_|_ProjectionExec: expr=[host@0 as host, job@1 as job, ts@2 as ts, __tsid@3 as __tsid, CAST(prom_gt(greptime_value@4, greptime_value@5) AS Float64) as tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value] REDACTED
|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@1, __tsid@3), (ts@2, ts@4)], projection=[host@4, job@5, ts@7, __tsid@6, greptime_value@0, greptime_value@3], NullsEqual: true REDACTED
|_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED
|_|_|_CooperativeExec REDACTED
|_|_|_MergeScanExec: REDACTED
@@ -432,7 +432,7 @@ TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > tsid_binary_join_right) / tsi
| 0_| 0_|_ProjectionExec: expr=[host@0 as host, job@1 as job, ts@2 as ts, __tsid@3 as __tsid, greptime_value@4 / greptime_value@5 * 100 as .greptime_value / tsid_binary_join_left.greptime_value * Float64(100)] REDACTED
|_|_|_HashJoinExec: mode=CollectLeft, join_type=Inner, on=[(__tsid@2, __tsid@3), (ts@0, ts@4)], projection=[host@4, job@5, ts@7, __tsid@6, greptime_value@1, greptime_value@3], NullsEqual: true REDACTED
|_|_|_CoalescePartitionsExec REDACTED
|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@2, ts@2)], filter=greptime_value@1 < greptime_value@0, projection=[ts@2, greptime_value@0, __tsid@1], NullsEqual: true REDACTED
|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@2, ts@2)], filter=prom_gt(greptime_value@0, greptime_value@1), projection=[ts@2, greptime_value@0, __tsid@1], NullsEqual: true REDACTED
|_|_|_RepartitionExec: partitioning=Hash([REDACTED
|_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED
|_|_|_MergeScanExec: REDACTED
@@ -487,8 +487,8 @@ TQL ANALYZE (0, 5, '5s') ((tsid_binary_join_left > bool tsid_binary_join_right)
|_|_|_CoalescePartitionsExec REDACTED
|_|_|_ProjectionExec: expr=[ts@0 as ts, __tsid@1 as __tsid, tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value@2 + greptime_value@3 as tsid_binary_join_right.tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value + tsid_binary_join_left.greptime_value] REDACTED
|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@0, ts@2)], projection=[ts@5, __tsid@4, tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value@2, greptime_value@3], NullsEqual: true REDACTED
|_|_|_ProjectionExec: expr=[ts@0 as ts, __tsid@1 as __tsid, CAST(greptime_value@2 < greptime_value@3 AS Float64) as tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value] REDACTED
|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@2, ts@2)], projection=[ts@5, __tsid@4, greptime_value@3, greptime_value@0], NullsEqual: true REDACTED
|_|_|_ProjectionExec: expr=[ts@0 as ts, __tsid@1 as __tsid, CAST(prom_gt(greptime_value@2, greptime_value@3) AS Float64) as tsid_binary_join_left.greptime_value > tsid_binary_join_right.greptime_value] REDACTED
|_|_|_HashJoinExec: mode=Partitioned, join_type=Inner, on=[(__tsid@1, __tsid@1), (ts@2, ts@2)], projection=[ts@5, __tsid@4, greptime_value@0, greptime_value@3], NullsEqual: true REDACTED
|_|_|_RepartitionExec: partitioning=Hash([REDACTED
|_|_|_ProjectionExec: expr=[greptime_value@0 as greptime_value, __tsid@3 as __tsid, ts@4 as ts] REDACTED
|_|_|_MergeScanExec: REDACTED