rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-10-30 07:26:56 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
28697c9e550aa8dba3b39bbc873c68a96d96b047
greptimedb/tests-integration/src/tests/promql_test.rs
T
dennis zhuang f3eb8e6c72 test: cut integration test time and make the storage matrix meaningful (#9308) 
* test: cut integration test time and make the storage matrix meaningful

tests-integration is ~85% of workspace test CPU, and 81% of that is the
S3/S3WithCache variants of the HTTP and gRPC suites. Those suites do not
touch the object store: of the 70 matrix HTTP tests only one flushed and
read back an SST, so the matrix was paying real AWS round trips to
re-prove protocol parsing.

- Point the PR CI object-store matrix at the MinIO already started by
  tests-integration/fixtures. Three GT_S3_* consumers did not read
  GT_S3_ENDPOINT_URL and would have hit real AWS with MinIO credentials;
  they now do.
- Add a nightly Linux job against real AWS S3, and pass GT_S3_* into the
  release integration-test container. The release previously ran every
  remote-backend case as a skip and only exercised the file backend.
- Give each S3WithCache test its own read cache directory. They shared
  /tmp/greptimedb_cache, which the datanode wipes on startup, so a
  starting test deleted the read cache of a running one.
- Add flush -> read-back assertions to the tests whose columns have a
  non-trivial SST representation: JSON/JSON2 columns, native histograms,
  metric-engine logical tables, and tables carrying fulltext or skipping
  indexes whose puffin files only exist after a flush.
- Move eight tests that create no table out of the storage matrix.
- Make the event recorder flush interval a constructor parameter and
  shorten it in the event tests, which otherwise wait a 5s window per DDL
  they assert on. It is skipped by serde and never reaches config files.
- Drop duplicates: test_grpc_zstd_compression was a verbatim copy of
  test_grpc_message_size_ok and is now rewritten to assert the negotiated
  grpc-encoding; test_execute_copy_to_{s3,oss,gcs,azblob} were strict
  prefixes of their copy_from siblings; two standalone/distributed event
  test pairs shared one assertion body.
- Fix and un-ignore stddev_by_label. stddev_pop merges partial aggregates
  in a parallelism-dependent order, so its last digits are unstable; the
  test now compares values with a tolerance.
- Rebase the jaeger v1 fixture on the current instant. It carries
  ttl=7d with 2025 timestamps, so its rows were only readable as long as
  they stayed in the memtable.

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

* test: address review — wire nightly real-S3 job into check-status, keep the short event interval

The nightly `check-status` job did not depend on the new real-S3 job, so a
failure there would not have reached the status or Slack notification.

In database_ddl_event the short interval was set by a first
`with_event_recorder_options` call and then overwritten by the pre-existing
one, which carries `..Default::default()`. Merged into a single call.

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>

---------

Signed-off-by: Dennis Zhuang <killme2008@gmail.com>
2026-09-23 01:52:52 +00:00

1179 lines
41 KiB
Rust
Raw Blame History

// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::Arc;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use api::v1::value::ValueData;
use api::v1::{
ColumnDataType, ColumnSchema, Row, RowInsertRequest, RowInsertRequests, Rows, SemanticType,
};
use common_query::prometheus::PROMETHEUS_STALE_NAN_BITS;
use common_query::{Output, OutputData};
use common_recordbatch::util::collect_batches;
use datatypes::arrow::array::{Float64Array, Int64Array, StringArray, TimestampMillisecondArray};
use datatypes::arrow::compute::cast;
use datatypes::arrow::datatypes::DataType;
use frontend::instance::Instance;
use query::parser::{PromQuery, QueryLanguageParser, QueryStatement};
use rstest::rstest;
use rstest_reuse::apply;
use servers::query_handler::sql::SqlQueryHandler;
use session::context::QueryContext;
use super::test_util::{
both_instances_cases, check_unordered_output_stream, distributed, standalone,
};
use crate::tests::test_util::MockInstance;
#[allow(clippy::too_many_arguments)]
async fn promql_query(
ins: Arc<Instance>,
promql: &str,
alias: Option<String>,
query_ctx: Arc<QueryContext>,
start: SystemTime,
end: SystemTime,
interval: Duration,
lookback: Duration,
) -> operator::error::Result<Output> {
let query = PromQuery {
query: promql.to_string(),
alias,
..PromQuery::default()
};
let QueryStatement::Promql(mut eval_stmt, alias) =
QueryLanguageParser::parse_promql(&query, &query_ctx).unwrap()
else {
unreachable!()
};
eval_stmt.start = start;
eval_stmt.end = end;
eval_stmt.interval = interval;
eval_stmt.lookback_delta = lookback;
ins.statement_executor()
.execute_stmt(QueryStatement::Promql(eval_stmt, alias), query_ctx)
.await
}
#[allow(clippy::too_many_arguments)]
async fn create_insert_query_assert(
instance: Arc<Instance>,
create: &str,
insert: &str,
promql: &str,
start: SystemTime,
end: SystemTime,
interval: Duration,
lookback: Duration,
expected: &str,
) {
create_insert_query_assert_with_alias(
instance, create, insert, promql, None, start, end, interval, lookback, expected,
)
.await
}
#[allow(clippy::too_many_arguments)]
async fn create_insert_query_assert_with_alias(
instance: Arc<Instance>,
create: &str,
insert: &str,
promql: &str,
alias: Option<String>,
start: SystemTime,
end: SystemTime,
interval: Duration,
lookback: Duration,
expected: &str,
) {
instance
.do_query(create, QueryContext::arc())
.await
.into_iter()
.for_each(|v| {
let _ = v.unwrap();
});
instance
.do_query(insert, QueryContext::arc())
.await
.into_iter()
.for_each(|v| {
let _ = v.unwrap();
});
let query_output = promql_query(
instance,
promql,
alias,
QueryContext::arc(),
start,
end,
interval,
lookback,
)
.await
.unwrap();
check_unordered_output_stream(query_output, expected).await;
}
#[allow(clippy::too_many_arguments)]
async fn create_insert_tql_assert(
instance: Arc<Instance>,
create: &str,
insert: &str,
tql: &str,
expected: &str,
) {
instance
.do_query(create, QueryContext::arc())
.await
.into_iter()
.for_each(|v| {
let _ = v.unwrap();
});
instance
.do_query(insert, QueryContext::arc())
.await
.into_iter()
.for_each(|v| {
let _ = v.unwrap();
});
let query_output = instance
.do_query(tql, QueryContext::arc())
.await
.remove(0)
.unwrap();
check_unordered_output_stream(query_output, expected).await;
}
async fn execute_all(instance: &Arc<Instance>, sql: &str, query_ctx: Arc<QueryContext>) {
instance
.do_query(sql, query_ctx)
.await
.into_iter()
.for_each(|v| {
let _ = v.unwrap();
});
}
#[allow(clippy::too_many_arguments)]
async fn promql_query_as_batches(
ins: Arc<Instance>,
promql: &str,
alias: Option<String>,
query_ctx: Arc<QueryContext>,
start: SystemTime,
end: SystemTime,
interval: Duration,
lookback: Duration,
) -> common_recordbatch::RecordBatches {
let output = promql_query(
ins, promql, alias, query_ctx, start, end, interval, lookback,
)
.await
.unwrap();
match output.data {
OutputData::Stream(stream) => collect_batches(stream).await.unwrap(),
OutputData::RecordBatches(recordbatches) => recordbatches,
_ => unreachable!(),
}
}
const ANON_PROMQL_RATIO_REPRO_DB: &str = "repro_db";
const ANON_PROMQL_RATIO_REPRO_CREATE: &str = r#"
CREATE TABLE phy (
t TIMESTAMP TIME INDEX,
v DOUBLE
) ENGINE=metric WITH ("physical_metric_table" = "");
CREATE TABLE metric_a (
l1 STRING NULL,
l2 STRING NULL,
l3 STRING NULL,
l4 STRING NULL,
l5 STRING NULL,
t TIMESTAMP NOT NULL,
v DOUBLE NULL,
TIME INDEX (t),
PRIMARY KEY (l1, l2, l3, l4, l5)
) ENGINE=metric WITH (on_physical_table = 'phy');
CREATE TABLE metric_b (
l6 STRING NULL,
l1 STRING NULL,
l2 STRING NULL,
l3 STRING NULL,
l4 STRING NULL,
t TIMESTAMP NOT NULL,
v DOUBLE NULL,
TIME INDEX (t),
PRIMARY KEY (l6, l1, l2, l3, l4)
) ENGINE=metric WITH (on_physical_table = 'phy');
"#;
const ANON_PROMQL_RATIO_REPRO_INSERT: &str = r#"
INSERT INTO metric_a (l1, l2, l3, l4, l5, t, v) VALUES
('v1', 'v2', 'v3', 'v4a', 'v5a', 1, 0),
('v1', 'v2', 'v3', 'v4a', 'v5a', 180000, 120),
('v1', 'v2', 'v3', 'v4a', 'v5a', 360000, 240),
('v1', 'v2', 'v3', 'v4a', 'v5b', 1, 0),
('v1', 'v2', 'v3', 'v4a', 'v5b', 180000, 30),
('v1', 'v2', 'v3', 'v4a', 'v5b', 360000, 60),
('v1', 'v2', 'v3-b', 'v4b', 'v5c', 1, 0),
('v1', 'v2', 'v3-b', 'v4b', 'v5c', 180000, 60),
('v1', 'v2', 'v3-b', 'v4b', 'v5c', 360000, 120);
INSERT INTO metric_b (l6, l1, l2, l3, l4, t, v) VALUES
('v6', 'v1', 'v2', 'v3', 'v4a', 1, 1),
('v6', 'v1', 'v2', 'v3', 'v4a', 180000, 1),
('v6', 'v1', 'v2', 'v3', 'v4a', 360000, 1),
('v6', 'v1', 'v2', 'v3-b', 'v4b', 1, 2),
('v6', 'v1', 'v2', 'v3-b', 'v4b', 180000, 2),
('v6', 'v1', 'v2', 'v3-b', 'v4b', 360000, 2);
"#;
const ANON_PROMQL_RATIO_REPRO_NUMERATOR: &str = r#"count(((rate(metric_a{l1="v1",l2="v2",l3=~"v3(|-a|-b)",__schema__="repro_db"}[3m]) / on(l3,l4) group_left metric_b{l6="v6",l1="v1",l2="v2",l3=~"v3(|-a|-b)",__schema__="repro_db"}) > 0.50))"#;
const ANON_PROMQL_RATIO_REPRO_DENOMINATOR: &str =
r#"count(rate(metric_a{l1="v1",l2="v2",l3=~"v3(|-a|-b)",__schema__="repro_db"}[3m]))"#;
const ANON_PROMQL_RATIO_REPRO_WHOLE: &str = r#"(count(((rate(metric_a{l1="v1",l2="v2",l3=~"v3(|-a|-b)",__schema__="repro_db"}[3m]) / on(l3,l4) group_left metric_b{l6="v6",l1="v1",l2="v2",l3=~"v3(|-a|-b)",__schema__="repro_db"}) > 0.50)) / count(rate(metric_a{l1="v1",l2="v2",l3=~"v3(|-a|-b)",__schema__="repro_db"}[3m]))) * 100"#;
const ANON_PROMQL_RATIO_REPRO_SCALAR_DIV: &str =
r#"count(rate(metric_a{l1="v1",l2="v2",l3=~"v3(|-a|-b)",__schema__="repro_db"}[3m])) / 2"#;
#[apply(both_instances_cases)]
async fn sql_insert_tql_query_ceil(instance: Arc<dyn MockInstance>) {
let instance = instance.frontend();
create_insert_tql_assert(
instance,
r#"create table http_requests_total (
host string,
cpu double,
memory double,
ts timestamp TIME INDEX,
PRIMARY KEY (host),
);"#,
r#"insert into http_requests_total(host, cpu, memory, ts) values
('host1', 66.6, 1024, 0),
('host1', 66.6, 2048, 2000),
('host1', 66.6, 4096, 5000),
('host1', 43.1, 8192, 7000),
('host1', 19.1, 10240, 9000),
('host1', 99.1, 20480, 10000),
('host1', 999.9, 40960, 21000),
('host1', 31.9, 8192, 22000),
('host1', 95.4, 333.3, 32000),
('host1', 12423.1, 1333.3, 49000),
('host1', 0, 2333.3, 80000),
('host1', 49, 3333.3, 99000);
"#,
"TQL EVAL (0,100,10) ceil(http_requests_total{host=\"host1\"})",
"+---------------------+-----------+--------------+-------+\
\n| ts | ceil(cpu) | ceil(memory) | host |\
\n+---------------------+-----------+--------------+-------+\
\n| 1970-01-01T00:00:00 | 67.0 | 1024.0 | host1 |\
\n| 1970-01-01T00:00:10 | 100.0 | 20480.0 | host1 |\
\n| 1970-01-01T00:00:20 | 100.0 | 20480.0 | host1 |\
\n| 1970-01-01T00:00:30 | 32.0 | 8192.0 | host1 |\
\n| 1970-01-01T00:00:40 | 96.0 | 334.0 | host1 |\
\n| 1970-01-01T00:00:50 | 12424.0 | 1334.0 | host1 |\
\n| 1970-01-01T00:01:00 | 12424.0 | 1334.0 | host1 |\
\n| 1970-01-01T00:01:10 | 12424.0 | 1334.0 | host1 |\
\n| 1970-01-01T00:01:20 | 0.0 | 2334.0 | host1 |\
\n| 1970-01-01T00:01:30 | 0.0 | 2334.0 | host1 |\
\n| 1970-01-01T00:01:40 | 49.0 | 3334.0 | host1 |\
\n+---------------------+-----------+--------------+-------+",
)
.await;
}
#[apply(both_instances_cases)]
async fn sql_insert_promql_query_ceil(instance: Arc<dyn MockInstance>) {
let instance = instance.frontend();
create_insert_query_assert(
instance,
r#"create table http_requests_total (
host string,
cpu double,
memory double,
ts timestamp TIME INDEX,
PRIMARY KEY (host),
);"#,
r#"insert into http_requests_total(host, cpu, memory, ts) values
('host1', 66.6, 1024, 0),
('host1', 66.6, 2048, 2000),
('host1', 66.6, 4096, 5000),
('host1', 43.1, 8192, 7000),
('host1', 19.1, 10240, 9000),
('host1', 99.1, 20480, 10000),
('host1', 999.9, 40960, 21000),
('host1', 31.9, 8192, 22000),
('host1', 95.4, 333.3, 32000),
('host1', 12423.1, 1333.3, 49000),
('host1', 0, 2333.3, 80000),
('host1', 49, 3333.3, 99000);
"#,
"ceil(http_requests_total{host=\"host1\"})",
UNIX_EPOCH,
UNIX_EPOCH.checked_add(Duration::from_secs(100)).unwrap(),
Duration::from_secs(5),
Duration::from_secs(1),
"+---------------------+-----------+--------------+-------+\
\n| ts | ceil(cpu) | ceil(memory) | host |\
\n+---------------------+-----------+--------------+-------+\
\n| 1970-01-01T00:00:00 | 67.0 | 1024.0 | host1 |\
\n| 1970-01-01T00:00:05 | 67.0 | 4096.0 | host1 |\
\n| 1970-01-01T00:00:10 | 100.0 | 20480.0 | host1 |\
\n| 1970-01-01T00:01:20 | 0.0 | 2334.0 | host1 |\
\n+---------------------+-----------+--------------+-------+",
)
.await;
}
const AGGREGATORS_CREATE_TABLE: &str = r#"create table http_requests (
job string,
instance string,
"group" string,
"value" double,
ts timestamp TIME INDEX,
PRIMARY KEY (job, instance, "group"),
);"#;
// load 5m
// http_requests{job="api-server", instance="0", group="production"} 0+10x10
// http_requests{job="api-server", instance="1", group="production"} 0+20x10
// http_requests{job="api-server", instance="0", group="canary"} 0+30x10
// http_requests{job="api-server", instance="1", group="canary"} 0+40x10
// http_requests{job="app-server", instance="0", group="production"} 0+50x10
// http_requests{job="app-server", instance="1", group="production"} 0+60x10
// http_requests{job="app-server", instance="0", group="canary"} 0+70x10
// http_requests{job="app-server", instance="1", group="canary"} 0+80x10
const AGGREGATORS_INSERT_DATA: &str = r#"insert into http_requests(job, instance, "group", value, ts) values
('api-server', '0', 'production', 100, 0),
('api-server', '1', 'production', 200, 0),
('api-server', '0', 'canary', 300, 0),
('api-server', '1', 'canary', 400, 0),
('app-server', '0', 'production', 500, 0),
('app-server', '1', 'production', 600, 0),
('app-server', '0', 'canary', 700, 0),
('app-server', '1', 'canary', 800, 0);"#;
fn unix_epoch_plus_100s() -> SystemTime {
UNIX_EPOCH.checked_add(Duration::from_secs(100)).unwrap()
}
// # Simple sum.
// eval instant at 50m SUM BY (group) (http_requests{job="api-server"})
// {group="canary"} 700
// {group="production"} 300
#[apply(both_instances_cases)]
async fn aggregators_simple_sum(instance: Arc<dyn MockInstance>) {
let instance = instance.frontend();
create_insert_query_assert(
instance,
AGGREGATORS_CREATE_TABLE,
AGGREGATORS_INSERT_DATA,
"SUM BY (group) (http_requests{job=\"api-server\"})",
UNIX_EPOCH,
unix_epoch_plus_100s(),
Duration::from_secs(60),
Duration::from_secs(0),
"+------------+---------------------+--------------------------+\
\n| group | ts | sum(http_requests.value) |\
\n+------------+---------------------+--------------------------+\
\n| production | 1970-01-01T00:00:00 | 300.0 |\
\n| canary | 1970-01-01T00:00:00 | 700.0 |\
\n+------------+---------------------+--------------------------+",
)
.await;
}
// # Simple average.
// eval instant at 50m avg by (group) (http_requests{job="api-server"})
// {group="canary"} 350
// {group="production"} 150
#[apply(both_instances_cases)]
async fn aggregators_simple_avg(instance: Arc<dyn MockInstance>) {
let instance = instance.frontend();
create_insert_query_assert(
instance,
AGGREGATORS_CREATE_TABLE,
AGGREGATORS_INSERT_DATA,
"AVG BY (group) (http_requests{job=\"api-server\"})",
UNIX_EPOCH,
unix_epoch_plus_100s(),
Duration::from_secs(60),
Duration::from_secs(0),
"+------------+---------------------+--------------------------+\
\n| group | ts | avg(http_requests.value) |\
\n+------------+---------------------+--------------------------+\
\n| production | 1970-01-01T00:00:00 | 150.0 |\
\n| canary | 1970-01-01T00:00:00 | 350.0 |\
\n+------------+---------------------+--------------------------+",
)
.await;
}
// # Simple count.
// eval instant at 50m count by (group) (http_requests{job="api-server"})
// {group="canary"} 2
// {group="production"} 2
#[apply(both_instances_cases)]
async fn aggregators_simple_count(instance: Arc<dyn MockInstance>) {
let instance = instance.frontend();
create_insert_query_assert(
instance,
AGGREGATORS_CREATE_TABLE,
AGGREGATORS_INSERT_DATA,
"COUNT BY (group) (http_requests{job=\"api-server\"})",
UNIX_EPOCH,
unix_epoch_plus_100s(),
Duration::from_secs(60),
Duration::from_secs(0),
"+------------+---------------------+----------------------------+\
\n| group | ts | count(http_requests.value) |\
\n+------------+---------------------+----------------------------+\
\n| canary | 1970-01-01T00:00:00 | 2 |\
\n| production | 1970-01-01T00:00:00 | 2 |\
\n+------------+---------------------+----------------------------+",
)
.await;
}
// Test like `aggregators_simple_count` but with value aliasing.
#[apply(both_instances_cases)]
async fn value_alias(instance: Arc<dyn MockInstance>) {
let instance = instance.frontend();
create_insert_query_assert_with_alias(
instance,
AGGREGATORS_CREATE_TABLE,
AGGREGATORS_INSERT_DATA,
"COUNT BY (group) (http_requests{job=\"api-server\"})",
Some("my_series".to_string()),
UNIX_EPOCH,
unix_epoch_plus_100s(),
Duration::from_secs(60),
Duration::from_secs(0),
"+-----------+------------+---------------------+\
\n| my_series | group | ts |\
\n+-----------+------------+---------------------+\
\n| 2 | canary | 1970-01-01T00:00:00 |\
\n| 2 | production | 1970-01-01T00:00:00 |\
\n+-----------+------------+---------------------+",
)
.await;
}
// # Simple without.
// eval instant at 50m sum without (instance) (http_requests{job="api-server"})
// {group="canary",job="api-server"} 700
// {group="production",job="api-server"} 300
#[apply(both_instances_cases)]
async fn aggregators_simple_without(instance: Arc<dyn MockInstance>) {
let instance = instance.frontend();
create_insert_query_assert(
instance,
AGGREGATORS_CREATE_TABLE,
AGGREGATORS_INSERT_DATA,
"sum without (instance) (http_requests{job=\"api-server\"})",
UNIX_EPOCH,
unix_epoch_plus_100s(),
Duration::from_secs(60),
Duration::from_secs(0),
"+------------+------------+---------------------+--------------------------+\
\n| group | job | ts | sum(http_requests.value) |\
\n+------------+------------+---------------------+--------------------------+\
\n| production | api-server | 1970-01-01T00:00:00 | 300.0 |\
\n| canary | api-server | 1970-01-01T00:00:00 | 700.0 |\
\n+------------+------------+---------------------+--------------------------+",
)
.await;
}
// # Empty by.
// eval instant at 50m sum by () (http_requests{job="api-server"})
// {} 1000
#[apply(both_instances_cases)]
async fn aggregators_empty_by(instance: Arc<dyn MockInstance>) {
let instance = instance.frontend();
create_insert_query_assert(
instance,
AGGREGATORS_CREATE_TABLE,
AGGREGATORS_INSERT_DATA,
"sum by () (http_requests{job=\"api-server\"})",
UNIX_EPOCH,
unix_epoch_plus_100s(),
Duration::from_secs(60),
Duration::from_secs(0),
"+---------------------+--------------------------+\
\n| ts | sum(http_requests.value) |\
\n+---------------------+--------------------------+\
\n| 1970-01-01T00:00:00 | 1000.0 |\
\n+---------------------+--------------------------+",
)
.await;
}
// # No by/without.
// eval instant at 50m sum(http_requests{job="api-server"})
// {} 1000
#[apply(both_instances_cases)]
async fn aggregators_no_by_without(instance: Arc<dyn MockInstance>) {
let instance = instance.frontend();
create_insert_query_assert(
instance,
AGGREGATORS_CREATE_TABLE,
AGGREGATORS_INSERT_DATA,
r#"sum (http_requests{job="api-server"})"#,
UNIX_EPOCH,
unix_epoch_plus_100s(),
Duration::from_secs(60),
Duration::from_secs(0),
"+---------------------+--------------------------+\
\n| ts | sum(http_requests.value) |\
\n+---------------------+--------------------------+\
\n| 1970-01-01T00:00:00 | 1000.0 |\
\n+---------------------+--------------------------+",
)
.await;
}
// # Empty without.
// eval instant at 50m sum without () (http_requests{job="api-server",group="production"})
// {group="production",job="api-server",instance="0"} 100
// {group="production",job="api-server",instance="1"} 200
#[apply(both_instances_cases)]
async fn aggregators_empty_without(instance: Arc<dyn MockInstance>) {
let instance = instance.frontend();
create_insert_query_assert(
instance,
AGGREGATORS_CREATE_TABLE,
AGGREGATORS_INSERT_DATA,
r#"sum without () (http_requests{job="api-server",group="production"})"#,
UNIX_EPOCH,
unix_epoch_plus_100s(),
Duration::from_secs(60),
Duration::from_secs(0),
"+------------+----------+------------+---------------------+--------------------------+\
\n| group | instance | job | ts | sum(http_requests.value) |\
\n+------------+----------+------------+---------------------+--------------------------+\
\n| production | 0 | api-server | 1970-01-01T00:00:00 | 100.0 |\
\n| production | 1 | api-server | 1970-01-01T00:00:00 | 200.0 |\
\n+------------+----------+------------+---------------------+--------------------------+",
)
.await;
}
// # Lower-cased aggregation operators should work too.
// eval instant at 50m sum(http_requests) by (job) + min(http_requests) by (job) + max(http_requests) by (job) + avg(http_requests) by (job)
// {job="app-server"} 4550
// {job="api-server"} 1750
#[apply(both_instances_cases)]
async fn aggregators_complex_combined_aggrs(instance: Arc<dyn MockInstance>) {
let instance = instance.frontend();
create_insert_query_assert(
instance,
AGGREGATORS_CREATE_TABLE,
AGGREGATORS_INSERT_DATA,
"sum(http_requests) by (job) + min(http_requests) by (job) + max(http_requests) by (job) + avg(http_requests) by (job)",
UNIX_EPOCH,
unix_epoch_plus_100s(),
Duration::from_secs(60),
Duration::from_secs(0),
"+------------+---------------------+---------------------------------------------------------------------------------------------------------------------------------------------+\
\n| job | ts | lhs.rhs.lhs.sum(http_requests.value) + rhs.min(http_requests.value) + http_requests.max(http_requests.value) + rhs.avg(http_requests.value) |\
\n+------------+---------------------+---------------------------------------------------------------------------------------------------------------------------------------------+\
\n| api-server | 1970-01-01T00:00:00 | 1750.0 |\
\n| app-server | 1970-01-01T00:00:00 | 4550.0 |\
\n+------------+---------------------+---------------------------------------------------------------------------------------------------------------------------------------------+",
)
.await;
}
// This is not from prometheus test set. It's derived from `aggregators_complex_combined_aggrs()`
#[apply(both_instances_cases)]
async fn two_aggregators_combined_aggrs(instance: Arc<dyn MockInstance>) {
let instance = instance.frontend();
create_insert_query_assert(
instance,
AGGREGATORS_CREATE_TABLE,
AGGREGATORS_INSERT_DATA,
"sum(http_requests) by (job) + min(http_requests) by (job) ",
UNIX_EPOCH,
unix_epoch_plus_100s(),
Duration::from_secs(60),
Duration::from_secs(0),
"+------------+---------------------+-------------------------------------------------------------+\
\n| job | ts | lhs.sum(http_requests.value) + rhs.min(http_requests.value) |\
\n+------------+---------------------+-------------------------------------------------------------+\
\n| api-server | 1970-01-01T00:00:00 | 1100.0 |\
\n| app-server | 1970-01-01T00:00:00 | 3100.0 |\
\n+------------+---------------------+-------------------------------------------------------------+",
)
.await;
}
// eval instant at 50m stddev by (instance)(http_requests)
// {instance="0"} 223.60679774998
// {instance="1"} 223.60679774998
//
// The values are compared with a tolerance rather than against a rendered table:
// `stddev_pop` merges partial aggregates in a parallelism-dependent order, so the
// last digits of its output are not stable across runs.
#[apply(both_instances_cases)]
async fn stddev_by_label(instance: Arc<dyn MockInstance>) {
let instance = instance.frontend();
execute_all(&instance, AGGREGATORS_CREATE_TABLE, QueryContext::arc()).await;
execute_all(&instance, AGGREGATORS_INSERT_DATA, QueryContext::arc()).await;
let batches = promql_query_as_batches(
instance,
r#"stddev by (instance)(http_requests)"#,
None,
QueryContext::arc(),
UNIX_EPOCH,
unix_epoch_plus_100s(),
Duration::from_secs(60),
Duration::from_secs(0),
)
.await;
let batches = batches.iter().collect::<Vec<_>>();
assert_eq!(batches.len(), 1);
// Tag columns come back dictionary-encoded.
let instances = cast(
batches[0].column_by_name("instance").unwrap(),
&DataType::Utf8,
)
.unwrap();
let instances = instances.as_any().downcast_ref::<StringArray>().unwrap();
let values = batches[0]
.column_by_name("stddev_pop(http_requests.value)")
.unwrap()
.as_any()
.downcast_ref::<Float64Array>()
.unwrap();
let mut actual = (0..values.len())
.map(|i| (instances.value(i), values.value(i)))
.collect::<Vec<_>>();
actual.sort_by(|a, b| a.0.cmp(b.0));
assert_eq!(actual.len(), 2, "{}", batches[0].pretty_print());
for (expected_instance, (instance, value)) in ["0", "1"].into_iter().zip(actual) {
assert_eq!(instance, expected_instance);
assert!(
(value - 223.606_797_749_979).abs() < 1e-9,
"instance {instance}: {value}"
);
}
}
// This is not derived from prometheus
#[apply(both_instances_cases)]
async fn binary_op_plain_columns(instance: Arc<dyn MockInstance>) {
let instance = instance.frontend();
create_insert_query_assert(
instance,
AGGREGATORS_CREATE_TABLE,
AGGREGATORS_INSERT_DATA,
r#"http_requests - http_requests"#,
UNIX_EPOCH,
unix_epoch_plus_100s(),
Duration::from_secs(60),
Duration::from_secs(0),
"+------------+----------+------------+---------------------+-----------------------+\
\n| job | instance | group | ts | lhs.value - rhs.value |\
\n+------------+----------+------------+---------------------+-----------------------+\
\n| api-server | 0 | canary | 1970-01-01T00:00:00 | 0.0 |\
\n| api-server | 0 | production | 1970-01-01T00:00:00 | 0.0 |\
\n| api-server | 1 | canary | 1970-01-01T00:00:00 | 0.0 |\
\n| api-server | 1 | production | 1970-01-01T00:00:00 | 0.0 |\
\n| app-server | 0 | canary | 1970-01-01T00:00:00 | 0.0 |\
\n| app-server | 0 | production | 1970-01-01T00:00:00 | 0.0 |\
\n| app-server | 1 | canary | 1970-01-01T00:00:00 | 0.0 |\
\n| app-server | 1 | production | 1970-01-01T00:00:00 | 0.0 |\
\n+------------+----------+------------+---------------------+-----------------------+",
)
.await;
}
#[apply(both_instances_cases)]
async fn cross_schema_query(instance: Arc<dyn MockInstance>) {
let ins = instance.frontend();
ins.do_query(
AGGREGATORS_CREATE_TABLE,
QueryContext::with_db_name(Some("greptime_private")).into(),
)
.await
.into_iter()
.for_each(|v| {
let _ = v.unwrap();
});
ins.do_query(
AGGREGATORS_INSERT_DATA,
QueryContext::with_db_name(Some("greptime_private")).into(),
)
.await
.into_iter()
.for_each(|v| {
let _ = v.unwrap();
});
let start = UNIX_EPOCH;
let end = unix_epoch_plus_100s();
let interval = Duration::from_secs(60);
let lookback_delta = Duration::from_secs(0);
let query_output = promql_query(
ins.clone(),
r#"http_requests{__schema__="greptime_private"}"#,
None,
QueryContext::arc(),
start,
end,
interval,
lookback_delta,
)
.await
.unwrap();
let expected = r#"+------------+----------+------------+-------+---------------------+
| job | instance | group | value | ts |
+------------+----------+------------+-------+---------------------+
| api-server | 0 | production | 100.0 | 1970-01-01T00:00:00 |
| api-server | 0 | canary | 300.0 | 1970-01-01T00:00:00 |
| api-server | 1 | production | 200.0 | 1970-01-01T00:00:00 |
| api-server | 1 | canary | 400.0 | 1970-01-01T00:00:00 |
| app-server | 0 | canary | 700.0 | 1970-01-01T00:00:00 |
| app-server | 0 | production | 500.0 | 1970-01-01T00:00:00 |
| app-server | 1 | canary | 800.0 | 1970-01-01T00:00:00 |
| app-server | 1 | production | 600.0 | 1970-01-01T00:00:00 |
+------------+----------+------------+-------+---------------------+"#;
check_unordered_output_stream(query_output, expected).await;
let query_output = promql_query(
ins.clone(),
r#"http_requests{__database__="greptime_private"}"#,
None,
QueryContext::arc(),
start,
end,
interval,
lookback_delta,
)
.await
.unwrap();
let expected = r#"+------------+----------+------------+-------+---------------------+
| job | instance | group | value | ts |
+------------+----------+------------+-------+---------------------+
| api-server | 0 | production | 100.0 | 1970-01-01T00:00:00 |
| api-server | 0 | canary | 300.0 | 1970-01-01T00:00:00 |
| api-server | 1 | production | 200.0 | 1970-01-01T00:00:00 |
| api-server | 1 | canary | 400.0 | 1970-01-01T00:00:00 |
| app-server | 0 | canary | 700.0 | 1970-01-01T00:00:00 |
| app-server | 0 | production | 500.0 | 1970-01-01T00:00:00 |
| app-server | 1 | canary | 800.0 | 1970-01-01T00:00:00 |
| app-server | 1 | production | 600.0 | 1970-01-01T00:00:00 |
+------------+----------+------------+-------+---------------------+"#;
check_unordered_output_stream(query_output, expected).await;
let query_output = promql_query(
ins.clone(),
r#"http_requests"#,
None,
QueryContext::arc(),
start,
end,
interval,
lookback_delta,
)
.await
.unwrap();
let empty_result = r#"+------+-------+
| time | value |
+------+-------+
+------+-------+"#;
check_unordered_output_stream(query_output, empty_result).await;
let query_output = promql_query(
ins.clone(),
r#"http_requests"#,
None,
QueryContext::with_db_name(Some("greptime_private")).into(),
start,
end,
interval,
lookback_delta,
)
.await
.unwrap();
check_unordered_output_stream(query_output, expected).await;
}
#[apply(both_instances_cases)]
async fn anon_promql_ratio_repro(instance: Arc<dyn MockInstance>) {
let ins = instance.frontend();
execute_all(
&ins,
&format!("CREATE DATABASE {ANON_PROMQL_RATIO_REPRO_DB}"),
QueryContext::arc(),
)
.await;
let repro_ctx: Arc<QueryContext> =
QueryContext::with_db_name(Some(ANON_PROMQL_RATIO_REPRO_DB)).into();
execute_all(&ins, ANON_PROMQL_RATIO_REPRO_CREATE, repro_ctx.clone()).await;
execute_all(&ins, ANON_PROMQL_RATIO_REPRO_INSERT, repro_ctx).await;
let start = UNIX_EPOCH.checked_add(Duration::from_secs(180)).unwrap();
let end = UNIX_EPOCH.checked_add(Duration::from_secs(360)).unwrap();
let interval = Duration::from_secs(180);
let lookback = Duration::from_secs(1);
let numerator = promql_query_as_batches(
ins.clone(),
ANON_PROMQL_RATIO_REPRO_NUMERATOR,
Some("num".to_string()),
QueryContext::arc(),
start,
end,
interval,
lookback,
)
.await;
let denominator = promql_query_as_batches(
ins.clone(),
ANON_PROMQL_RATIO_REPRO_DENOMINATOR,
Some("den".to_string()),
QueryContext::arc(),
start,
end,
interval,
lookback,
)
.await;
let whole = promql_query_as_batches(
ins.clone(),
ANON_PROMQL_RATIO_REPRO_WHOLE,
Some("pct".to_string()),
QueryContext::arc(),
start,
end,
interval,
lookback,
)
.await;
let scalar_div = promql_query_as_batches(
ins,
ANON_PROMQL_RATIO_REPRO_SCALAR_DIV,
Some("half_den".to_string()),
QueryContext::arc(),
start,
end,
interval,
lookback,
)
.await;
let numerator = numerator.iter().collect::<Vec<_>>();
let denominator = denominator.iter().collect::<Vec<_>>();
let whole = whole.iter().collect::<Vec<_>>();
let scalar_div = scalar_div.iter().collect::<Vec<_>>();
let numerator_values = numerator[0]
.column_by_name("num")
.unwrap()
.as_any()
.downcast_ref::<Int64Array>()
.unwrap();
let denominator_values = denominator[0]
.column_by_name("den")
.unwrap()
.as_any()
.downcast_ref::<Int64Array>()
.unwrap();
let percentage_values = whole[0]
.column_by_name("pct")
.unwrap()
.as_any()
.downcast_ref::<Float64Array>()
.unwrap();
let scalar_div_values = scalar_div[0]
.column_by_name("half_den")
.unwrap()
.as_any()
.downcast_ref::<Float64Array>()
.unwrap();
assert_eq!(numerator_values.len(), 1, "{}", numerator[0].pretty_print());
assert_eq!(
denominator_values.len(),
1,
"{}",
denominator[0].pretty_print()
);
assert_eq!(percentage_values.len(), 1, "{}", whole[0].pretty_print());
assert_eq!(
scalar_div_values.len(),
1,
"{}",
scalar_div[0].pretty_print()
);
assert_eq!(
numerator_values.value(0),
1,
"{}",
numerator[0].pretty_print()
);
assert_eq!(
denominator_values.value(0),
3,
"{}",
denominator[0].pretty_print()
);
assert!(
(scalar_div_values.value(0) - 1.5).abs() < 1e-9,
"{}",
scalar_div[0].pretty_print()
);
let expected = 100.0 / 3.0;
assert!(
(percentage_values.value(0) - expected).abs() < 1e-9,
"{}",
whole[0].pretty_print()
);
}
#[apply(both_instances_cases)]
async fn promql_stale_marker_excludes_series_across_flushes(instance: Arc<dyn MockInstance>) {
let ins = instance.frontend();
let table = "promql_stale_marker";
execute_all(
&ins,
&format!(
"CREATE TABLE {table} (\
series STRING, \
ts TIMESTAMP TIME INDEX, \
val DOUBLE, \
PRIMARY KEY (series)\
) WITH (sst_format = 'flat')"
),
QueryContext::arc(),
)
.await;
let rows = |samples: &[(&str, f64, i64)]| Rows {
schema: vec![
ColumnSchema {
column_name: "series".to_string(),
datatype: ColumnDataType::String as i32,
semantic_type: SemanticType::Tag as i32,
..Default::default()
},
ColumnSchema {
column_name: "ts".to_string(),
datatype: ColumnDataType::TimestampMillisecond as i32,
semantic_type: SemanticType::Timestamp as i32,
..Default::default()
},
ColumnSchema {
column_name: "val".to_string(),
datatype: ColumnDataType::Float64 as i32,
semantic_type: SemanticType::Field as i32,
..Default::default()
},
],
rows: samples
.iter()
.map(|(series, value, timestamp)| Row {
values: vec![
ValueData::StringValue((*series).to_string()).into(),
ValueData::TimestampMillisecondValue(*timestamp).into(),
ValueData::F64Value(*value).into(),
],
})
.collect(),
};
let insert = |samples: &[(&str, f64, i64)]| RowInsertRequests {
inserts: vec![RowInsertRequest {
table_name: table.to_string(),
rows: Some(rows(samples)),
}],
};
ins.handle_row_inserts(
insert(&[("stale", 10.0, 1_000), ("ordinary", 20.0, 1_000)]),
QueryContext::arc(),
false,
false,
)
.await
.unwrap();
execute_all(
&ins,
&format!("ADMIN FLUSH_TABLE('{table}')"),
QueryContext::arc(),
)
.await;
let ordinary_nan = f64::from_bits(0x7ff8_0000_0000_0000);
ins.handle_row_inserts(
insert(&[
("stale", f64::from_bits(PROMETHEUS_STALE_NAN_BITS), 2_000),
("ordinary", ordinary_nan, 2_000),
]),
QueryContext::arc(),
false,
false,
)
.await
.unwrap();
let extract_samples = |batches: common_recordbatch::RecordBatches| {
let mut samples = batches
.iter()
.flat_map(|batch| {
let series = cast(
batch.column_by_name("series").unwrap().as_ref(),
&DataType::Utf8,
)
.unwrap();
let series = series.as_any().downcast_ref::<StringArray>().unwrap();
let values = batch
.column_by_name("val")
.unwrap()
.as_any()
.downcast_ref::<Float64Array>()
.unwrap();
let timestamps = batch
.column_by_name("ts")
.unwrap()
.as_any()
.downcast_ref::<TimestampMillisecondArray>()
.unwrap();
(0..batch.num_rows())
.map(|row| {
(
series.value(row).to_string(),
values.value(row),
timestamps.value(row),
)
})
.collect::<Vec<_>>()
})
.collect::<Vec<_>>();
samples.sort_by(|left, right| left.0.cmp(&right.0));
samples
};
let raw_output = ins
.do_query(
&format!("SELECT series, val, ts FROM {table} WHERE ts = 2000"),
QueryContext::arc(),
)
.await
.remove(0)
.unwrap();
let raw_batches = match raw_output.data {
OutputData::Stream(stream) => collect_batches(stream).await.unwrap(),
OutputData::RecordBatches(recordbatches) => recordbatches,
_ => unreachable!(),
};
let raw_samples = extract_samples(raw_batches);
assert_eq!(raw_samples.len(), 2, "{raw_samples:?}");
let stale_marker = raw_samples
.iter()
.find(|(series, _, _)| series == "stale")
.unwrap();
assert_eq!(stale_marker.1.to_bits(), PROMETHEUS_STALE_NAN_BITS);
let ordinary_marker = raw_samples
.iter()
.find(|(series, _, _)| series == "ordinary")
.unwrap();
assert_eq!(ordinary_marker.1.to_bits(), ordinary_nan.to_bits());
// Keep the finite 1000ms sample eligible so a stale marker cannot fall back to it.
let query_at = |at| {
let time = UNIX_EPOCH.checked_add(Duration::from_millis(at)).unwrap();
promql_query_as_batches(
ins.clone(),
table,
None,
QueryContext::arc(),
time,
time,
Duration::from_secs(1),
Duration::from_secs(3),
)
};
let assert_nan_series = |samples: Vec<(String, f64, i64)>, at| {
assert_eq!(samples.len(), 1, "{samples:?}");
assert_eq!(samples[0].0, "ordinary", "{samples:?}");
assert_eq!(samples[0].2, at, "{samples:?}");
assert_eq!(
samples[0].1.to_bits(),
ordinary_nan.to_bits(),
"{samples:?}"
);
};
for flush_markers in [false, true] {
if flush_markers {
execute_all(
&ins,
&format!("ADMIN FLUSH_TABLE('{table}')"),
QueryContext::arc(),
)
.await;
}
assert_eq!(
extract_samples(query_at(1_500).await),
vec![
("ordinary".to_string(), 20.0, 1_500),
("stale".to_string(), 10.0, 1_500)
]
);
assert_nan_series(extract_samples(query_at(2_000).await), 2_000);
assert_nan_series(extract_samples(query_at(2_500).await), 2_500);
}
}
Reference in New Issue View Git Blame Copy Permalink