feat: align influxdb line timestamp with table time index (#7057)

* feat: align influxdb line timestamp with table time index

Signed-off-by: luofucong <luofc@foxmail.com>

* fix ci

Signed-off-by: luofucong <luofc@foxmail.com>

---------

Signed-off-by: luofucong <luofc@foxmail.com>
This commit is contained in:
LFC
2025-10-10 15:37:52 +08:00
committed by GitHub
parent aa84642afc
commit 3738440753
2 changed files with 208 additions and 11 deletions

View File

@@ -17,7 +17,9 @@ mod test {
use std::sync::Arc;
use client::OutputData;
use common_grpc::precision::Precision;
use common_recordbatch::RecordBatches;
use common_test_util::recordbatch::check_output_stream;
use rstest::rstest;
use rstest_reuse::apply;
use servers::influxdb::InfluxdbRequest;
@@ -110,19 +112,95 @@ monitor1,host=host2 cpu=32 1663840496400340001";
)
.await;
let output = output.remove(0).unwrap();
let OutputData::Stream(stream) = output.data else {
unreachable!()
};
let recordbatches = RecordBatches::try_collect(stream).await.unwrap();
assert_eq!(
recordbatches.pretty_print().unwrap(),
"\
let expected = "\
+-------------------------------+-------+------+--------+
| greptime_timestamp | host | cpu | memory |
+-------------------------------+-------+------+--------+
| 2022-09-22T09:54:56.100023100 | host1 | 66.6 | 1024.0 |
| 2022-09-22T09:54:56.400340001 | host2 | 32.0 | 1027.0 |
+-------------------------------+-------+------+--------+"
);
+-------------------------------+-------+------+--------+";
check_output_stream(output.data, expected).await;
}
#[apply(both_instances_cases)]
async fn test_put_influxdb_lines_with_auto_aligning_timestamps(
instance: Arc<dyn MockInstance>,
) {
let instance = instance.frontend();
// First create a table with millisecond time index.
let sql = "create table monitor (
ts timestamp time index,
host string primary key,
cpu double,
memory double,
)";
instance.do_query(sql, QueryContext::arc()).await;
// Insert some influxdb lines with millisecond precision.
let lines = r"
monitor,host=127.0.0.1 cpu=0.1,memory=1.0 1719460800001
monitor,host=127.0.0.2 cpu=0.2,memory=2.0 1719460800002";
let request = InfluxdbRequest {
precision: Some(Precision::Millisecond),
lines: lines.to_string(),
};
instance.exec(request, QueryContext::arc()).await.unwrap();
// Insert some influxdb lines without precision.
// According to the specification (both v1 and v2), if precision is not set, it is default
// to "nanosecond". The lines here will be converted to insert requests as usual and then
// be aligned to millisecond time unit.
let lines = r"
monitor,host=127.0.0.1 cpu=0.3,memory=3.0 1719460800003000000
monitor,host=127.0.0.2 cpu=0.4,memory=4.0 1719460800004000000";
let request = InfluxdbRequest {
precision: None,
lines: lines.to_string(),
};
instance.exec(request, QueryContext::arc()).await.unwrap();
// Insert other influxdb lines with nanosecond precision.
let lines = r"
monitor,host=127.0.0.1 cpu=0.5,memory=5.0 1719460800005000000
monitor,host=127.0.0.2 cpu=0.6,memory=6.0 1719460800006000000";
let request = InfluxdbRequest {
precision: Some(Precision::Nanosecond),
lines: lines.to_string(),
};
instance.exec(request, QueryContext::arc()).await.unwrap();
// Insert other influxdb lines with second precision.
let lines = r"
monitor,host=127.0.0.1 cpu=0.7,memory=7.0 1719460801
monitor,host=127.0.0.2 cpu=0.8,memory=8.0 1719460802";
let request = InfluxdbRequest {
precision: Some(Precision::Second),
lines: lines.to_string(),
};
instance.exec(request, QueryContext::arc()).await.unwrap();
// Check the data.
let mut output = instance
.do_query(
"SELECT ts, host, cpu, memory FROM monitor ORDER BY ts",
QueryContext::arc(),
)
.await;
let output = output.remove(0).unwrap();
let expected = "\
+-------------------------+-----------+-----+--------+
| ts | host | cpu | memory |
+-------------------------+-----------+-----+--------+
| 2024-06-27T04:00:00.001 | 127.0.0.1 | 0.1 | 1.0 |
| 2024-06-27T04:00:00.002 | 127.0.0.2 | 0.2 | 2.0 |
| 2024-06-27T04:00:00.003 | 127.0.0.1 | 0.3 | 3.0 |
| 2024-06-27T04:00:00.004 | 127.0.0.2 | 0.4 | 4.0 |
| 2024-06-27T04:00:00.005 | 127.0.0.1 | 0.5 | 5.0 |
| 2024-06-27T04:00:00.006 | 127.0.0.2 | 0.6 | 6.0 |
| 2024-06-27T04:00:01 | 127.0.0.1 | 0.7 | 7.0 |
| 2024-06-27T04:00:02 | 127.0.0.2 | 0.8 | 8.0 |
+-------------------------+-----------+-----+--------+";
check_output_stream(output.data, expected).await;
}
}