From aa953dcc34283ee0db3fd7edbe63609fad3e4142 Mon Sep 17 00:00:00 2001 From: shuiyisong <113876041+shuiyisong@users.noreply.github.com> Date: Mon, 11 Mar 2024 17:07:10 +0800 Subject: [PATCH 01/17] fix: impl `RecordBatchStream` method explicitly (#3482) fix: impl RecordBatchStream method explicitly --- src/common/recordbatch/src/adapter.rs | 22 +++++++++++++++++++++- src/common/recordbatch/src/lib.rs | 24 ++++++++++++++++++------ src/common/recordbatch/src/util.rs | 11 ++++++++++- src/file-engine/src/query.rs | 11 ++++++++++- src/query/src/metrics.rs | 11 ++++++++++- src/script/src/python/engine.rs | 11 ++++++++++- src/table/src/table/numbers.rs | 11 ++++++++++- src/table/src/table/scan.rs | 6 +++++- src/table/src/test_util/memtable.rs | 11 ++++++++++- 9 files changed, 104 insertions(+), 14 deletions(-) diff --git a/src/common/recordbatch/src/adapter.rs b/src/common/recordbatch/src/adapter.rs index d26296d449..12f5ecfdc9 100644 --- a/src/common/recordbatch/src/adapter.rs +++ b/src/common/recordbatch/src/adapter.rs @@ -32,7 +32,7 @@ use snafu::ResultExt; use crate::error::{self, Result}; use crate::{ - DfRecordBatch, DfSendableRecordBatchStream, RecordBatch, RecordBatchStream, + DfRecordBatch, DfSendableRecordBatchStream, OrderOption, RecordBatch, RecordBatchStream, SendableRecordBatchStream, Stream, }; @@ -228,6 +228,10 @@ impl RecordBatchStream for RecordBatchStreamAdapter { Metrics::Unavailable | Metrics::Unresolved(_) => None, } } + + fn output_ordering(&self) -> Option<&[OrderOption]> { + None + } } impl Stream for RecordBatchStreamAdapter { @@ -316,6 +320,14 @@ impl RecordBatchStream for AsyncRecordBatchStreamAdapter { fn schema(&self) -> SchemaRef { self.schema.clone() } + + fn output_ordering(&self) -> Option<&[OrderOption]> { + None + } + + fn metrics(&self) -> Option { + None + } } impl Stream for AsyncRecordBatchStreamAdapter { @@ -375,6 +387,14 @@ mod test { fn schema(&self) -> SchemaRef { unimplemented!() } + + fn output_ordering(&self) -> Option<&[OrderOption]> { + None + } + + fn metrics(&self) -> Option { + None + } } impl Stream for MaybeErrorRecordBatchStream { diff --git a/src/common/recordbatch/src/lib.rs b/src/common/recordbatch/src/lib.rs index 93ba03a333..f062a6474f 100644 --- a/src/common/recordbatch/src/lib.rs +++ b/src/common/recordbatch/src/lib.rs @@ -39,13 +39,9 @@ use snafu::{ensure, ResultExt}; pub trait RecordBatchStream: Stream> { fn schema(&self) -> SchemaRef; - fn output_ordering(&self) -> Option<&[OrderOption]> { - None - } + fn output_ordering(&self) -> Option<&[OrderOption]>; - fn metrics(&self) -> Option { - None - } + fn metrics(&self) -> Option; } pub type SendableRecordBatchStream = Pin>; @@ -74,6 +70,14 @@ impl RecordBatchStream for EmptyRecordBatchStream { fn schema(&self) -> SchemaRef { self.schema.clone() } + + fn output_ordering(&self) -> Option<&[OrderOption]> { + None + } + + fn metrics(&self) -> Option { + None + } } impl Stream for EmptyRecordBatchStream { @@ -192,6 +196,14 @@ impl RecordBatchStream for SimpleRecordBatchStream { fn schema(&self) -> SchemaRef { self.inner.schema() } + + fn output_ordering(&self) -> Option<&[OrderOption]> { + None + } + + fn metrics(&self) -> Option { + None + } } impl Stream for SimpleRecordBatchStream { diff --git a/src/common/recordbatch/src/util.rs b/src/common/recordbatch/src/util.rs index 258e113d85..723a0f9dca 100644 --- a/src/common/recordbatch/src/util.rs +++ b/src/common/recordbatch/src/util.rs @@ -41,7 +41,8 @@ mod tests { use futures::Stream; use super::*; - use crate::RecordBatchStream; + use crate::adapter::RecordBatchMetrics; + use crate::{OrderOption, RecordBatchStream}; struct MockRecordBatchStream { batch: Option, @@ -52,6 +53,14 @@ mod tests { fn schema(&self) -> SchemaRef { self.schema.clone() } + + fn output_ordering(&self) -> Option<&[OrderOption]> { + None + } + + fn metrics(&self) -> Option { + None + } } impl Stream for MockRecordBatchStream { diff --git a/src/file-engine/src/query.rs b/src/file-engine/src/query.rs index e305376b2a..31e14a9898 100644 --- a/src/file-engine/src/query.rs +++ b/src/file-engine/src/query.rs @@ -22,8 +22,9 @@ use std::task::{Context, Poll}; use common_datasource::object_store::build_backend; use common_error::ext::BoxedError; use common_query::prelude::Expr; +use common_recordbatch::adapter::RecordBatchMetrics; use common_recordbatch::error::{CastVectorSnafu, ExternalSnafu, Result as RecordBatchResult}; -use common_recordbatch::{RecordBatch, RecordBatchStream, SendableRecordBatchStream}; +use common_recordbatch::{OrderOption, RecordBatch, RecordBatchStream, SendableRecordBatchStream}; use datafusion::logical_expr::utils as df_logical_expr_utils; use datatypes::prelude::ConcreteDataType; use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; @@ -151,6 +152,14 @@ impl RecordBatchStream for FileToScanRegionStream { fn schema(&self) -> SchemaRef { self.scan_schema.clone() } + + fn output_ordering(&self) -> Option<&[OrderOption]> { + None + } + + fn metrics(&self) -> Option { + None + } } impl Stream for FileToScanRegionStream { diff --git a/src/query/src/metrics.rs b/src/query/src/metrics.rs index eb5f9c3ede..ac9397eec3 100644 --- a/src/query/src/metrics.rs +++ b/src/query/src/metrics.rs @@ -15,7 +15,8 @@ use std::pin::Pin; use std::task::{Context, Poll}; -use common_recordbatch::{RecordBatch, RecordBatchStream, SendableRecordBatchStream}; +use common_recordbatch::adapter::RecordBatchMetrics; +use common_recordbatch::{OrderOption, RecordBatch, RecordBatchStream, SendableRecordBatchStream}; use datatypes::schema::SchemaRef; use futures::Stream; use futures_util::ready; @@ -78,6 +79,14 @@ impl RecordBatchStream for OnDone { fn schema(&self) -> SchemaRef { self.stream.schema() } + + fn output_ordering(&self) -> Option<&[OrderOption]> { + self.stream.output_ordering() + } + + fn metrics(&self) -> Option { + self.stream.metrics() + } } impl Stream for OnDone { diff --git a/src/script/src/python/engine.rs b/src/script/src/python/engine.rs index dbf1eec6e3..31a814dedf 100644 --- a/src/script/src/python/engine.rs +++ b/src/script/src/python/engine.rs @@ -26,9 +26,10 @@ use common_function::function_registry::FUNCTION_REGISTRY; use common_query::error::{PyUdfSnafu, UdfTempRecordBatchSnafu}; use common_query::prelude::Signature; use common_query::{Output, OutputData}; +use common_recordbatch::adapter::RecordBatchMetrics; use common_recordbatch::error::{ExternalSnafu, Result as RecordBatchResult}; use common_recordbatch::{ - RecordBatch, RecordBatchStream, RecordBatches, SendableRecordBatchStream, + OrderOption, RecordBatch, RecordBatchStream, RecordBatches, SendableRecordBatchStream, }; use datafusion_expr::Volatility; use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; @@ -255,6 +256,14 @@ impl RecordBatchStream for CoprStream { // FIXME(discord9): use copr returns for schema self.ret_schema.clone() } + + fn output_ordering(&self) -> Option<&[OrderOption]> { + None + } + + fn metrics(&self) -> Option { + None + } } impl Stream for CoprStream { diff --git a/src/table/src/table/numbers.rs b/src/table/src/table/numbers.rs index d761abc50a..3ae0bf5438 100644 --- a/src/table/src/table/numbers.rs +++ b/src/table/src/table/numbers.rs @@ -17,8 +17,9 @@ use std::sync::Arc; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_error::ext::BoxedError; +use common_recordbatch::adapter::RecordBatchMetrics; use common_recordbatch::error::Result as RecordBatchResult; -use common_recordbatch::{RecordBatch, RecordBatchStream, SendableRecordBatchStream}; +use common_recordbatch::{OrderOption, RecordBatch, RecordBatchStream, SendableRecordBatchStream}; use datafusion::arrow::record_batch::RecordBatch as DfRecordBatch; use datatypes::arrow::array::UInt32Array; use datatypes::data_type::ConcreteDataType; @@ -123,6 +124,14 @@ impl RecordBatchStream for NumbersStream { fn schema(&self) -> SchemaRef { self.schema.clone() } + + fn output_ordering(&self) -> Option<&[OrderOption]> { + None + } + + fn metrics(&self) -> Option { + None + } } impl Stream for NumbersStream { diff --git a/src/table/src/table/scan.rs b/src/table/src/table/scan.rs index e1f19c5730..0d5b769136 100644 --- a/src/table/src/table/scan.rs +++ b/src/table/src/table/scan.rs @@ -23,7 +23,7 @@ use common_query::error::Result as QueryResult; use common_query::physical_plan::{Partitioning, PhysicalPlan, PhysicalPlanRef}; use common_recordbatch::adapter::RecordBatchMetrics; use common_recordbatch::error::Result as RecordBatchResult; -use common_recordbatch::{RecordBatch, RecordBatchStream, SendableRecordBatchStream}; +use common_recordbatch::{OrderOption, RecordBatch, RecordBatchStream, SendableRecordBatchStream}; use common_telemetry::tracing::Span; use common_telemetry::tracing_context::TracingContext; use datafusion::execution::context::TaskContext; @@ -157,6 +157,10 @@ impl RecordBatchStream for StreamWithMetricWrapper { fn metrics(&self) -> Option { self.stream.metrics() } + + fn output_ordering(&self) -> Option<&[OrderOption]> { + self.stream.output_ordering() + } } #[cfg(test)] diff --git a/src/table/src/test_util/memtable.rs b/src/table/src/test_util/memtable.rs index cb36bac2c7..837a6055b5 100644 --- a/src/table/src/test_util/memtable.rs +++ b/src/table/src/test_util/memtable.rs @@ -17,8 +17,9 @@ use std::sync::Arc; use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME}; use common_error::ext::BoxedError; +use common_recordbatch::adapter::RecordBatchMetrics; use common_recordbatch::error::Result as RecordBatchResult; -use common_recordbatch::{RecordBatch, RecordBatchStream, SendableRecordBatchStream}; +use common_recordbatch::{OrderOption, RecordBatch, RecordBatchStream, SendableRecordBatchStream}; use datatypes::prelude::*; use datatypes::schema::{ColumnSchema, Schema, SchemaRef}; use datatypes::vectors::UInt32Vector; @@ -165,6 +166,14 @@ impl RecordBatchStream for MemtableStream { fn schema(&self) -> SchemaRef { self.schema.clone() } + + fn output_ordering(&self) -> Option<&[OrderOption]> { + None + } + + fn metrics(&self) -> Option { + None + } } struct MemtableStream { From da098f556840424bd55972c41300b452e1cf0397 Mon Sep 17 00:00:00 2001 From: JeremyHi Date: Mon, 11 Mar 2024 17:27:51 +0800 Subject: [PATCH 02/17] fix: make max-txn-ops limit valid (#3481) --- src/common/meta/src/kv_backend/chroot.rs | 4 ++++ src/common/meta/src/kv_backend/memory.rs | 4 ++++ src/common/meta/src/kv_backend/txn.rs | 2 +- src/log-store/src/raft_engine/backend.rs | 4 ++++ src/meta-srv/src/service/store/cached_kv.rs | 4 ++++ 5 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/common/meta/src/kv_backend/chroot.rs b/src/common/meta/src/kv_backend/chroot.rs index 6f097cd83e..0cadc7b346 100644 --- a/src/common/meta/src/kv_backend/chroot.rs +++ b/src/common/meta/src/kv_backend/chroot.rs @@ -45,6 +45,10 @@ impl TxnService for ChrootKvBackend { let txn_res = self.inner.txn(txn).await?; Ok(self.chroot_txn_response(txn_res)) } + + fn max_txn_ops(&self) -> usize { + self.inner.max_txn_ops() + } } #[async_trait::async_trait] diff --git a/src/common/meta/src/kv_backend/memory.rs b/src/common/meta/src/kv_backend/memory.rs index 90a79d4456..bb8cf2d95d 100644 --- a/src/common/meta/src/kv_backend/memory.rs +++ b/src/common/meta/src/kv_backend/memory.rs @@ -323,6 +323,10 @@ impl TxnService for MemoryKvBackend { responses, }) } + + fn max_txn_ops(&self) -> usize { + usize::MAX + } } impl ResettableKvBackend for MemoryKvBackend { diff --git a/src/common/meta/src/kv_backend/txn.rs b/src/common/meta/src/kv_backend/txn.rs index f5db5a5cdb..c170b93fbe 100644 --- a/src/common/meta/src/kv_backend/txn.rs +++ b/src/common/meta/src/kv_backend/txn.rs @@ -30,7 +30,7 @@ pub trait TxnService: Sync + Send { /// Maximum number of operations permitted in a transaction. fn max_txn_ops(&self) -> usize { - usize::MAX + unimplemented!("txn is not implemented") } } diff --git a/src/log-store/src/raft_engine/backend.rs b/src/log-store/src/raft_engine/backend.rs index ab2f9896f9..86d6ae39d7 100644 --- a/src/log-store/src/raft_engine/backend.rs +++ b/src/log-store/src/raft_engine/backend.rs @@ -152,6 +152,10 @@ impl TxnService for RaftEngineBackend { responses, }) } + + fn max_txn_ops(&self) -> usize { + usize::MAX + } } #[async_trait::async_trait] diff --git a/src/meta-srv/src/service/store/cached_kv.rs b/src/meta-srv/src/service/store/cached_kv.rs index af9e0a5cec..df898164d1 100644 --- a/src/meta-srv/src/service/store/cached_kv.rs +++ b/src/meta-srv/src/service/store/cached_kv.rs @@ -380,6 +380,10 @@ impl TxnService for LeaderCachedKvBackend { Ok(res) } + + fn max_txn_ops(&self) -> usize { + self.store.max_txn_ops() + } } impl ResettableKvBackend for LeaderCachedKvBackend { From e767f372417e99f44d9fe08eef55e65c55c505f3 Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Mon, 11 Mar 2024 17:28:40 +0800 Subject: [PATCH 03/17] fix: fix f64 has no sufficient precision during parsing (#3483) --- Cargo.toml | 2 +- src/datatypes/src/value.rs | 8 +++++++ src/sql/src/parsers/create_parser.rs | 21 +++++++++++++++++ src/sql/src/statements.rs | 23 +++++++++++++++++++ .../standalone/common/create/create.result | 21 +++++++++++++++++ .../cases/standalone/common/create/create.sql | 10 ++++++++ 6 files changed, 84 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 83fd64c677..c1604c1ff9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -134,7 +134,7 @@ reqwest = { version = "0.11", default-features = false, features = [ rskafka = "0.5" rust_decimal = "1.33" serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +serde_json = { version = "1.0", features = ["float_roundtrip"] } serde_with = "3" smallvec = { version = "1", features = ["serde"] } snafu = "0.7" diff --git a/src/datatypes/src/value.rs b/src/datatypes/src/value.rs index bfd4a11103..64635eeae2 100644 --- a/src/datatypes/src/value.rs +++ b/src/datatypes/src/value.rs @@ -2417,4 +2417,12 @@ mod tests { ); check_value_ref_size_eq(&ValueRef::Decimal128(Decimal128::new(1234, 3, 1)), 32) } + + #[test] + fn test_incorrect_default_value_issue_3479() { + let value = OrderedF64::from(0.047318541668048164); + let serialized = serde_json::to_string(&value).unwrap(); + let deserialized: OrderedF64 = serde_json::from_str(&serialized).unwrap(); + assert_eq!(value, deserialized); + } } diff --git a/src/sql/src/parsers/create_parser.rs b/src/sql/src/parsers/create_parser.rs index 00a797d44b..606ee3ebef 100644 --- a/src/sql/src/parsers/create_parser.rs +++ b/src/sql/src/parsers/create_parser.rs @@ -1495,4 +1495,25 @@ ENGINE=mito"; ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); let _ = result.unwrap(); } + + #[test] + fn test_incorrect_default_value_issue_3479() { + let sql = r#"CREATE TABLE `ExcePTuRi`( +non TIMESTAMP(6) TIME INDEX, +`iUSTO` DOUBLE DEFAULT 0.047318541668048164 +)"#; + let result = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) + .unwrap(); + assert_eq!(1, result.len()); + match &result[0] { + Statement::CreateTable(c) => { + assert_eq!( + "`iUSTO` DOUBLE DEFAULT 0.047318541668048164", + c.columns[1].to_string() + ); + } + _ => unreachable!(), + } + } } diff --git a/src/sql/src/statements.rs b/src/sql/src/statements.rs index a8f38e545d..d51378c098 100644 --- a/src/sql/src/statements.rs +++ b/src/sql/src/statements.rs @@ -904,6 +904,29 @@ mod tests { ); } + #[test] + fn test_incorrect_default_value_issue_3479() { + let opts = vec![ColumnOptionDef { + name: None, + option: ColumnOption::Default(Expr::Value(SqlValue::Number( + "0.047318541668048164".into(), + false, + ))), + }]; + let constraint = parse_column_default_constraint( + "coll", + &ConcreteDataType::float64_datatype(), + &opts, + None, + ) + .unwrap() + .unwrap(); + assert_eq!("0.047318541668048164", constraint.to_string()); + let encoded: Vec = constraint.clone().try_into().unwrap(); + let decoded = ColumnDefaultConstraint::try_from(encoded.as_ref()).unwrap(); + assert_eq!(decoded, constraint); + } + #[test] pub fn test_sql_column_def_to_grpc_column_def() { // test basic diff --git a/tests/cases/standalone/common/create/create.result b/tests/cases/standalone/common/create/create.result index 664c94fede..ed0529b934 100644 --- a/tests/cases/standalone/common/create/create.result +++ b/tests/cases/standalone/common/create/create.result @@ -155,6 +155,23 @@ CREATE TABLE test_like_2 LIKE test_like_1; Error: 4000(TableAlreadyExists), Table already exists: `greptime.public.test_like_2` +CREATE TABLE `ExcePTuRi`( +non TIMESTAMP(6) TIME INDEX, +`iUSTO` DOUBLE DEFAULT 0.047318541668048164 +) +ENGINE=mito; + +Affected Rows: 0 + +DESC table `ExcePTuRi`; + ++--------+----------------------+-----+------+----------------------+---------------+ +| Column | Type | Key | Null | Default | Semantic Type | ++--------+----------------------+-----+------+----------------------+---------------+ +| non | TimestampMicrosecond | PRI | NO | | TIMESTAMP | +| iUSTO | Float64 | | YES | 0.047318541668048164 | FIELD | ++--------+----------------------+-----+------+----------------------+---------------+ + DESC TABLE test_like_1; +--------+----------------------+-----+------+---------+---------------+ @@ -183,3 +200,7 @@ DROP TABLE test_like_2; Affected Rows: 0 +DROP table `ExcePTuRi`; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/create/create.sql b/tests/cases/standalone/common/create/create.sql index f27b2bc6e8..9f541da312 100644 --- a/tests/cases/standalone/common/create/create.sql +++ b/tests/cases/standalone/common/create/create.sql @@ -64,6 +64,14 @@ CREATE TABLE test_like_2 LIKE test_like_1; CREATE TABLE test_like_2 LIKE test_like_1; +CREATE TABLE `ExcePTuRi`( +non TIMESTAMP(6) TIME INDEX, +`iUSTO` DOUBLE DEFAULT 0.047318541668048164 +) +ENGINE=mito; + +DESC table `ExcePTuRi`; + DESC TABLE test_like_1; DESC TABLE test_like_2; @@ -71,3 +79,5 @@ DESC TABLE test_like_2; DROP TABLE test_like_1; DROP TABLE test_like_2; + +DROP table `ExcePTuRi`; From b7ac8d6aa84cd5deb3fd2f3a4133429e34ce7b75 Mon Sep 17 00:00:00 2001 From: Ruihang Xia Date: Mon, 11 Mar 2024 18:40:19 +0800 Subject: [PATCH 04/17] ci: use another mirror for etcd image (#3486) Signed-off-by: Ruihang Xia --- tests-integration/fixtures/etcd/docker-compose-standalone.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests-integration/fixtures/etcd/docker-compose-standalone.yml b/tests-integration/fixtures/etcd/docker-compose-standalone.yml index 2f57f9b425..3c6715030a 100644 --- a/tests-integration/fixtures/etcd/docker-compose-standalone.yml +++ b/tests-integration/fixtures/etcd/docker-compose-standalone.yml @@ -1,7 +1,7 @@ version: '3.8' services: etcd: - image: public.ecr.aws/bitnami/etcd:3.5 + image: ghcr.io/zcube/bitnami-compat/etcd:3.5 ports: - "2379:2379" - "2380:2380" From 0a4444a43a7aa7132eb4a5b2efc0b0a315925a2b Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Mon, 11 Mar 2024 19:34:50 +0800 Subject: [PATCH 05/17] feat(fuzz): validate columns (#3485) --- Cargo.lock | 4 +- tests-fuzz/src/error.rs | 3 + tests-fuzz/src/generator/create_expr.rs | 9 + tests-fuzz/src/lib.rs | 1 + tests-fuzz/src/validator.rs | 15 ++ tests-fuzz/src/validator/column.rs | 240 ++++++++++++++++++++++++ tests-fuzz/targets/fuzz_create_table.rs | 15 +- 7 files changed, 283 insertions(+), 4 deletions(-) create mode 100644 tests-fuzz/src/validator.rs create mode 100644 tests-fuzz/src/validator/column.rs diff --git a/Cargo.lock b/Cargo.lock index c2e8b8b9f2..3ec7ea27d9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4815,9 +4815,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.151" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302d7ab3130588088d277783b1e2d2e10c9e9e4a16dd9050e6ec93fb3e7048f4" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libfuzzer-sys" diff --git a/tests-fuzz/src/error.rs b/tests-fuzz/src/error.rs index 9cf7728b81..def6414cb1 100644 --- a/tests-fuzz/src/error.rs +++ b/tests-fuzz/src/error.rs @@ -46,4 +46,7 @@ pub enum Error { error: sqlx::error::Error, location: Location, }, + + #[snafu(display("Failed to assert: {}", reason))] + Assert { reason: String, location: Location }, } diff --git a/tests-fuzz/src/generator/create_expr.rs b/tests-fuzz/src/generator/create_expr.rs index 534fac827b..3aeb03b954 100644 --- a/tests-fuzz/src/generator/create_expr.rs +++ b/tests-fuzz/src/generator/create_expr.rs @@ -189,10 +189,19 @@ impl Generator for CreateTableExprGenerato #[cfg(test)] mod tests { + use datatypes::value::Value; use rand::SeedableRng; use super::*; + #[test] + fn test_float64() { + let value = Value::from(0.047318541668048164); + assert_eq!("0.047318541668048164", value.to_string()); + let value: f64 = "0.047318541668048164".parse().unwrap(); + assert_eq!("0.047318541668048164", value.to_string()); + } + #[test] fn test_create_table_expr_generator() { let mut rng = rand::thread_rng(); diff --git a/tests-fuzz/src/lib.rs b/tests-fuzz/src/lib.rs index 2666a35051..406927d6b4 100644 --- a/tests-fuzz/src/lib.rs +++ b/tests-fuzz/src/lib.rs @@ -22,6 +22,7 @@ pub mod generator; pub mod ir; pub mod translator; pub mod utils; +pub mod validator; #[cfg(test)] pub mod test_utils; diff --git a/tests-fuzz/src/validator.rs b/tests-fuzz/src/validator.rs new file mode 100644 index 0000000000..198d009a15 --- /dev/null +++ b/tests-fuzz/src/validator.rs @@ -0,0 +1,15 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod column; diff --git a/tests-fuzz/src/validator/column.rs b/tests-fuzz/src/validator/column.rs new file mode 100644 index 0000000000..5b148f7f02 --- /dev/null +++ b/tests-fuzz/src/validator/column.rs @@ -0,0 +1,240 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use common_telemetry::debug; +use datatypes::data_type::DataType; +use snafu::{ensure, ResultExt}; +use sqlx::database::HasArguments; +use sqlx::{ColumnIndex, Database, Decode, Encode, Executor, IntoArguments, Type}; + +use crate::error::{self, Result}; +use crate::ir::create_expr::ColumnOption; +use crate::ir::{Column, Ident}; + +#[derive(Debug, sqlx::FromRow)] +pub struct ColumnEntry { + pub table_schema: String, + pub table_name: String, + pub column_name: String, + pub data_type: String, + pub semantic_type: String, + pub column_default: Option, + pub is_nullable: String, +} + +fn is_nullable(str: &str) -> bool { + str.to_uppercase() == "YES" +} + +impl PartialEq for ColumnEntry { + fn eq(&self, other: &Column) -> bool { + // Checks `table_name` + if other.name.value != self.column_name { + debug!( + "expected name: {}, got: {}", + other.name.value, self.column_name + ); + return false; + } + // Checks `data_type` + if other.column_type.name() != self.data_type { + debug!( + "expected column_type: {}, got: {}", + other.column_type.name(), + self.data_type + ); + return false; + } + // Checks `column_default` + match &self.column_default { + Some(value) => { + let default_value_opt = other.options.iter().find(|opt| { + matches!( + opt, + ColumnOption::DefaultFn(_) | ColumnOption::DefaultValue(_) + ) + }); + if default_value_opt.is_none() { + debug!("default value options is not found"); + return false; + } + let default_value = match default_value_opt.unwrap() { + ColumnOption::DefaultValue(v) => v.to_string(), + ColumnOption::DefaultFn(f) => f.to_string(), + _ => unreachable!(), + }; + if &default_value != value { + debug!("expected default value: {default_value}, got: {value}"); + return false; + } + } + None => { + if other.options.iter().any(|opt| { + matches!( + opt, + ColumnOption::DefaultFn(_) | ColumnOption::DefaultValue(_) + ) + }) { + return false; + } + } + }; + // Checks `is_nullable` + if is_nullable(&self.is_nullable) { + // Null is the default value. Therefore, we only ensure there is no `ColumnOption::NotNull` option. + if other + .options + .iter() + .any(|opt| matches!(opt, ColumnOption::NotNull)) + { + debug!("ColumnOption::NotNull is not found"); + return false; + } + } else { + // `ColumnOption::TimeIndex` imply means the field is not nullable. + if !other + .options + .iter() + .any(|opt| matches!(opt, ColumnOption::NotNull | ColumnOption::TimeIndex)) + { + debug!("unexpected ColumnOption::NotNull or ColumnOption::TimeIndex"); + return false; + } + } + //TODO: Checks `semantic_type` + + true + } +} + +/// Asserts [&[ColumnEntry]] is equal to [&[Column]] +pub fn assert_eq(fetched_columns: &[ColumnEntry], columns: &[Column]) -> Result<()> { + ensure!( + columns.len() == fetched_columns.len(), + error::AssertSnafu { + reason: format!( + "Expected columns length: {}, got: {}", + columns.len(), + fetched_columns.len(), + ) + } + ); + + for (idx, fetched) in fetched_columns.iter().enumerate() { + ensure!( + fetched == &columns[idx], + error::AssertSnafu { + reason: format!( + "ColumnEntry {fetched:?} is not equal to Column {:?}", + columns[idx] + ) + } + ); + } + + Ok(()) +} + +/// Returns all [ColumnEntry] of the `table_name` from `information_schema`. +pub async fn fetch_columns<'a, DB, E>( + e: E, + schema_name: Ident, + table_name: Ident, +) -> Result> +where + DB: Database, + >::Arguments: IntoArguments<'a, DB>, + for<'c> E: 'a + Executor<'c, Database = DB>, + for<'c> String: Decode<'c, DB> + Type, + for<'c> String: Encode<'c, DB> + Type, + for<'c> &'c str: ColumnIndex<::Row>, +{ + let sql = "SELECT * FROM information_schema.columns WHERE table_schema = ? AND table_name = ?"; + sqlx::query_as::<_, ColumnEntry>(sql) + .bind(schema_name.value.to_string()) + .bind(table_name.value.to_string()) + .fetch_all(e) + .await + .context(error::ExecuteQuerySnafu { sql }) +} + +#[cfg(test)] +mod tests { + use datatypes::data_type::{ConcreteDataType, DataType}; + use datatypes::value::Value; + + use super::ColumnEntry; + use crate::ir::create_expr::ColumnOption; + use crate::ir::{Column, Ident}; + + #[test] + fn test_column_eq() { + common_telemetry::init_default_ut_logging(); + let column_entry = ColumnEntry { + table_schema: String::new(), + table_name: String::new(), + column_name: "test".to_string(), + data_type: ConcreteDataType::int8_datatype().name(), + semantic_type: String::new(), + column_default: None, + is_nullable: "Yes".to_string(), + }; + // Naive + let column = Column { + name: Ident::new("test"), + column_type: ConcreteDataType::int8_datatype(), + options: vec![], + }; + assert!(column_entry == column); + // With quote + let column = Column { + name: Ident::with_quote('\'', "test"), + column_type: ConcreteDataType::int8_datatype(), + options: vec![], + }; + assert!(column_entry == column); + // With default value + let column_entry = ColumnEntry { + table_schema: String::new(), + table_name: String::new(), + column_name: "test".to_string(), + data_type: ConcreteDataType::int8_datatype().to_string(), + semantic_type: String::new(), + column_default: Some("1".to_string()), + is_nullable: "Yes".to_string(), + }; + let column = Column { + name: Ident::with_quote('\'', "test"), + column_type: ConcreteDataType::int8_datatype(), + options: vec![ColumnOption::DefaultValue(Value::from(1))], + }; + assert!(column_entry == column); + // With default function + let column_entry = ColumnEntry { + table_schema: String::new(), + table_name: String::new(), + column_name: "test".to_string(), + data_type: ConcreteDataType::int8_datatype().to_string(), + semantic_type: String::new(), + column_default: Some("Hello()".to_string()), + is_nullable: "Yes".to_string(), + }; + let column = Column { + name: Ident::with_quote('\'', "test"), + column_type: ConcreteDataType::int8_datatype(), + options: vec![ColumnOption::DefaultFn("Hello()".to_string())], + }; + assert!(column_entry == column); + } +} diff --git a/tests-fuzz/targets/fuzz_create_table.rs b/tests-fuzz/targets/fuzz_create_table.rs index f3e3cdd7f2..7af489b1c2 100644 --- a/tests-fuzz/targets/fuzz_create_table.rs +++ b/tests-fuzz/targets/fuzz_create_table.rs @@ -32,6 +32,8 @@ use tests_fuzz::ir::CreateTableExpr; use tests_fuzz::translator::mysql::create_expr::CreateTableExprTranslator; use tests_fuzz::translator::DslTranslator; use tests_fuzz::utils::{init_greptime_connections, Connections}; +use tests_fuzz::validator; +use tests_fuzz::validator::column::fetch_columns; struct FuzzContext { greptime: Pool, @@ -52,7 +54,8 @@ struct FuzzInput { impl Arbitrary<'_> for FuzzInput { fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result { let seed = u.int_in_range(u64::MIN..=u64::MAX)?; - let columns = u.int_in_range(2..=10)?; + let mut rng = ChaChaRng::seed_from_u64(seed); + let columns = rng.gen_range(2..30); Ok(FuzzInput { columns, seed }) } } @@ -64,7 +67,7 @@ fn generate_expr(input: FuzzInput) -> Result { WordGenerator, merge_two_word_map_fn(random_capitalize_map, uppercase_and_keyword_backtick_map), ))) - .columns(rng.gen_range(1..input.columns)) + .columns(input.columns) .engine("mito") .build() .unwrap(); @@ -82,6 +85,14 @@ async fn execute_create_table(ctx: FuzzContext, input: FuzzInput) -> Result<()> .context(error::ExecuteQuerySnafu { sql: &sql })?; info!("Create table: {sql}, result: {result:?}"); + // Validate columns + let mut column_entries = + fetch_columns(&ctx.greptime, "public".into(), expr.table_name.clone()).await?; + column_entries.sort_by(|a, b| a.column_name.cmp(&b.column_name)); + let mut columns = expr.columns.clone(); + columns.sort_by(|a, b| a.name.value.cmp(&b.name.value)); + validator::column::assert_eq(&column_entries, &columns)?; + // Cleans up let sql = format!("DROP TABLE {}", expr.table_name); let result = sqlx::query(&sql) From 06dcd0f6edb4c9e9ce822d618601568167f2cc53 Mon Sep 17 00:00:00 2001 From: Yingwen Date: Mon, 11 Mar 2024 22:51:06 +0800 Subject: [PATCH 06/17] fix: freeze data buffer in shard (#3468) * feat: call freeze if the active data buffer in a shard is full * chore: more metrics * chore: print metrics * chore: enlarge freeze threshold * test: test freeze * test: fix config test --- src/mito2/src/memtable/merge_tree.rs | 2 +- src/mito2/src/memtable/merge_tree/data.rs | 17 ++++ .../src/memtable/merge_tree/partition.rs | 61 ++++++++---- src/mito2/src/memtable/merge_tree/shard.rs | 99 +++++++++++++++---- .../src/memtable/merge_tree/shard_builder.rs | 8 +- src/mito2/src/memtable/merge_tree/tree.rs | 4 +- tests-integration/tests/http.rs | 2 +- 7 files changed, 151 insertions(+), 42 deletions(-) diff --git a/src/mito2/src/memtable/merge_tree.rs b/src/mito2/src/memtable/merge_tree.rs index 5f80ba746a..1789959adf 100644 --- a/src/mito2/src/memtable/merge_tree.rs +++ b/src/mito2/src/memtable/merge_tree.rs @@ -85,7 +85,7 @@ impl Default for MergeTreeConfig { Self { index_max_keys_per_shard: 8192, - data_freeze_threshold: 32768, + data_freeze_threshold: 131072, dedup: true, fork_dictionary_bytes, } diff --git a/src/mito2/src/memtable/merge_tree/data.rs b/src/mito2/src/memtable/merge_tree/data.rs index e43d06a223..3f6557ee03 100644 --- a/src/mito2/src/memtable/merge_tree/data.rs +++ b/src/mito2/src/memtable/merge_tree/data.rs @@ -957,6 +957,18 @@ impl DataParts { self.active.write_row(pk_index, kv) } + /// Returns the number of rows in the active buffer. + pub fn num_active_rows(&self) -> usize { + self.active.num_rows() + } + + /// Freezes active buffer and creates a new active buffer. + pub fn freeze(&mut self) -> Result<()> { + let part = self.active.freeze(None, false)?; + self.frozen.push(part); + Ok(()) + } + /// Reads data from all parts including active and frozen parts. /// The returned iterator yields a record batch of one primary key at a time. /// The order of yielding primary keys is determined by provided weights. @@ -976,6 +988,11 @@ impl DataParts { pub(crate) fn is_empty(&self) -> bool { self.active.is_empty() && self.frozen.iter().all(|part| part.is_empty()) } + + #[cfg(test)] + pub(crate) fn frozen_len(&self) -> usize { + self.frozen.len() + } } pub struct DataPartsReaderBuilder { diff --git a/src/mito2/src/memtable/merge_tree/partition.rs b/src/mito2/src/memtable/merge_tree/partition.rs index d4bd0644b5..f031de57eb 100644 --- a/src/mito2/src/memtable/merge_tree/partition.rs +++ b/src/mito2/src/memtable/merge_tree/partition.rs @@ -78,7 +78,7 @@ impl Partition { // Finds key in shards, now we ensure one key only exists in one shard. if let Some(pk_id) = inner.find_key_in_shards(primary_key) { - inner.write_to_shard(pk_id, &key_value); + inner.write_to_shard(pk_id, &key_value)?; inner.num_rows += 1; return Ok(()); } @@ -106,7 +106,7 @@ impl Partition { } /// Writes to the partition without a primary key. - pub fn write_no_key(&self, key_value: KeyValue) { + pub fn write_no_key(&self, key_value: KeyValue) -> Result<()> { let mut inner = self.inner.write().unwrap(); // If no primary key, always write to the first shard. debug_assert!(!inner.shards.is_empty()); @@ -117,12 +117,15 @@ impl Partition { shard_id: 0, pk_index: 0, }; - inner.shards[0].write_with_pk_id(pk_id, &key_value); + inner.shards[0].write_with_pk_id(pk_id, &key_value)?; inner.num_rows += 1; + + Ok(()) } /// Scans data in the partition. pub fn read(&self, mut context: ReadPartitionContext) -> Result { + let start = Instant::now(); let key_filter = if context.need_prune_key { Some(PrimaryKeyFilter::new( context.metadata.clone(), @@ -150,7 +153,7 @@ impl Partition { (builder_reader, shard_source) }; - context.metrics.num_shards = shard_reader_builders.len(); + context.metrics.num_shards += shard_reader_builders.len(); let mut nodes = shard_reader_builders .into_iter() .map(|builder| { @@ -161,7 +164,7 @@ impl Partition { .collect::>>()?; if let Some(builder) = builder_source { - context.metrics.read_builder = true; + context.metrics.num_builder += 1; // Move the initialization of ShardBuilderReader out of read lock. let shard_builder_reader = builder.build(Some(&context.pk_weights), key_filter.clone())?; @@ -172,8 +175,10 @@ impl Partition { let merger = ShardMerger::try_new(nodes)?; if self.dedup { let source = DedupReader::try_new(merger)?; + context.metrics.build_partition_reader += start.elapsed(); PartitionReader::new(context, Box::new(source)) } else { + context.metrics.build_partition_reader += start.elapsed(); PartitionReader::new(context, Box::new(merger)) } } @@ -282,9 +287,10 @@ pub(crate) struct PartitionStats { #[derive(Default)] struct PartitionReaderMetrics { + build_partition_reader: Duration, read_source: Duration, data_batch_to_batch: Duration, - read_builder: bool, + num_builder: usize, num_shards: usize, } @@ -440,9 +446,15 @@ impl Drop for ReadPartitionContext { .observe(partition_data_batch_to_batch); common_telemetry::debug!( - "TreeIter partitions metrics, read_builder: {}, num_shards: {}, partition_read_source: {}s, partition_data_batch_to_batch: {}s", - self.metrics.read_builder, + "TreeIter partitions metrics, \ + num_builder: {}, \ + num_shards: {}, \ + build_partition_reader: {}s, \ + partition_read_source: {}s, \ + partition_data_batch_to_batch: {}s", + self.metrics.num_builder, self.metrics.num_shards, + self.metrics.build_partition_reader.as_secs_f64(), partition_read_source, partition_data_batch_to_batch, ); @@ -549,7 +561,16 @@ impl Inner { fn new(metadata: RegionMetadataRef, config: &MergeTreeConfig) -> Self { let (shards, current_shard_id) = if metadata.primary_key.is_empty() { let data_parts = DataParts::new(metadata.clone(), DATA_INIT_CAP, config.dedup); - (vec![Shard::new(0, None, data_parts, config.dedup)], 1) + ( + vec![Shard::new( + 0, + None, + data_parts, + config.dedup, + config.data_freeze_threshold, + )], + 1, + ) } else { (Vec::new(), 0) }; @@ -569,18 +590,22 @@ impl Inner { self.pk_to_pk_id.get(primary_key).copied() } - fn write_to_shard(&mut self, pk_id: PkId, key_value: &KeyValue) { + fn write_to_shard(&mut self, pk_id: PkId, key_value: &KeyValue) -> Result<()> { if pk_id.shard_id == self.shard_builder.current_shard_id() { self.shard_builder.write_with_pk_id(pk_id, key_value); - return; - } - for shard in &mut self.shards { - if shard.shard_id == pk_id.shard_id { - shard.write_with_pk_id(pk_id, key_value); - self.num_rows += 1; - return; - } + return Ok(()); } + + // Safety: We find the shard by shard id. + let shard = self + .shards + .iter_mut() + .find(|shard| shard.shard_id == pk_id.shard_id) + .unwrap(); + shard.write_with_pk_id(pk_id, key_value)?; + self.num_rows += 1; + + Ok(()) } fn freeze_active_shard(&mut self) -> Result<()> { diff --git a/src/mito2/src/memtable/merge_tree/shard.rs b/src/mito2/src/memtable/merge_tree/shard.rs index 7f981f9162..2ac1ee90bc 100644 --- a/src/mito2/src/memtable/merge_tree/shard.rs +++ b/src/mito2/src/memtable/merge_tree/shard.rs @@ -39,6 +39,8 @@ pub struct Shard { /// Data in the shard. data_parts: DataParts, dedup: bool, + /// Number of rows to freeze a data part. + data_freeze_threshold: usize, } impl Shard { @@ -48,20 +50,29 @@ impl Shard { key_dict: Option, data_parts: DataParts, dedup: bool, + data_freeze_threshold: usize, ) -> Shard { Shard { shard_id, key_dict, data_parts, dedup, + data_freeze_threshold, } } /// Writes a key value into the shard. - pub fn write_with_pk_id(&mut self, pk_id: PkId, key_value: &KeyValue) { + /// + /// It will freezes the active buffer if it is full. + pub fn write_with_pk_id(&mut self, pk_id: PkId, key_value: &KeyValue) -> Result<()> { debug_assert_eq!(self.shard_id, pk_id.shard_id); + if self.data_parts.num_active_rows() >= self.data_freeze_threshold { + self.data_parts.freeze()?; + } + self.data_parts.write_row(pk_id.pk_index, key_value); + Ok(()) } /// Scans the shard. @@ -83,6 +94,7 @@ impl Shard { key_dict: self.key_dict.clone(), data_parts: DataParts::new(metadata, DATA_INIT_CAP, self.dedup), dedup: self.dedup, + data_freeze_threshold: self.data_freeze_threshold, } } @@ -467,6 +479,7 @@ mod tests { shard_id: ShardId, metadata: RegionMetadataRef, input: &[(KeyValues, PkIndex)], + data_freeze_threshold: usize, ) -> Shard { let mut dict_builder = KeyDictBuilder::new(1024); let mut metrics = WriteMetrics::default(); @@ -481,26 +494,16 @@ mod tests { let dict = dict_builder.finish(&mut BTreeMap::new()).unwrap(); let data_parts = DataParts::new(metadata, DATA_INIT_CAP, true); - Shard::new(shard_id, Some(Arc::new(dict)), data_parts, true) + Shard::new( + shard_id, + Some(Arc::new(dict)), + data_parts, + true, + data_freeze_threshold, + ) } - #[test] - fn test_write_read_shard() { - let metadata = metadata_for_test(); - let input = input_with_key(&metadata); - let mut shard = new_shard_with_dict(8, metadata, &input); - assert!(shard.is_empty()); - for (key_values, pk_index) in &input { - for kv in key_values.iter() { - let pk_id = PkId { - shard_id: shard.shard_id, - pk_index: *pk_index, - }; - shard.write_with_pk_id(pk_id, &kv); - } - } - assert!(!shard.is_empty()); - + fn collect_timestamps(shard: &Shard) -> Vec { let mut reader = shard.read().unwrap().build(None).unwrap(); let mut timestamps = Vec::new(); while reader.is_valid() { @@ -511,6 +514,64 @@ mod tests { reader.next().unwrap(); } + timestamps + } + + #[test] + fn test_write_read_shard() { + let metadata = metadata_for_test(); + let input = input_with_key(&metadata); + let mut shard = new_shard_with_dict(8, metadata, &input, 100); + assert!(shard.is_empty()); + for (key_values, pk_index) in &input { + for kv in key_values.iter() { + let pk_id = PkId { + shard_id: shard.shard_id, + pk_index: *pk_index, + }; + shard.write_with_pk_id(pk_id, &kv).unwrap(); + } + } + assert!(!shard.is_empty()); + + let timestamps = collect_timestamps(&shard); assert_eq!(vec![0, 1, 10, 11, 20, 21], timestamps); } + + #[test] + fn test_shard_freeze() { + let metadata = metadata_for_test(); + let kvs = build_key_values_with_ts_seq_values( + &metadata, + "shard".to_string(), + 0, + [0].into_iter(), + [Some(0.0)].into_iter(), + 0, + ); + let mut shard = new_shard_with_dict(8, metadata.clone(), &[(kvs, 0)], 50); + let expected: Vec<_> = (0..200).collect(); + for i in &expected { + let kvs = build_key_values_with_ts_seq_values( + &metadata, + "shard".to_string(), + 0, + [*i].into_iter(), + [Some(0.0)].into_iter(), + *i as u64, + ); + let pk_id = PkId { + shard_id: shard.shard_id, + pk_index: *i as PkIndex, + }; + for kv in kvs.iter() { + shard.write_with_pk_id(pk_id, &kv).unwrap(); + } + } + assert!(!shard.is_empty()); + assert_eq!(3, shard.data_parts.frozen_len()); + + let timestamps = collect_timestamps(&shard); + assert_eq!(expected, timestamps); + } } diff --git a/src/mito2/src/memtable/merge_tree/shard_builder.rs b/src/mito2/src/memtable/merge_tree/shard_builder.rs index 2b007ebd87..01cb2de25a 100644 --- a/src/mito2/src/memtable/merge_tree/shard_builder.rs +++ b/src/mito2/src/memtable/merge_tree/shard_builder.rs @@ -138,7 +138,13 @@ impl ShardBuilder { let shard_id = self.current_shard_id; self.current_shard_id += 1; - Ok(Some(Shard::new(shard_id, key_dict, data_parts, self.dedup))) + Ok(Some(Shard::new( + shard_id, + key_dict, + data_parts, + self.dedup, + self.data_freeze_threshold, + ))) } /// Scans the shard builder. diff --git a/src/mito2/src/memtable/merge_tree/tree.rs b/src/mito2/src/memtable/merge_tree/tree.rs index 94c87f7583..0a42e13fde 100644 --- a/src/mito2/src/memtable/merge_tree/tree.rs +++ b/src/mito2/src/memtable/merge_tree/tree.rs @@ -124,7 +124,7 @@ impl MergeTree { if !has_pk { // No primary key. - self.write_no_key(kv); + self.write_no_key(kv)?; continue; } @@ -299,7 +299,7 @@ impl MergeTree { ) } - fn write_no_key(&self, key_value: KeyValue) { + fn write_no_key(&self, key_value: KeyValue) -> Result<()> { let partition_key = Partition::get_partition_key(&key_value, self.is_partitioned); let partition = self.get_or_create_partition(partition_key); diff --git a/tests-integration/tests/http.rs b/tests-integration/tests/http.rs index 32feebe161..a75e9e2196 100644 --- a/tests-integration/tests/http.rs +++ b/tests-integration/tests/http.rs @@ -789,7 +789,7 @@ intermediate_path = "" [datanode.region_engine.mito.memtable] type = "experimental" index_max_keys_per_shard = 8192 -data_freeze_threshold = 32768 +data_freeze_threshold = 131072 dedup = true fork_dictionary_bytes = "1GiB" From 1255c1fc9e6f2875ac6b4c1c043db5d4c6479906 Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 12 Mar 2024 09:46:19 +0800 Subject: [PATCH 07/17] feat: to_timezone function (#3470) * feat: to_timezone function Signed-off-by: tison * impl Function for ToTimezoneFunction Signed-off-by: tison * add test Signed-off-by: tison * Add original authors Co-authored-by: parkma99 Co-authored-by: Yingwen * fixup Signed-off-by: tison * address comments Signed-off-by: tison * add issue link Signed-off-by: tison * code refactor Signed-off-by: tison * further tidy Signed-off-by: tison --------- Signed-off-by: tison Co-authored-by: parkma99 Co-authored-by: Yingwen --- .editorconfig | 10 + src/common/function/src/scalars/timestamp.rs | 3 + .../src/scalars/timestamp/to_timezone.rs | 260 ++++++++++++++++++ .../standalone/common/function/time.result | 36 +++ .../cases/standalone/common/function/time.sql | 8 +- 5 files changed, 316 insertions(+), 1 deletion(-) create mode 100644 .editorconfig create mode 100644 src/common/function/src/scalars/timestamp/to_timezone.rs diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..4de4965659 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,10 @@ +root = true + +[*] +end_of_line = lf +indent_style = space +insert_final_newline = true +trim_trailing_whitespace = true + +[{Makefile,**.mk}] +indent_style = tab diff --git a/src/common/function/src/scalars/timestamp.rs b/src/common/function/src/scalars/timestamp.rs index fecf884ce0..fbee427921 100644 --- a/src/common/function/src/scalars/timestamp.rs +++ b/src/common/function/src/scalars/timestamp.rs @@ -14,9 +14,11 @@ use std::sync::Arc; mod greatest; +mod to_timezone; mod to_unixtime; use greatest::GreatestFunction; +use to_timezone::ToTimezoneFunction; use to_unixtime::ToUnixtimeFunction; use crate::function_registry::FunctionRegistry; @@ -25,6 +27,7 @@ pub(crate) struct TimestampFunction; impl TimestampFunction { pub fn register(registry: &FunctionRegistry) { + registry.register(Arc::new(ToTimezoneFunction)); registry.register(Arc::new(ToUnixtimeFunction)); registry.register(Arc::new(GreatestFunction)); } diff --git a/src/common/function/src/scalars/timestamp/to_timezone.rs b/src/common/function/src/scalars/timestamp/to_timezone.rs new file mode 100644 index 0000000000..1160267dbd --- /dev/null +++ b/src/common/function/src/scalars/timestamp/to_timezone.rs @@ -0,0 +1,260 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt; +use std::sync::Arc; + +use common_query::error::{InvalidFuncArgsSnafu, Result, UnsupportedInputDataTypeSnafu}; +use common_query::prelude::Signature; +use common_time::{Timestamp, Timezone}; +use datatypes::data_type::ConcreteDataType; +use datatypes::prelude::VectorRef; +use datatypes::types::TimestampType; +use datatypes::value::Value; +use datatypes::vectors::{ + StringVector, TimestampMicrosecondVector, TimestampMillisecondVector, + TimestampNanosecondVector, TimestampSecondVector, Vector, +}; +use snafu::{ensure, OptionExt}; + +use crate::function::{Function, FunctionContext}; +use crate::helper; + +#[derive(Clone, Debug, Default)] +pub struct ToTimezoneFunction; + +const NAME: &str = "to_timezone"; + +fn convert_to_timezone(arg: &str) -> Option { + Timezone::from_tz_string(arg).ok() +} + +fn convert_to_timestamp(arg: &Value) -> Option { + match arg { + Value::Timestamp(ts) => Some(*ts), + _ => None, + } +} + +impl fmt::Display for ToTimezoneFunction { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "TO_TIMEZONE") + } +} + +impl Function for ToTimezoneFunction { + fn name(&self) -> &str { + NAME + } + + fn return_type(&self, input_types: &[ConcreteDataType]) -> Result { + // type checked by signature - MUST BE timestamp + Ok(input_types[0].clone()) + } + + fn signature(&self) -> Signature { + helper::one_of_sigs2( + vec![ + ConcreteDataType::timestamp_second_datatype(), + ConcreteDataType::timestamp_millisecond_datatype(), + ConcreteDataType::timestamp_microsecond_datatype(), + ConcreteDataType::timestamp_nanosecond_datatype(), + ], + vec![ConcreteDataType::string_datatype()], + ) + } + + fn eval(&self, _ctx: FunctionContext, columns: &[VectorRef]) -> Result { + ensure!( + columns.len() == 2, + InvalidFuncArgsSnafu { + err_msg: format!( + "The length of the args is not correct, expect exactly 2, have: {}", + columns.len() + ), + } + ); + + // TODO: maybe support epoch timestamp? https://github.com/GreptimeTeam/greptimedb/issues/3477 + let ts = columns[0].data_type().as_timestamp().with_context(|| { + UnsupportedInputDataTypeSnafu { + function: NAME, + datatypes: columns.iter().map(|c| c.data_type()).collect::>(), + } + })?; + let array = columns[0].to_arrow_array(); + let times = match ts { + TimestampType::Second(_) => { + let vector = TimestampSecondVector::try_from_arrow_array(array).unwrap(); + (0..vector.len()) + .map(|i| convert_to_timestamp(&vector.get(i))) + .collect::>() + } + TimestampType::Millisecond(_) => { + let vector = TimestampMillisecondVector::try_from_arrow_array(array).unwrap(); + (0..vector.len()) + .map(|i| convert_to_timestamp(&vector.get(i))) + .collect::>() + } + TimestampType::Microsecond(_) => { + let vector = TimestampMicrosecondVector::try_from_arrow_array(array).unwrap(); + (0..vector.len()) + .map(|i| convert_to_timestamp(&vector.get(i))) + .collect::>() + } + TimestampType::Nanosecond(_) => { + let vector = TimestampNanosecondVector::try_from_arrow_array(array).unwrap(); + (0..vector.len()) + .map(|i| convert_to_timestamp(&vector.get(i))) + .collect::>() + } + }; + + let tzs = { + let array = columns[1].to_arrow_array(); + let vector = StringVector::try_from_arrow_array(&array) + .ok() + .with_context(|| UnsupportedInputDataTypeSnafu { + function: NAME, + datatypes: columns.iter().map(|c| c.data_type()).collect::>(), + })?; + (0..vector.len()) + .map(|i| convert_to_timezone(&vector.get(i).to_string())) + .collect::>() + }; + + let result = times + .iter() + .zip(tzs.iter()) + .map(|(time, tz)| match (time, tz) { + (Some(time), _) => Some(time.to_timezone_aware_string(tz.as_ref())), + _ => None, + }) + .collect::>>(); + Ok(Arc::new(StringVector::from(result))) + } +} + +#[cfg(test)] +mod tests { + + use datatypes::scalars::ScalarVector; + use datatypes::timestamp::{ + TimestampMicrosecond, TimestampMillisecond, TimestampNanosecond, TimestampSecond, + }; + use datatypes::vectors::StringVector; + + use super::*; + + #[test] + fn test_timestamp_to_timezone() { + let f = ToTimezoneFunction; + assert_eq!("to_timezone", f.name()); + + let results = vec![ + Some("1969-12-31 19:00:01"), + None, + Some("1970-01-01 03:00:01"), + None, + ]; + let times: Vec> = vec![ + Some(TimestampSecond::new(1)), + None, + Some(TimestampSecond::new(1)), + None, + ]; + let ts_vector: TimestampSecondVector = + TimestampSecondVector::from_owned_iterator(times.into_iter()); + let tzs = vec![Some("America/New_York"), None, Some("Europe/Moscow"), None]; + let args: Vec = vec![ + Arc::new(ts_vector), + Arc::new(StringVector::from(tzs.clone())), + ]; + let vector = f.eval(FunctionContext::default(), &args).unwrap(); + assert_eq!(4, vector.len()); + let expect_times: VectorRef = Arc::new(StringVector::from(results)); + assert_eq!(expect_times, vector); + + let results = vec![ + Some("1969-12-31 19:00:00.001"), + None, + Some("1970-01-01 03:00:00.001"), + None, + ]; + let times: Vec> = vec![ + Some(TimestampMillisecond::new(1)), + None, + Some(TimestampMillisecond::new(1)), + None, + ]; + let ts_vector: TimestampMillisecondVector = + TimestampMillisecondVector::from_owned_iterator(times.into_iter()); + let args: Vec = vec![ + Arc::new(ts_vector), + Arc::new(StringVector::from(tzs.clone())), + ]; + let vector = f.eval(FunctionContext::default(), &args).unwrap(); + assert_eq!(4, vector.len()); + let expect_times: VectorRef = Arc::new(StringVector::from(results)); + assert_eq!(expect_times, vector); + + let results = vec![ + Some("1969-12-31 19:00:00.000001"), + None, + Some("1970-01-01 03:00:00.000001"), + None, + ]; + let times: Vec> = vec![ + Some(TimestampMicrosecond::new(1)), + None, + Some(TimestampMicrosecond::new(1)), + None, + ]; + let ts_vector: TimestampMicrosecondVector = + TimestampMicrosecondVector::from_owned_iterator(times.into_iter()); + + let args: Vec = vec![ + Arc::new(ts_vector), + Arc::new(StringVector::from(tzs.clone())), + ]; + let vector = f.eval(FunctionContext::default(), &args).unwrap(); + assert_eq!(4, vector.len()); + let expect_times: VectorRef = Arc::new(StringVector::from(results)); + assert_eq!(expect_times, vector); + + let results = vec![ + Some("1969-12-31 19:00:00.000000001"), + None, + Some("1970-01-01 03:00:00.000000001"), + None, + ]; + let times: Vec> = vec![ + Some(TimestampNanosecond::new(1)), + None, + Some(TimestampNanosecond::new(1)), + None, + ]; + let ts_vector: TimestampNanosecondVector = + TimestampNanosecondVector::from_owned_iterator(times.into_iter()); + + let args: Vec = vec![ + Arc::new(ts_vector), + Arc::new(StringVector::from(tzs.clone())), + ]; + let vector = f.eval(FunctionContext::default(), &args).unwrap(); + assert_eq!(4, vector.len()); + let expect_times: VectorRef = Arc::new(StringVector::from(results)); + assert_eq!(expect_times, vector); + } +} diff --git a/tests/cases/standalone/common/function/time.result b/tests/cases/standalone/common/function/time.result index 123b6a3f2f..7c6815682c 100644 --- a/tests/cases/standalone/common/function/time.result +++ b/tests/cases/standalone/common/function/time.result @@ -20,3 +20,39 @@ select GREATEST('2000-02-11'::Date, '2020-12-30'::Date); | 2020-12-30 | +-------------------------------------------------+ +select to_timezone('2022-09-20T14:16:43.012345+08:00', 'Europe/Berlin'); + ++-----------------------------------------------------------------------------+ +| to_timezone(Utf8("2022-09-20T14:16:43.012345+08:00"),Utf8("Europe/Berlin")) | ++-----------------------------------------------------------------------------+ +| 2022-09-20 08:16:43.012345 | ++-----------------------------------------------------------------------------+ + +select to_timezone('2022-09-20T14:16:43.012345+08:00'::Timestamp, 'Europe/Berlin'); + ++-----------------------------------------------------------------------------+ +| to_timezone(Utf8("2022-09-20T14:16:43.012345+08:00"),Utf8("Europe/Berlin")) | ++-----------------------------------------------------------------------------+ +| 2022-09-20 08:16:43.012 | ++-----------------------------------------------------------------------------+ + +select to_timezone('2024-03-29T14:16:43.012345Z', 'Asia/Shanghai'); + ++------------------------------------------------------------------------+ +| to_timezone(Utf8("2024-03-29T14:16:43.012345Z"),Utf8("Asia/Shanghai")) | ++------------------------------------------------------------------------+ +| 2024-03-29 22:16:43.012345 | ++------------------------------------------------------------------------+ + +select to_timezone('2024-03-29T14:16:43.012345Z'::Timestamp, 'Asia/Shanghai'); + ++------------------------------------------------------------------------+ +| to_timezone(Utf8("2024-03-29T14:16:43.012345Z"),Utf8("Asia/Shanghai")) | ++------------------------------------------------------------------------+ +| 2024-03-29 22:16:43.012 | ++------------------------------------------------------------------------+ + +select to_timezone(1709992225, 'Asia/Shanghai'); + +Error: 3001(EngineExecuteQuery), DataFusion error: Error during planning: Coercion from [Int64, Utf8] to the signature OneOf([Exact([Timestamp(Second, None), Utf8]), Exact([Timestamp(Millisecond, None), Utf8]), Exact([Timestamp(Microsecond, None), Utf8]), Exact([Timestamp(Nanosecond, None), Utf8])]) failed. + diff --git a/tests/cases/standalone/common/function/time.sql b/tests/cases/standalone/common/function/time.sql index 46d5c2347f..a8e59132a5 100644 --- a/tests/cases/standalone/common/function/time.sql +++ b/tests/cases/standalone/common/function/time.sql @@ -3,5 +3,11 @@ select current_time(); select GREATEST('1999-01-30', '2023-03-01'); - select GREATEST('2000-02-11'::Date, '2020-12-30'::Date); + +select to_timezone('2022-09-20T14:16:43.012345+08:00', 'Europe/Berlin'); +select to_timezone('2022-09-20T14:16:43.012345+08:00'::Timestamp, 'Europe/Berlin'); +select to_timezone('2024-03-29T14:16:43.012345Z', 'Asia/Shanghai'); +select to_timezone('2024-03-29T14:16:43.012345Z'::Timestamp, 'Asia/Shanghai'); + +select to_timezone(1709992225, 'Asia/Shanghai'); From 7639c227caac8ca100df21cfe1cff3dadd195f4f Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Tue, 12 Mar 2024 10:09:27 +0800 Subject: [PATCH 08/17] feat(flow): accumlator for aggr func (#3396) * feat: Accumlator trait * feat: add `OrdValue` accum&use enum_dispatch * test: more accum test * feat: eval aggr funcs * chore: refactor test&fmt clippy * refactor: less verbose * test: more tests * refactor: better err handling&use OrdValue for Count * refactor: ignore null&more tests for error handle * refactor: OrdValue accum * chore: extract null check * refactor: def&use fn signature * chore: use extra cond with match guard * chore: per review --- Cargo.lock | 2 + src/flow/Cargo.toml | 4 + src/flow/src/expr/error.rs | 3 + src/flow/src/expr/relation/accum.rs | 923 +++++++++++++++++++++++++++- src/flow/src/expr/relation/func.rs | 281 ++++++++- src/flow/src/repr.rs | 5 +- 6 files changed, 1187 insertions(+), 31 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3ec7ea27d9..4ae09096a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3474,10 +3474,12 @@ dependencies = [ "common-telemetry", "common-time", "datatypes", + "enum_dispatch", "hydroflow", "itertools 0.10.5", "num-traits", "serde", + "serde_json", "servers", "session", "snafu", diff --git a/src/flow/Cargo.toml b/src/flow/Cargo.toml index 0dc614a6c0..cc67850eac 100644 --- a/src/flow/Cargo.toml +++ b/src/flow/Cargo.toml @@ -18,6 +18,7 @@ common-query.workspace = true common-telemetry.workspace = true common-time.workspace = true datatypes.workspace = true +enum_dispatch = "0.3" hydroflow = "0.5.0" itertools.workspace = true num-traits = "0.2" @@ -27,3 +28,6 @@ session.workspace = true snafu.workspace = true tokio.workspace = true tonic.workspace = true + +[dev-dependencies] +serde_json = "1.0" diff --git a/src/flow/src/expr/error.rs b/src/flow/src/expr/error.rs index 233538fb65..9de1892316 100644 --- a/src/flow/src/expr/error.rs +++ b/src/flow/src/expr/error.rs @@ -61,4 +61,7 @@ pub enum EvalError { #[snafu(display("Unsupported temporal filter: {reason}"))] UnsupportedTemporalFilter { reason: String, location: Location }, + + #[snafu(display("Overflowed during evaluation"))] + Overflow { location: Location }, } diff --git a/src/flow/src/expr/relation/accum.rs b/src/flow/src/expr/relation/accum.rs index e2b136e849..06df89eb8e 100644 --- a/src/flow/src/expr/relation/accum.rs +++ b/src/flow/src/expr/relation/accum.rs @@ -14,7 +14,10 @@ //! Accumulators for aggregate functions that's is accumulatable. i.e. sum/count //! -//! Currently support sum, count, any, all +//! Accumulator will only be restore from row and being updated every time dataflow need process a new batch of rows. +//! So the overhead is acceptable. +//! +//! Currently support sum, count, any, all and min/max(with one caveat that min/max can't support delete with aggregate). use std::fmt::Display; @@ -22,13 +25,506 @@ use common_decimal::Decimal128; use common_time::{Date, DateTime}; use datatypes::data_type::ConcreteDataType; use datatypes::value::{OrderedF32, OrderedF64, OrderedFloat, Value}; +use enum_dispatch::enum_dispatch; use hydroflow::futures::stream::Concat; use serde::{Deserialize, Serialize}; +use snafu::ensure; -use crate::expr::error::{InternalSnafu, TryFromValueSnafu, TypeMismatchSnafu}; +use crate::expr::error::{InternalSnafu, OverflowSnafu, TryFromValueSnafu, TypeMismatchSnafu}; +use crate::expr::relation::func::GenericFn; use crate::expr::{AggregateFunc, EvalError}; use crate::repr::Diff; +/// Accumulates values for the various types of accumulable aggregations. +#[enum_dispatch] +pub trait Accumulator: Sized { + fn into_state(self) -> Vec; + fn update( + &mut self, + aggr_fn: &AggregateFunc, + value: Value, + diff: Diff, + ) -> Result<(), EvalError>; + + fn update_batch(&mut self, aggr_fn: &AggregateFunc, value_diffs: I) -> Result<(), EvalError> + where + I: IntoIterator, + { + for (v, d) in value_diffs { + self.update(aggr_fn, v, d)?; + } + Ok(()) + } + + fn eval(&self, aggr_fn: &AggregateFunc) -> Result; +} + +/// Bool accumulator, used for `Any` `All` `Max/MinBool` +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct Bool { + /// The number of `true` values observed. + trues: Diff, + /// The number of `false` values observed. + falses: Diff, +} + +impl TryFrom> for Bool { + type Error = EvalError; + + fn try_from(state: Vec) -> Result { + ensure!( + state.len() == 2, + InternalSnafu { + reason: "Bool Accumulator state should have 2 values", + } + ); + + let mut iter = state.into_iter(); + + Ok(Self { + trues: Diff::try_from(iter.next().unwrap()).map_err(err_try_from_val)?, + falses: Diff::try_from(iter.next().unwrap()).map_err(err_try_from_val)?, + }) + } +} + +impl Accumulator for Bool { + fn into_state(self) -> Vec { + vec![self.trues.into(), self.falses.into()] + } + + /// Null values are ignored + fn update( + &mut self, + aggr_fn: &AggregateFunc, + value: Value, + diff: Diff, + ) -> Result<(), EvalError> { + ensure!( + matches!( + aggr_fn, + AggregateFunc::Any + | AggregateFunc::All + | AggregateFunc::MaxBool + | AggregateFunc::MinBool + ), + InternalSnafu { + reason: format!( + "Bool Accumulator does not support this aggregation function: {:?}", + aggr_fn + ), + } + ); + + match value { + Value::Boolean(true) => self.trues += diff, + Value::Boolean(false) => self.falses += diff, + Value::Null => (), // ignore nulls + x => { + return Err(TypeMismatchSnafu { + expected: ConcreteDataType::boolean_datatype(), + actual: x.data_type(), + } + .build()); + } + }; + Ok(()) + } + + fn eval(&self, aggr_fn: &AggregateFunc) -> Result { + match aggr_fn { + AggregateFunc::Any => Ok(Value::from(self.trues > 0)), + AggregateFunc::All => Ok(Value::from(self.falses == 0)), + AggregateFunc::MaxBool => Ok(Value::from(self.trues > 0)), + AggregateFunc::MinBool => Ok(Value::from(self.falses == 0)), + _ => Err(InternalSnafu { + reason: format!( + "Bool Accumulator does not support this aggregation function: {:?}", + aggr_fn + ), + } + .build()), + } + } +} + +/// Accumulates simple numeric values for sum over integer. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct SimpleNumber { + /// The accumulation of all non-NULL values observed. + accum: i128, + /// The number of non-NULL values observed. + non_nulls: Diff, +} + +impl TryFrom> for SimpleNumber { + type Error = EvalError; + + fn try_from(state: Vec) -> Result { + ensure!( + state.len() == 2, + InternalSnafu { + reason: "Number Accumulator state should have 2 values", + } + ); + let mut iter = state.into_iter(); + + Ok(Self { + accum: Decimal128::try_from(iter.next().unwrap()) + .map_err(err_try_from_val)? + .val(), + non_nulls: Diff::try_from(iter.next().unwrap()).map_err(err_try_from_val)?, + }) + } +} + +impl Accumulator for SimpleNumber { + fn into_state(self) -> Vec { + vec![ + Value::Decimal128(Decimal128::new(self.accum, 38, 0)), + self.non_nulls.into(), + ] + } + + fn update( + &mut self, + aggr_fn: &AggregateFunc, + value: Value, + diff: Diff, + ) -> Result<(), EvalError> { + ensure!( + matches!( + aggr_fn, + AggregateFunc::SumInt16 + | AggregateFunc::SumInt32 + | AggregateFunc::SumInt64 + | AggregateFunc::SumUInt16 + | AggregateFunc::SumUInt32 + | AggregateFunc::SumUInt64 + ), + InternalSnafu { + reason: format!( + "SimpleNumber Accumulator does not support this aggregation function: {:?}", + aggr_fn + ), + } + ); + + let v = match (aggr_fn, value) { + (AggregateFunc::SumInt16, Value::Int16(x)) => i128::from(x), + (AggregateFunc::SumInt32, Value::Int32(x)) => i128::from(x), + (AggregateFunc::SumInt64, Value::Int64(x)) => i128::from(x), + (AggregateFunc::SumUInt16, Value::UInt16(x)) => i128::from(x), + (AggregateFunc::SumUInt32, Value::UInt32(x)) => i128::from(x), + (AggregateFunc::SumUInt64, Value::UInt64(x)) => i128::from(x), + (_f, Value::Null) => return Ok(()), // ignore null + (f, v) => { + let expected_datatype = f.signature().input; + return Err(TypeMismatchSnafu { + expected: expected_datatype, + actual: v.data_type(), + } + .build())?; + } + }; + + self.accum += v * i128::from(diff); + + self.non_nulls += diff; + Ok(()) + } + + fn eval(&self, aggr_fn: &AggregateFunc) -> Result { + match aggr_fn { + AggregateFunc::SumInt16 | AggregateFunc::SumInt32 | AggregateFunc::SumInt64 => { + i64::try_from(self.accum) + .map_err(|_e| OverflowSnafu {}.build()) + .map(Value::from) + } + AggregateFunc::SumUInt16 | AggregateFunc::SumUInt32 | AggregateFunc::SumUInt64 => { + u64::try_from(self.accum) + .map_err(|_e| OverflowSnafu {}.build()) + .map(Value::from) + } + _ => Err(InternalSnafu { + reason: format!( + "SimpleNumber Accumulator does not support this aggregation function: {:?}", + aggr_fn + ), + } + .build()), + } + } +} +/// Accumulates float values for sum over floating numbers. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] + +pub struct Float { + /// Accumulates non-special float values, i.e. not NaN, +inf, -inf. + /// accum will be set to zero if `non_nulls` is zero. + accum: OrderedF64, + /// Counts +inf + pos_infs: Diff, + /// Counts -inf + neg_infs: Diff, + /// Counts NaNs + nans: Diff, + /// Counts non-NULL values + non_nulls: Diff, +} + +impl TryFrom> for Float { + type Error = EvalError; + + fn try_from(state: Vec) -> Result { + ensure!( + state.len() == 5, + InternalSnafu { + reason: "Float Accumulator state should have 5 values", + } + ); + + let mut iter = state.into_iter(); + + let mut ret = Self { + accum: OrderedF64::try_from(iter.next().unwrap()).map_err(err_try_from_val)?, + pos_infs: Diff::try_from(iter.next().unwrap()).map_err(err_try_from_val)?, + neg_infs: Diff::try_from(iter.next().unwrap()).map_err(err_try_from_val)?, + nans: Diff::try_from(iter.next().unwrap()).map_err(err_try_from_val)?, + non_nulls: Diff::try_from(iter.next().unwrap()).map_err(err_try_from_val)?, + }; + + // This prevent counter-intuitive behavior of summing over no values + if ret.non_nulls == 0 { + ret.accum = OrderedFloat::from(0.0); + } + + Ok(ret) + } +} + +impl Accumulator for Float { + fn into_state(self) -> Vec { + vec![ + self.accum.into(), + self.pos_infs.into(), + self.neg_infs.into(), + self.nans.into(), + self.non_nulls.into(), + ] + } + + /// sum ignore null + fn update( + &mut self, + aggr_fn: &AggregateFunc, + value: Value, + diff: Diff, + ) -> Result<(), EvalError> { + ensure!( + matches!( + aggr_fn, + AggregateFunc::SumFloat32 | AggregateFunc::SumFloat64 + ), + InternalSnafu { + reason: format!( + "Float Accumulator does not support this aggregation function: {:?}", + aggr_fn + ), + } + ); + + let x = match (aggr_fn, value) { + (AggregateFunc::SumFloat32, Value::Float32(x)) => OrderedF64::from(*x as f64), + (AggregateFunc::SumFloat64, Value::Float64(x)) => OrderedF64::from(x), + (_f, Value::Null) => return Ok(()), // ignore null + (f, v) => { + let expected_datatype = f.signature().input; + return Err(TypeMismatchSnafu { + expected: expected_datatype, + actual: v.data_type(), + } + .build())?; + } + }; + + if x.is_nan() { + self.nans += diff; + } else if x.is_infinite() { + if x.is_sign_positive() { + self.pos_infs += diff; + } else { + self.neg_infs += diff; + } + } else { + self.accum += *(x * OrderedF64::from(diff as f64)); + } + + self.non_nulls += diff; + Ok(()) + } + + fn eval(&self, aggr_fn: &AggregateFunc) -> Result { + match aggr_fn { + AggregateFunc::SumFloat32 => Ok(Value::Float32(OrderedF32::from(self.accum.0 as f32))), + AggregateFunc::SumFloat64 => Ok(Value::Float64(self.accum)), + _ => Err(InternalSnafu { + reason: format!( + "Float Accumulator does not support this aggregation function: {:?}", + aggr_fn + ), + } + .build()), + } + } +} + +/// Accumulates a single `Ord`ed `Value`, useful for min/max aggregations. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct OrdValue { + val: Option, + non_nulls: Diff, +} + +impl TryFrom> for OrdValue { + type Error = EvalError; + + fn try_from(state: Vec) -> Result { + ensure!( + state.len() == 2, + InternalSnafu { + reason: "OrdValue Accumulator state should have 2 values", + } + ); + + let mut iter = state.into_iter(); + + Ok(Self { + val: { + let v = iter.next().unwrap(); + if v == Value::Null { + None + } else { + Some(v) + } + }, + non_nulls: Diff::try_from(iter.next().unwrap()).map_err(err_try_from_val)?, + }) + } +} + +impl Accumulator for OrdValue { + fn into_state(self) -> Vec { + vec![self.val.unwrap_or(Value::Null), self.non_nulls.into()] + } + + /// min/max try to find results in all non-null values, if all values are null, the result is null. + /// count(col_name) gives the number of non-null values, count(*) gives the number of rows including nulls. + /// TODO(discord9): add count(*) as a aggr function + fn update( + &mut self, + aggr_fn: &AggregateFunc, + value: Value, + diff: Diff, + ) -> Result<(), EvalError> { + ensure!( + aggr_fn.is_max() || aggr_fn.is_min() || matches!(aggr_fn, AggregateFunc::Count), + InternalSnafu { + reason: format!( + "OrdValue Accumulator does not support this aggregation function: {:?}", + aggr_fn + ), + } + ); + if diff <= 0 && (aggr_fn.is_max() || aggr_fn.is_min()) { + return Err(InternalSnafu { + reason: "OrdValue Accumulator does not support non-monotonic input for min/max aggregation".to_string(), + }.build()); + } + + // if aggr_fn is count, the incoming value type doesn't matter in type checking + // otherwise, type need to be the same or value can be null + let check_type_aggr_fn_and_arg_value = + ty_eq_without_precision(value.data_type(), aggr_fn.signature().input) + || matches!(aggr_fn, AggregateFunc::Count) + || value.is_null(); + let check_type_aggr_fn_and_self_val = self + .val + .as_ref() + .map(|zelf| ty_eq_without_precision(zelf.data_type(), aggr_fn.signature().input)) + .unwrap_or(true) + || matches!(aggr_fn, AggregateFunc::Count); + + if !check_type_aggr_fn_and_arg_value { + return Err(TypeMismatchSnafu { + expected: aggr_fn.signature().input, + actual: value.data_type(), + } + .build()); + } else if !check_type_aggr_fn_and_self_val { + return Err(TypeMismatchSnafu { + expected: aggr_fn.signature().input, + actual: self + .val + .as_ref() + .map(|v| v.data_type()) + .unwrap_or(ConcreteDataType::null_datatype()), + } + .build()); + } + + let is_null = value.is_null(); + if is_null { + return Ok(()); + } + + if !is_null { + // compile count(*) to count(true) to include null/non-nulls + // And the counts of non-null values are updated here + self.non_nulls += diff; + + match aggr_fn.signature().generic_fn { + GenericFn::Max => { + self.val = self + .val + .clone() + .map(|v| v.max(value.clone())) + .or_else(|| Some(value)) + } + GenericFn::Min => { + self.val = self + .val + .clone() + .map(|v| v.min(value.clone())) + .or_else(|| Some(value)) + } + + GenericFn::Count => (), + _ => unreachable!("already checked by ensure!"), + } + }; + // min/max ignore nulls + + Ok(()) + } + + fn eval(&self, aggr_fn: &AggregateFunc) -> Result { + if aggr_fn.is_max() || aggr_fn.is_min() { + Ok(self.val.clone().unwrap_or(Value::Null)) + } else if matches!(aggr_fn, AggregateFunc::Count) { + Ok(self.non_nulls.into()) + } else { + Err(InternalSnafu { + reason: format!( + "OrdValue Accumulator does not support this aggregation function: {:?}", + aggr_fn + ), + } + .build()) + } + } +} + /// Accumulates values for the various types of accumulable aggregations. /// /// We assume that there are not more than 2^32 elements for the aggregation. @@ -38,34 +534,407 @@ use crate::repr::Diff; /// The float accumulator performs accumulation with tolerance for floating point error. /// /// TODO(discord9): check for overflowing +#[enum_dispatch(Accumulator)] #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] pub enum Accum { /// Accumulates boolean values. - Bool { - /// The number of `true` values observed. - trues: Diff, - /// The number of `false` values observed. - falses: Diff, - }, + Bool(Bool), /// Accumulates simple numeric values. - SimpleNumber { - /// The accumulation of all non-NULL values observed. - accum: i128, - /// The number of non-NULL values observed. - non_nulls: Diff, - }, + SimpleNumber(SimpleNumber), /// Accumulates float values. - Float { - /// Accumulates non-special float values, i.e. not NaN, +inf, -inf. - /// accum will be set to zero if `non_nulls` is zero. - accum: OrderedF64, - /// Counts +inf - pos_infs: Diff, - /// Counts -inf - neg_infs: Diff, - /// Counts NaNs - nans: Diff, - /// Counts non-NULL values - non_nulls: Diff, - }, + Float(Float), + /// Accumulate Values that impl `Ord` + OrdValue(OrdValue), +} + +impl Accum { + pub fn new_accum(aggr_fn: &AggregateFunc) -> Result { + Ok(match aggr_fn { + AggregateFunc::Any + | AggregateFunc::All + | AggregateFunc::MaxBool + | AggregateFunc::MinBool => Self::from(Bool { + trues: 0, + falses: 0, + }), + AggregateFunc::SumInt16 + | AggregateFunc::SumInt32 + | AggregateFunc::SumInt64 + | AggregateFunc::SumUInt16 + | AggregateFunc::SumUInt32 + | AggregateFunc::SumUInt64 => Self::from(SimpleNumber { + accum: 0, + non_nulls: 0, + }), + AggregateFunc::SumFloat32 | AggregateFunc::SumFloat64 => Self::from(Float { + accum: OrderedF64::from(0.0), + pos_infs: 0, + neg_infs: 0, + nans: 0, + non_nulls: 0, + }), + f if f.is_max() || f.is_min() || matches!(f, AggregateFunc::Count) => { + Self::from(OrdValue { + val: None, + non_nulls: 0, + }) + } + f => { + return Err(InternalSnafu { + reason: format!( + "Accumulator does not support this aggregation function: {:?}", + f + ), + } + .build()); + } + }) + } + pub fn try_into_accum(aggr_fn: &AggregateFunc, state: Vec) -> Result { + match aggr_fn { + AggregateFunc::Any + | AggregateFunc::All + | AggregateFunc::MaxBool + | AggregateFunc::MinBool => Ok(Self::from(Bool::try_from(state)?)), + AggregateFunc::SumInt16 + | AggregateFunc::SumInt32 + | AggregateFunc::SumInt64 + | AggregateFunc::SumUInt16 + | AggregateFunc::SumUInt32 + | AggregateFunc::SumUInt64 => Ok(Self::from(SimpleNumber::try_from(state)?)), + AggregateFunc::SumFloat32 | AggregateFunc::SumFloat64 => { + Ok(Self::from(Float::try_from(state)?)) + } + f if f.is_max() || f.is_min() || matches!(f, AggregateFunc::Count) => { + Ok(Self::from(OrdValue::try_from(state)?)) + } + f => Err(InternalSnafu { + reason: format!( + "Accumulator does not support this aggregation function: {:?}", + f + ), + } + .build()), + } + } +} + +fn err_try_from_val(reason: T) -> EvalError { + TryFromValueSnafu { + msg: reason.to_string(), + } + .build() +} + +/// compare type while ignore their precision, including `TimeStamp`, `Time`, +/// `Duration`, `Interval` +fn ty_eq_without_precision(left: ConcreteDataType, right: ConcreteDataType) -> bool { + left == right + || matches!(left, ConcreteDataType::Timestamp(..)) + && matches!(right, ConcreteDataType::Timestamp(..)) + || matches!(left, ConcreteDataType::Time(..)) && matches!(right, ConcreteDataType::Time(..)) + || matches!(left, ConcreteDataType::Duration(..)) + && matches!(right, ConcreteDataType::Duration(..)) + || matches!(left, ConcreteDataType::Interval(..)) + && matches!(right, ConcreteDataType::Interval(..)) +} + +#[cfg(test)] +mod test { + use super::*; + #[test] + fn test_accum() { + let testcases = vec![ + ( + AggregateFunc::SumInt32, + vec![(Value::Int32(1), 1), (Value::Null, 1)], + ( + Value::Int64(1), + vec![Value::Decimal128(Decimal128::new(1, 38, 0)), 1i64.into()], + ), + ), + ( + AggregateFunc::SumFloat32, + vec![(Value::Float32(OrderedF32::from(1.0)), 1), (Value::Null, 1)], + ( + Value::Float32(OrderedF32::from(1.0)), + vec![ + Value::Float64(OrderedF64::from(1.0)), + 0i64.into(), + 0i64.into(), + 0i64.into(), + 1i64.into(), + ], + ), + ), + ( + AggregateFunc::MaxInt32, + vec![(Value::Int32(1), 1), (Value::Int32(2), 1), (Value::Null, 1)], + (Value::Int32(2), vec![Value::Int32(2), 2i64.into()]), + ), + ( + AggregateFunc::MinInt32, + vec![(Value::Int32(2), 1), (Value::Int32(1), 1), (Value::Null, 1)], + (Value::Int32(1), vec![Value::Int32(1), 2i64.into()]), + ), + ( + AggregateFunc::MaxFloat32, + vec![ + (Value::Float32(OrderedF32::from(1.0)), 1), + (Value::Float32(OrderedF32::from(2.0)), 1), + (Value::Null, 1), + ], + ( + Value::Float32(OrderedF32::from(2.0)), + vec![Value::Float32(OrderedF32::from(2.0)), 2i64.into()], + ), + ), + ( + AggregateFunc::MaxDateTime, + vec![ + (Value::DateTime(DateTime::from(0)), 1), + (Value::DateTime(DateTime::from(1)), 1), + (Value::Null, 1), + ], + ( + Value::DateTime(DateTime::from(1)), + vec![Value::DateTime(DateTime::from(1)), 2i64.into()], + ), + ), + ( + AggregateFunc::Count, + vec![ + (Value::Int32(1), 1), + (Value::Int32(2), 1), + (Value::Null, 1), + (Value::Null, 1), + ], + (2i64.into(), vec![Value::Null, 2i64.into()]), + ), + ( + AggregateFunc::Any, + vec![ + (Value::Boolean(false), 1), + (Value::Boolean(false), 1), + (Value::Boolean(true), 1), + (Value::Null, 1), + ], + ( + Value::Boolean(true), + vec![Value::from(1i64), Value::from(2i64)], + ), + ), + ( + AggregateFunc::All, + vec![ + (Value::Boolean(false), 1), + (Value::Boolean(false), 1), + (Value::Boolean(true), 1), + (Value::Null, 1), + ], + ( + Value::Boolean(false), + vec![Value::from(1i64), Value::from(2i64)], + ), + ), + ( + AggregateFunc::MaxBool, + vec![ + (Value::Boolean(false), 1), + (Value::Boolean(false), 1), + (Value::Boolean(true), 1), + (Value::Null, 1), + ], + ( + Value::Boolean(true), + vec![Value::from(1i64), Value::from(2i64)], + ), + ), + ( + AggregateFunc::MinBool, + vec![ + (Value::Boolean(false), 1), + (Value::Boolean(false), 1), + (Value::Boolean(true), 1), + (Value::Null, 1), + ], + ( + Value::Boolean(false), + vec![Value::from(1i64), Value::from(2i64)], + ), + ), + ]; + + for (aggr_fn, input, (eval_res, state)) in testcases { + let create_and_insert = || -> Result { + let mut acc = Accum::new_accum(&aggr_fn)?; + acc.update_batch(&aggr_fn, input.clone())?; + let row = acc.into_state(); + let acc = Accum::try_into_accum(&aggr_fn, row)?; + Ok(acc) + }; + let acc = match create_and_insert() { + Ok(acc) => acc, + Err(err) => panic!( + "Failed to create accum for {:?} with input {:?} with error: {:?}", + aggr_fn, input, err + ), + }; + + if acc.eval(&aggr_fn).unwrap() != eval_res { + panic!( + "Failed to eval accum for {:?} with input {:?}, expect {:?}, got {:?}", + aggr_fn, + input, + eval_res, + acc.eval(&aggr_fn).unwrap() + ); + } + let actual_state = acc.into_state(); + if actual_state != state { + panic!( + "Failed to cast into state from accum for {:?} with input {:?}, expect state {:?}, got state {:?}", + aggr_fn, + input, + state, + actual_state + ); + } + } + } + #[test] + fn test_fail_path_accum() { + { + let bool_accum = Bool::try_from(vec![Value::Null]); + assert!(matches!(bool_accum, Err(EvalError::Internal { .. }))); + } + + { + let mut bool_accum = Bool::try_from(vec![1i64.into(), 1i64.into()]).unwrap(); + // serde + let bool_accum_serde = serde_json::to_string(&bool_accum).unwrap(); + let bool_accum_de = serde_json::from_str::(&bool_accum_serde).unwrap(); + assert_eq!(bool_accum, bool_accum_de); + assert!(matches!( + bool_accum.update(&AggregateFunc::MaxDate, 1.into(), 1), + Err(EvalError::Internal { .. }) + )); + assert!(matches!( + bool_accum.update(&AggregateFunc::Any, 1.into(), 1), + Err(EvalError::TypeMismatch { .. }) + )); + assert!(matches!( + bool_accum.eval(&AggregateFunc::MaxDate), + Err(EvalError::Internal { .. }) + )); + } + + { + let ret = SimpleNumber::try_from(vec![Value::Null]); + assert!(matches!(ret, Err(EvalError::Internal { .. }))); + let mut accum = + SimpleNumber::try_from(vec![Decimal128::new(0, 38, 0).into(), 0i64.into()]) + .unwrap(); + + assert!(matches!( + accum.update(&AggregateFunc::All, 0.into(), 1), + Err(EvalError::Internal { .. }) + )); + assert!(matches!( + accum.update(&AggregateFunc::SumInt64, 0i32.into(), 1), + Err(EvalError::TypeMismatch { .. }) + )); + assert!(matches!( + accum.eval(&AggregateFunc::All), + Err(EvalError::Internal { .. }) + )); + accum + .update(&AggregateFunc::SumInt64, 1i64.into(), 1) + .unwrap(); + accum + .update(&AggregateFunc::SumInt64, i64::MAX.into(), 1) + .unwrap(); + assert!(matches!( + accum.eval(&AggregateFunc::SumInt64), + Err(EvalError::Overflow { .. }) + )); + } + + { + let ret = Float::try_from(vec![2f64.into(), 0i64.into(), 0i64.into(), 0i64.into()]); + assert!(matches!(ret, Err(EvalError::Internal { .. }))); + let mut accum = Float::try_from(vec![ + 2f64.into(), + 0i64.into(), + 0i64.into(), + 0i64.into(), + 1i64.into(), + ]) + .unwrap(); + accum + .update(&AggregateFunc::SumFloat64, 2f64.into(), -1) + .unwrap(); + assert!(matches!( + accum.update(&AggregateFunc::All, 0.into(), 1), + Err(EvalError::Internal { .. }) + )); + assert!(matches!( + accum.update(&AggregateFunc::SumFloat64, 0.0f32.into(), 1), + Err(EvalError::TypeMismatch { .. }) + )); + // no record, no accum + assert_eq!( + accum.eval(&AggregateFunc::SumFloat64).unwrap(), + 0.0f64.into() + ); + + assert!(matches!( + accum.eval(&AggregateFunc::All), + Err(EvalError::Internal { .. }) + )); + + accum + .update(&AggregateFunc::SumFloat64, f64::INFINITY.into(), 1) + .unwrap(); + accum + .update(&AggregateFunc::SumFloat64, (-f64::INFINITY).into(), 1) + .unwrap(); + accum + .update(&AggregateFunc::SumFloat64, f64::NAN.into(), 1) + .unwrap(); + } + + { + let ret = OrdValue::try_from(vec![Value::Null]); + assert!(matches!(ret, Err(EvalError::Internal { .. }))); + let mut accum = OrdValue::try_from(vec![Value::Null, 0i64.into()]).unwrap(); + assert!(matches!( + accum.update(&AggregateFunc::All, 0.into(), 1), + Err(EvalError::Internal { .. }) + )); + accum + .update(&AggregateFunc::MaxInt16, 1i16.into(), 1) + .unwrap(); + assert!(matches!( + accum.update(&AggregateFunc::MaxInt16, 0i32.into(), 1), + Err(EvalError::TypeMismatch { .. }) + )); + assert!(matches!( + accum.update(&AggregateFunc::MaxInt16, 0i16.into(), -1), + Err(EvalError::Internal { .. }) + )); + accum + .update(&AggregateFunc::MaxInt16, Value::Null, 1) + .unwrap(); + } + + // insert uint64 into max_int64 should fail + { + let mut accum = OrdValue::try_from(vec![Value::Null, 0i64.into()]).unwrap(); + assert!(matches!( + accum.update(&AggregateFunc::MaxInt64, 0u64.into(), 1), + Err(EvalError::TypeMismatch { .. }) + )); + } + } } diff --git a/src/flow/src/expr/relation/func.rs b/src/flow/src/expr/relation/func.rs index 4c82281736..8d28533d8c 100644 --- a/src/flow/src/expr/relation/func.rs +++ b/src/flow/src/expr/relation/func.rs @@ -12,15 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::any::type_name; - use common_time::{Date, DateTime}; use datatypes::prelude::ConcreteDataType; use datatypes::value::{OrderedF32, OrderedF64, Value}; use serde::{Deserialize, Serialize}; use crate::expr::error::{EvalError, TryFromValueSnafu, TypeMismatchSnafu}; -use crate::expr::relation::accum::Accum; +use crate::expr::relation::accum::{Accum, Accumulator}; use crate::repr::Diff; /// Aggregate functions that can be applied to a group of rows. @@ -83,3 +81,280 @@ pub enum AggregateFunc { Any, All, } + +impl AggregateFunc { + pub fn is_max(&self) -> bool { + self.signature().generic_fn == GenericFn::Max + } + + pub fn is_min(&self) -> bool { + self.signature().generic_fn == GenericFn::Min + } + + pub fn is_sum(&self) -> bool { + self.signature().generic_fn == GenericFn::Sum + } + + /// Eval value, diff with accumulator + /// + /// Expect self to be accumulable aggregate functio, i.e. sum/count + /// + /// TODO(discord9): deal with overflow&better accumulator + pub fn eval_diff_accumulable( + &self, + accum: Vec, + value_diffs: I, + ) -> Result<(Value, Vec), EvalError> + where + I: IntoIterator, + { + let mut accum = if accum.is_empty() { + Accum::new_accum(self)? + } else { + Accum::try_into_accum(self, accum)? + }; + accum.update_batch(self, value_diffs)?; + let res = accum.eval(self)?; + Ok((res, accum.into_state())) + } +} + +pub struct Signature { + pub input: ConcreteDataType, + pub output: ConcreteDataType, + pub generic_fn: GenericFn, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum GenericFn { + Max, + Min, + Sum, + Count, + Any, + All, +} + +impl AggregateFunc { + /// all concrete datatypes with precision types will be returned with largest possible variant + /// as a exception, count have a signature of `null -> i64`, but it's actually `anytype -> i64` + pub fn signature(&self) -> Signature { + match self { + AggregateFunc::MaxInt16 => Signature { + input: ConcreteDataType::int16_datatype(), + output: ConcreteDataType::int16_datatype(), + generic_fn: GenericFn::Max, + }, + AggregateFunc::MaxInt32 => Signature { + input: ConcreteDataType::int32_datatype(), + output: ConcreteDataType::int32_datatype(), + generic_fn: GenericFn::Max, + }, + AggregateFunc::MaxInt64 => Signature { + input: ConcreteDataType::int64_datatype(), + output: ConcreteDataType::int64_datatype(), + generic_fn: GenericFn::Max, + }, + AggregateFunc::MaxUInt16 => Signature { + input: ConcreteDataType::uint16_datatype(), + output: ConcreteDataType::uint16_datatype(), + generic_fn: GenericFn::Max, + }, + AggregateFunc::MaxUInt32 => Signature { + input: ConcreteDataType::uint32_datatype(), + output: ConcreteDataType::uint32_datatype(), + generic_fn: GenericFn::Max, + }, + AggregateFunc::MaxUInt64 => Signature { + input: ConcreteDataType::uint64_datatype(), + output: ConcreteDataType::uint64_datatype(), + generic_fn: GenericFn::Max, + }, + AggregateFunc::MaxFloat32 => Signature { + input: ConcreteDataType::float32_datatype(), + output: ConcreteDataType::float32_datatype(), + generic_fn: GenericFn::Max, + }, + AggregateFunc::MaxFloat64 => Signature { + input: ConcreteDataType::float64_datatype(), + output: ConcreteDataType::float64_datatype(), + generic_fn: GenericFn::Max, + }, + AggregateFunc::MaxBool => Signature { + input: ConcreteDataType::boolean_datatype(), + output: ConcreteDataType::boolean_datatype(), + generic_fn: GenericFn::Max, + }, + AggregateFunc::MaxString => Signature { + input: ConcreteDataType::string_datatype(), + output: ConcreteDataType::string_datatype(), + generic_fn: GenericFn::Max, + }, + AggregateFunc::MaxDate => Signature { + input: ConcreteDataType::date_datatype(), + output: ConcreteDataType::date_datatype(), + generic_fn: GenericFn::Max, + }, + AggregateFunc::MaxDateTime => Signature { + input: ConcreteDataType::datetime_datatype(), + output: ConcreteDataType::datetime_datatype(), + generic_fn: GenericFn::Max, + }, + AggregateFunc::MaxTimestamp => Signature { + input: ConcreteDataType::timestamp_second_datatype(), + output: ConcreteDataType::timestamp_second_datatype(), + generic_fn: GenericFn::Max, + }, + AggregateFunc::MaxTime => Signature { + input: ConcreteDataType::time_second_datatype(), + output: ConcreteDataType::time_second_datatype(), + generic_fn: GenericFn::Max, + }, + AggregateFunc::MaxDuration => Signature { + input: ConcreteDataType::duration_second_datatype(), + output: ConcreteDataType::duration_second_datatype(), + generic_fn: GenericFn::Max, + }, + AggregateFunc::MaxInterval => Signature { + input: ConcreteDataType::interval_year_month_datatype(), + output: ConcreteDataType::interval_year_month_datatype(), + generic_fn: GenericFn::Max, + }, + AggregateFunc::MinInt16 => Signature { + input: ConcreteDataType::int16_datatype(), + output: ConcreteDataType::int16_datatype(), + generic_fn: GenericFn::Min, + }, + AggregateFunc::MinInt32 => Signature { + input: ConcreteDataType::int32_datatype(), + output: ConcreteDataType::int32_datatype(), + generic_fn: GenericFn::Min, + }, + AggregateFunc::MinInt64 => Signature { + input: ConcreteDataType::int64_datatype(), + output: ConcreteDataType::int64_datatype(), + generic_fn: GenericFn::Min, + }, + AggregateFunc::MinUInt16 => Signature { + input: ConcreteDataType::uint16_datatype(), + output: ConcreteDataType::uint16_datatype(), + generic_fn: GenericFn::Min, + }, + AggregateFunc::MinUInt32 => Signature { + input: ConcreteDataType::uint32_datatype(), + output: ConcreteDataType::uint32_datatype(), + generic_fn: GenericFn::Min, + }, + AggregateFunc::MinUInt64 => Signature { + input: ConcreteDataType::uint64_datatype(), + output: ConcreteDataType::uint64_datatype(), + generic_fn: GenericFn::Min, + }, + AggregateFunc::MinFloat32 => Signature { + input: ConcreteDataType::float32_datatype(), + output: ConcreteDataType::float32_datatype(), + generic_fn: GenericFn::Min, + }, + AggregateFunc::MinFloat64 => Signature { + input: ConcreteDataType::float64_datatype(), + output: ConcreteDataType::float64_datatype(), + generic_fn: GenericFn::Min, + }, + AggregateFunc::MinBool => Signature { + input: ConcreteDataType::boolean_datatype(), + output: ConcreteDataType::boolean_datatype(), + generic_fn: GenericFn::Min, + }, + AggregateFunc::MinString => Signature { + input: ConcreteDataType::string_datatype(), + output: ConcreteDataType::string_datatype(), + generic_fn: GenericFn::Min, + }, + AggregateFunc::MinDate => Signature { + input: ConcreteDataType::date_datatype(), + output: ConcreteDataType::date_datatype(), + generic_fn: GenericFn::Min, + }, + AggregateFunc::MinDateTime => Signature { + input: ConcreteDataType::datetime_datatype(), + output: ConcreteDataType::datetime_datatype(), + generic_fn: GenericFn::Min, + }, + AggregateFunc::MinTimestamp => Signature { + input: ConcreteDataType::timestamp_second_datatype(), + output: ConcreteDataType::timestamp_second_datatype(), + generic_fn: GenericFn::Min, + }, + AggregateFunc::MinTime => Signature { + input: ConcreteDataType::time_second_datatype(), + output: ConcreteDataType::time_second_datatype(), + generic_fn: GenericFn::Min, + }, + AggregateFunc::MinDuration => Signature { + input: ConcreteDataType::duration_second_datatype(), + output: ConcreteDataType::duration_second_datatype(), + generic_fn: GenericFn::Min, + }, + AggregateFunc::MinInterval => Signature { + input: ConcreteDataType::interval_year_month_datatype(), + output: ConcreteDataType::interval_year_month_datatype(), + generic_fn: GenericFn::Min, + }, + AggregateFunc::SumInt16 => Signature { + input: ConcreteDataType::int16_datatype(), + output: ConcreteDataType::int16_datatype(), + generic_fn: GenericFn::Sum, + }, + AggregateFunc::SumInt32 => Signature { + input: ConcreteDataType::int32_datatype(), + output: ConcreteDataType::int32_datatype(), + generic_fn: GenericFn::Sum, + }, + AggregateFunc::SumInt64 => Signature { + input: ConcreteDataType::int64_datatype(), + output: ConcreteDataType::int64_datatype(), + generic_fn: GenericFn::Sum, + }, + AggregateFunc::SumUInt16 => Signature { + input: ConcreteDataType::uint16_datatype(), + output: ConcreteDataType::uint16_datatype(), + generic_fn: GenericFn::Sum, + }, + AggregateFunc::SumUInt32 => Signature { + input: ConcreteDataType::uint32_datatype(), + output: ConcreteDataType::uint32_datatype(), + generic_fn: GenericFn::Sum, + }, + AggregateFunc::SumUInt64 => Signature { + input: ConcreteDataType::uint64_datatype(), + output: ConcreteDataType::uint64_datatype(), + generic_fn: GenericFn::Sum, + }, + AggregateFunc::SumFloat32 => Signature { + input: ConcreteDataType::float32_datatype(), + output: ConcreteDataType::float32_datatype(), + generic_fn: GenericFn::Sum, + }, + AggregateFunc::SumFloat64 => Signature { + input: ConcreteDataType::float64_datatype(), + output: ConcreteDataType::float64_datatype(), + generic_fn: GenericFn::Sum, + }, + AggregateFunc::Count => Signature { + input: ConcreteDataType::null_datatype(), + output: ConcreteDataType::int64_datatype(), + generic_fn: GenericFn::Count, + }, + AggregateFunc::Any => Signature { + input: ConcreteDataType::boolean_datatype(), + output: ConcreteDataType::boolean_datatype(), + generic_fn: GenericFn::Any, + }, + AggregateFunc::All => Signature { + input: ConcreteDataType::boolean_datatype(), + output: ConcreteDataType::boolean_datatype(), + generic_fn: GenericFn::All, + }, + } + } +} diff --git a/src/flow/src/repr.rs b/src/flow/src/repr.rs index 8c15483666..1a4547b871 100644 --- a/src/flow/src/repr.rs +++ b/src/flow/src/repr.rs @@ -33,7 +33,10 @@ use snafu::ResultExt; use crate::expr::error::{CastValueSnafu, EvalError}; -/// System-wide Record count difference type. +/// System-wide Record count difference type. Useful for capture data change +/// +/// i.e. +1 means insert one record, -1 means remove, +/// and +/-n means insert/remove multiple duplicate records. pub type Diff = i64; /// System-wide default timestamp type From 9aa8f756abc51b62322fde08c135495e056c00eb Mon Sep 17 00:00:00 2001 From: Yingwen Date: Tue, 12 Mar 2024 15:03:52 +0800 Subject: [PATCH 09/17] fix: allow passing extra table options (#3484) * fix: do not check options in parser * test: fix tests * test: fix sqlness * test: add sqlness test * chore: log options * chore: must specify compaction type * feat: validate option key * feat: add option key validation back --- src/common/datasource/src/object_store/s3.rs | 15 ++-- src/mito2/src/region/opener.rs | 5 ++ src/mito2/src/region/options.rs | 5 +- src/sql/src/error.rs | 2 +- src/sql/src/parsers/create_parser.rs | 17 +++- src/sql/src/statements/create.rs | 28 ++++++- src/store-api/src/lib.rs | 1 + src/store-api/src/mito_engine_options.rs | 61 ++++++++++++++ src/table/src/requests.rs | 59 ++++++++----- tests-integration/src/tests/instance_test.rs | 2 +- .../common/create/create_with_options.result | 84 +++++++++++++++++++ .../common/create/create_with_options.sql | 69 +++++++++++++++ .../standalone/common/show/show_create.result | 23 ----- .../standalone/common/show/show_create.sql | 20 ----- 14 files changed, 310 insertions(+), 81 deletions(-) create mode 100644 src/store-api/src/mito_engine_options.rs create mode 100644 tests/cases/standalone/common/create/create_with_options.result create mode 100644 tests/cases/standalone/common/create/create_with_options.sql diff --git a/src/common/datasource/src/object_store/s3.rs b/src/common/datasource/src/object_store/s3.rs index 577a33eb68..2b6ac7c2ee 100644 --- a/src/common/datasource/src/object_store/s3.rs +++ b/src/common/datasource/src/object_store/s3.rs @@ -28,12 +28,15 @@ const REGION: &str = "region"; const ENABLE_VIRTUAL_HOST_STYLE: &str = "enable_virtual_host_style"; pub fn is_supported_in_s3(key: &str) -> bool { - key == ENDPOINT - || key == ACCESS_KEY_ID - || key == SECRET_ACCESS_KEY - || key == SESSION_TOKEN - || key == REGION - || key == ENABLE_VIRTUAL_HOST_STYLE + [ + ENDPOINT, + ACCESS_KEY_ID, + SECRET_ACCESS_KEY, + SESSION_TOKEN, + REGION, + ENABLE_VIRTUAL_HOST_STYLE, + ] + .contains(&key) } pub fn build_s3_backend( diff --git a/src/mito2/src/region/opener.rs b/src/mito2/src/region/opener.rs index 3b5ba500bd..75d0b9dcfb 100644 --- a/src/mito2/src/region/opener.rs +++ b/src/mito2/src/region/opener.rs @@ -171,6 +171,8 @@ impl RegionOpener { // Initial memtable id is 0. let mutable = self.memtable_builder.build(0, &metadata); + debug!("Create region {} with options: {:?}", region_id, options); + let version = VersionBuilder::new(metadata, mutable) .options(options) .build(); @@ -249,6 +251,9 @@ impl RegionOpener { let region_id = self.region_id; let object_store = self.object_store(®ion_options.storage)?.clone(); + + debug!("Open region {} with options: {:?}", region_id, self.options); + let access_layer = Arc::new(AccessLayer::new( self.region_dir.clone(), object_store, diff --git a/src/mito2/src/region/options.rs b/src/mito2/src/region/options.rs index d9205eea6c..e69c7193ff 100644 --- a/src/mito2/src/region/options.rs +++ b/src/mito2/src/region/options.rs @@ -13,6 +13,8 @@ // limitations under the License. //! Options for a region. +//! +//! If we add options in this mod, we also need to modify [store_api::mito_engine_options]. use std::collections::HashMap; use std::time::Duration; @@ -358,6 +360,7 @@ mod tests { ("compaction.type", "twcs"), ("storage", "S3"), ("index.inverted_index.ignore_column_ids", "1,2,3"), + ("index.inverted_index.segment_row_count", "512"), ( WAL_OPTIONS_KEY, &serde_json::to_string(&wal_options).unwrap(), @@ -376,7 +379,7 @@ mod tests { index_options: IndexOptions { inverted_index: InvertedIndexOptions { ignore_column_ids: vec![1, 2, 3], - segment_row_count: 1024, + segment_row_count: 512, }, }, }; diff --git a/src/sql/src/error.rs b/src/sql/src/error.rs index 50df979e80..24643b4a65 100644 --- a/src/sql/src/error.rs +++ b/src/sql/src/error.rs @@ -117,7 +117,7 @@ pub enum Error { source: datatypes::error::Error, }, - #[snafu(display("Invalid table option key: {}", key))] + #[snafu(display("Unrecognized table option key: {}", key))] InvalidTableOption { key: String, location: Location }, #[snafu(display("Failed to serialize column default constraint"))] diff --git a/src/sql/src/parsers/create_parser.rs b/src/sql/src/parsers/create_parser.rs index 606ee3ebef..b154ee48ab 100644 --- a/src/sql/src/parsers/create_parser.rs +++ b/src/sql/src/parsers/create_parser.rs @@ -23,7 +23,7 @@ use sqlparser::keywords::ALL_KEYWORDS; use sqlparser::parser::IsOptional::Mandatory; use sqlparser::parser::{Parser, ParserError}; use sqlparser::tokenizer::{Token, TokenWithLocation, Word}; -use table::requests::valid_table_option; +use table::requests::validate_table_option; use crate::ast::{ColumnDef, Ident, TableConstraint}; use crate::error::{ @@ -84,7 +84,7 @@ impl<'a> ParserContext<'a> { .collect::>(); for key in options.keys() { ensure!( - valid_table_option(key), + validate_table_option(key), InvalidTableOptionSnafu { key: key.to_string() } @@ -150,7 +150,7 @@ impl<'a> ParserContext<'a> { .context(error::SyntaxSnafu)?; for option in options.iter() { ensure!( - valid_table_option(&option.name.value), + validate_table_option(&option.name.value), InvalidTableOptionSnafu { key: option.name.value.to_string() } @@ -799,6 +799,17 @@ mod tests { expected_engine: "foo", expected_if_not_exist: true, }, + Test { + sql: "CREATE EXTERNAL TABLE IF NOT EXISTS city ENGINE=foo with(location='/var/data/city.csv',format='csv','compaction.type'='bar');", + expected_table_name: "city", + expected_options: HashMap::from([ + ("location".to_string(), "/var/data/city.csv".to_string()), + ("format".to_string(), "csv".to_string()), + ("compaction.type".to_string(), "bar".to_string()), + ]), + expected_engine: "foo", + expected_if_not_exist: true, + }, ]; for test in tests { diff --git a/src/sql/src/statements/create.rs b/src/sql/src/statements/create.rs index 5cc6b4ccca..8bbf64d86e 100644 --- a/src/sql/src/statements/create.rs +++ b/src/sql/src/statements/create.rs @@ -230,7 +230,7 @@ pub struct CreateTableLike { #[cfg(test)] mod tests { use crate::dialect::GreptimeDbDialect; - use crate::error::Error::InvalidTableOption; + use crate::error::Error; use crate::parser::{ParseOptions, ParserContext}; use crate::statements::statement::Statement; @@ -344,7 +344,29 @@ ENGINE=mito fn test_validate_table_options() { let sql = r"create table if not exists demo( host string, - ts bigint, + ts timestamp, + cpu double default 0, + memory double, + TIME INDEX (ts), + PRIMARY KEY(host) + ) + PARTITION ON COLUMNS (host) () + engine=mito + with(regions=1, ttl='7d', 'compaction.type'='world'); +"; + let result = + ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()) + .unwrap(); + match &result[0] { + Statement::CreateTable(c) => { + assert_eq!(3, c.options.len()); + } + _ => unreachable!(), + } + + let sql = r"create table if not exists demo( + host string, + ts timestamp, cpu double default 0, memory double, TIME INDEX (ts), @@ -356,6 +378,6 @@ ENGINE=mito "; let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); - assert!(matches!(result, Err(InvalidTableOption { .. }))) + assert!(matches!(result, Err(Error::InvalidTableOption { .. }))); } } diff --git a/src/store-api/src/lib.rs b/src/store-api/src/lib.rs index ffda779d33..95ea196f71 100644 --- a/src/store-api/src/lib.rs +++ b/src/store-api/src/lib.rs @@ -20,6 +20,7 @@ pub mod logstore; pub mod manifest; pub mod metadata; pub mod metric_engine_consts; +pub mod mito_engine_options; pub mod path_utils; pub mod region_engine; pub mod region_request; diff --git a/src/store-api/src/mito_engine_options.rs b/src/store-api/src/mito_engine_options.rs new file mode 100644 index 0000000000..7ef963a4c0 --- /dev/null +++ b/src/store-api/src/mito_engine_options.rs @@ -0,0 +1,61 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Option keys for the mito engine. +//! We define them in this mod so the create parser can use it to validate table options. + +use common_wal::options::WAL_OPTIONS_KEY; + +/// Returns true if the `key` is a valid option key for the mito engine. +pub fn is_mito_engine_option_key(key: &str) -> bool { + [ + "ttl", + "compaction.type", + "compaction.twcs.max_active_window_files", + "compaction.twcs.max_inactive_window_files", + "compaction.twcs.time_window", + "storage", + "index.inverted_index.ignore_column_ids", + "index.inverted_index.segment_row_count", + WAL_OPTIONS_KEY, + ] + .contains(&key) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_mito_engine_option_key() { + assert!(is_mito_engine_option_key("ttl")); + assert!(is_mito_engine_option_key("compaction.type")); + assert!(is_mito_engine_option_key( + "compaction.twcs.max_active_window_files" + )); + assert!(is_mito_engine_option_key( + "compaction.twcs.max_inactive_window_files" + )); + assert!(is_mito_engine_option_key("compaction.twcs.time_window")); + assert!(is_mito_engine_option_key("storage")); + assert!(is_mito_engine_option_key( + "index.inverted_index.ignore_column_ids" + )); + assert!(is_mito_engine_option_key( + "index.inverted_index.segment_row_count" + )); + assert!(is_mito_engine_option_key("wal_options")); + assert!(!is_mito_engine_option_key("foo")); + } +} diff --git a/src/table/src/requests.rs b/src/table/src/requests.rs index bdd40a7282..21541218ff 100644 --- a/src/table/src/requests.rs +++ b/src/table/src/requests.rs @@ -26,6 +26,7 @@ use datatypes::prelude::VectorRef; use datatypes::schema::{ColumnSchema, RawSchema}; use serde::{Deserialize, Serialize}; use store_api::metric_engine_consts::{LOGICAL_TABLE_METADATA_KEY, PHYSICAL_TABLE_METADATA_KEY}; +use store_api::mito_engine_options::is_mito_engine_option_key; use store_api::storage::RegionNumber; use crate::error; @@ -38,6 +39,33 @@ pub const FILE_TABLE_LOCATION_KEY: &str = "location"; pub const FILE_TABLE_PATTERN_KEY: &str = "pattern"; pub const FILE_TABLE_FORMAT_KEY: &str = "format"; +/// Returns true if the `key` is a valid key for any engine or storage. +pub fn validate_table_option(key: &str) -> bool { + if is_supported_in_s3(key) { + return true; + } + + if is_mito_engine_option_key(key) { + return true; + } + + [ + // common keys: + WRITE_BUFFER_SIZE_KEY, + TTL_KEY, + REGIONS_KEY, + STORAGE_KEY, + // file engine keys: + FILE_TABLE_LOCATION_KEY, + FILE_TABLE_FORMAT_KEY, + FILE_TABLE_PATTERN_KEY, + // metric engine keys: + PHYSICAL_TABLE_METADATA_KEY, + LOGICAL_TABLE_METADATA_KEY, + ] + .contains(&key) +} + #[derive(Debug, Clone)] pub struct CreateDatabaseRequest { pub db_name: String, @@ -315,21 +343,6 @@ impl TruncateTableRequest { } } -pub fn valid_table_option(key: &str) -> bool { - matches!( - key, - FILE_TABLE_LOCATION_KEY - | FILE_TABLE_FORMAT_KEY - | FILE_TABLE_PATTERN_KEY - | WRITE_BUFFER_SIZE_KEY - | TTL_KEY - | REGIONS_KEY - | STORAGE_KEY - | PHYSICAL_TABLE_METADATA_KEY - | LOGICAL_TABLE_METADATA_KEY - ) | is_supported_in_s3(key) -} - #[derive(Debug, Clone, Default, Deserialize, Serialize)] pub struct CopyDatabaseRequest { pub catalog_name: String, @@ -346,14 +359,14 @@ mod tests { #[test] fn test_validate_table_option() { - assert!(valid_table_option(FILE_TABLE_LOCATION_KEY)); - assert!(valid_table_option(FILE_TABLE_FORMAT_KEY)); - assert!(valid_table_option(FILE_TABLE_PATTERN_KEY)); - assert!(valid_table_option(TTL_KEY)); - assert!(valid_table_option(REGIONS_KEY)); - assert!(valid_table_option(WRITE_BUFFER_SIZE_KEY)); - assert!(valid_table_option(STORAGE_KEY)); - assert!(!valid_table_option("foo")); + assert!(validate_table_option(FILE_TABLE_LOCATION_KEY)); + assert!(validate_table_option(FILE_TABLE_FORMAT_KEY)); + assert!(validate_table_option(FILE_TABLE_PATTERN_KEY)); + assert!(validate_table_option(TTL_KEY)); + assert!(validate_table_option(REGIONS_KEY)); + assert!(validate_table_option(WRITE_BUFFER_SIZE_KEY)); + assert!(validate_table_option(STORAGE_KEY)); + assert!(!validate_table_option("foo")); } #[test] diff --git a/tests-integration/src/tests/instance_test.rs b/tests-integration/src/tests/instance_test.rs index 7bede82e30..5084b5b015 100644 --- a/tests-integration/src/tests/instance_test.rs +++ b/tests-integration/src/tests/instance_test.rs @@ -160,7 +160,7 @@ PARTITION ON COLUMNS (n) ( } #[apply(both_instances_cases)] -async fn test_validate_external_table_options(instance: Arc) { +async fn test_extra_external_table_options(instance: Arc) { let frontend = instance.frontend(); let format = "json"; let location = find_testing_resource("/tests/data/json/various_type.json"); diff --git a/tests/cases/standalone/common/create/create_with_options.result b/tests/cases/standalone/common/create/create_with_options.result new file mode 100644 index 0000000000..e50746d127 --- /dev/null +++ b/tests/cases/standalone/common/create/create_with_options.result @@ -0,0 +1,84 @@ +CREATE TABLE not_supported_table_options_keys ( + id INT UNSIGNED, + host STRING, + cpu DOUBLE, + disk FLOAT, + ts TIMESTAMP NOT NULL DEFAULT current_timestamp(), + TIME INDEX (ts), + PRIMARY KEY (id, host) +) +PARTITION ON COLUMNS (id) ( + id < 5, + id >= 5 AND id < 9, + id >= 9 +) +ENGINE=mito +WITH( + foo = 123, + ttl = '7d', + write_buffer_size = 1024 +); + +Error: 1004(InvalidArguments), Unrecognized table option key: foo + +create table if not exists test_opts( + host string, + ts timestamp, + cpu double default 0, + memory double, + TIME INDEX (ts), + PRIMARY KEY(host) +) +engine=mito +with(regions=1, ttl='7d', 'compaction.type'='twcs', 'compaction.twcs.time_window'='1d'); + +Affected Rows: 0 + +drop table test_opts; + +Affected Rows: 0 + +create table if not exists test_opts( + host string, + ts timestamp, + cpu double default 0, + memory double, + TIME INDEX (ts), + PRIMARY KEY(host) +) +engine=mito +with('regions'=1, 'ttl'='7d', 'compaction.type'='twcs', 'compaction.twcs.time_window'='1d'); + +Affected Rows: 0 + +drop table test_opts; + +Affected Rows: 0 + +create table if not exists test_mito_options( + host string, + ts timestamp, + cpu double default 0, + memory double, + TIME INDEX (ts), + PRIMARY KEY(host) +) +engine=mito +with( + 'regions'=1, + 'ttl'='7d', + 'compaction.type'='twcs', + 'compaction.twcs.max_active_window_files'='8', + 'compaction.twcs.max_inactive_window_files'='2', + 'compaction.twcs.time_window'='1d', + 'index.inverted_index.ignore_column_ids'='1,2,3', + 'index.inverted_index.segment_row_count'='512', + 'wal_options'='{"wal.provider":"raft_engine"}', +); + +Affected Rows: 0 + +drop table test_mito_options; + +Affected Rows: 0 + diff --git a/tests/cases/standalone/common/create/create_with_options.sql b/tests/cases/standalone/common/create/create_with_options.sql new file mode 100644 index 0000000000..094d905eda --- /dev/null +++ b/tests/cases/standalone/common/create/create_with_options.sql @@ -0,0 +1,69 @@ +CREATE TABLE not_supported_table_options_keys ( + id INT UNSIGNED, + host STRING, + cpu DOUBLE, + disk FLOAT, + ts TIMESTAMP NOT NULL DEFAULT current_timestamp(), + TIME INDEX (ts), + PRIMARY KEY (id, host) +) +PARTITION ON COLUMNS (id) ( + id < 5, + id >= 5 AND id < 9, + id >= 9 +) +ENGINE=mito +WITH( + foo = 123, + ttl = '7d', + write_buffer_size = 1024 +); + +create table if not exists test_opts( + host string, + ts timestamp, + cpu double default 0, + memory double, + TIME INDEX (ts), + PRIMARY KEY(host) +) +engine=mito +with(regions=1, ttl='7d', 'compaction.type'='twcs', 'compaction.twcs.time_window'='1d'); + +drop table test_opts; + +create table if not exists test_opts( + host string, + ts timestamp, + cpu double default 0, + memory double, + TIME INDEX (ts), + PRIMARY KEY(host) +) +engine=mito +with('regions'=1, 'ttl'='7d', 'compaction.type'='twcs', 'compaction.twcs.time_window'='1d'); + +drop table test_opts; + +create table if not exists test_mito_options( + host string, + ts timestamp, + cpu double default 0, + memory double, + TIME INDEX (ts), + PRIMARY KEY(host) +) +engine=mito +with( + 'regions'=1, + 'ttl'='7d', + 'compaction.type'='twcs', + 'compaction.twcs.max_active_window_files'='8', + 'compaction.twcs.max_inactive_window_files'='2', + 'compaction.twcs.time_window'='1d', + 'index.inverted_index.ignore_column_ids'='1,2,3', + 'index.inverted_index.segment_row_count'='512', + 'wal_options'='{"wal.provider":"raft_engine"}', +); + +drop table test_mito_options; diff --git a/tests/cases/standalone/common/show/show_create.result b/tests/cases/standalone/common/show/show_create.result index dbc714849b..b016174a38 100644 --- a/tests/cases/standalone/common/show/show_create.result +++ b/tests/cases/standalone/common/show/show_create.result @@ -77,29 +77,6 @@ drop table table_without_partition; Affected Rows: 0 -CREATE TABLE not_supported_table_options_keys ( - id INT UNSIGNED, - host STRING, - cpu DOUBLE, - disk FLOAT, - ts TIMESTAMP NOT NULL DEFAULT current_timestamp(), - TIME INDEX (ts), - PRIMARY KEY (id, host) -) -PARTITION ON COLUMNS (id) ( - id < 5, - id >= 5 AND id < 9, - id >= 9 -) -ENGINE=mito -WITH( - foo = 123, - ttl = '7d', - write_buffer_size = 1024 -); - -Error: 1004(InvalidArguments), Invalid table option key: foo - CREATE TABLE not_supported_table_storage_option ( id INT UNSIGNED, host STRING, diff --git a/tests/cases/standalone/common/show/show_create.sql b/tests/cases/standalone/common/show/show_create.sql index 890ac7329d..ccc3333f51 100644 --- a/tests/cases/standalone/common/show/show_create.sql +++ b/tests/cases/standalone/common/show/show_create.sql @@ -30,26 +30,6 @@ show create table table_without_partition; drop table table_without_partition; -CREATE TABLE not_supported_table_options_keys ( - id INT UNSIGNED, - host STRING, - cpu DOUBLE, - disk FLOAT, - ts TIMESTAMP NOT NULL DEFAULT current_timestamp(), - TIME INDEX (ts), - PRIMARY KEY (id, host) -) -PARTITION ON COLUMNS (id) ( - id < 5, - id >= 5 AND id < 9, - id >= 9 -) -ENGINE=mito -WITH( - foo = 123, - ttl = '7d', - write_buffer_size = 1024 -); CREATE TABLE not_supported_table_storage_option ( id INT UNSIGNED, host STRING, From 58bd065c6b2028024e2941262adf5dd71981901e Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Tue, 12 Mar 2024 18:59:07 +0800 Subject: [PATCH 10/17] feat(flow): plan def (#3490) * feat: plan def * chore: add license * docs: remove TODO done * chore: add derive Ord --- src/flow/src/expr.rs | 1 + src/flow/src/expr/linear.rs | 4 +- src/flow/src/expr/relation.rs | 2 +- src/flow/src/lib.rs | 1 + src/flow/src/plan.rs | 98 +++++++++++++++++++++++++++++++++++ src/flow/src/plan/join.rs | 78 ++++++++++++++++++++++++++++ src/flow/src/plan/reduce.rs | 50 ++++++++++++++++++ 7 files changed, 231 insertions(+), 3 deletions(-) create mode 100644 src/flow/src/plan.rs create mode 100644 src/flow/src/plan/join.rs create mode 100644 src/flow/src/plan/reduce.rs diff --git a/src/flow/src/expr.rs b/src/flow/src/expr.rs index 043c37b2be..d54dfa4b9f 100644 --- a/src/flow/src/expr.rs +++ b/src/flow/src/expr.rs @@ -24,5 +24,6 @@ mod scalar; pub(crate) use error::{EvalError, InvalidArgumentSnafu, OptimizeSnafu}; pub(crate) use func::{BinaryFunc, UnaryFunc, UnmaterializableFunc, VariadicFunc}; pub(crate) use id::{GlobalId, Id, LocalId}; +pub(crate) use linear::{MapFilterProject, MfpPlan, SafeMfpPlan}; pub(crate) use relation::{AggregateExpr, AggregateFunc}; pub(crate) use scalar::ScalarExpr; diff --git a/src/flow/src/expr/linear.rs b/src/flow/src/expr/linear.rs index 331b883996..d4a0ef5eda 100644 --- a/src/flow/src/expr/linear.rs +++ b/src/flow/src/expr/linear.rs @@ -45,7 +45,7 @@ use crate::repr::{self, value_to_internal_ts, Diff, Row}; /// expressions in `self.expressions`, even though this is not something /// we can directly evaluate. The plan creation methods will defensively /// ensure that the right thing happens. -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] pub struct MapFilterProject { /// A sequence of expressions that should be appended to the row. /// @@ -415,7 +415,7 @@ impl MapFilterProject { } /// A wrapper type which indicates it is safe to simply evaluate all expressions. -#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq)] +#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd)] pub struct SafeMfpPlan { pub(crate) mfp: MapFilterProject, } diff --git a/src/flow/src/expr/relation.rs b/src/flow/src/expr/relation.rs index 520c858534..db82c75425 100644 --- a/src/flow/src/expr/relation.rs +++ b/src/flow/src/expr/relation.rs @@ -21,7 +21,7 @@ mod accum; mod func; /// Describes an aggregation expression. -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] pub struct AggregateExpr { /// Names the aggregation function. pub func: AggregateFunc, diff --git a/src/flow/src/lib.rs b/src/flow/src/lib.rs index c144f8ab50..ef7419888e 100644 --- a/src/flow/src/lib.rs +++ b/src/flow/src/lib.rs @@ -17,4 +17,5 @@ // allow unused for now because it should be use later mod adapter; mod expr; +mod plan; mod repr; diff --git a/src/flow/src/plan.rs b/src/flow/src/plan.rs new file mode 100644 index 0000000000..77b8ffb1f6 --- /dev/null +++ b/src/flow/src/plan.rs @@ -0,0 +1,98 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This module contain basic definition for dataflow's plan +//! that can be translate to hydro dataflow + +mod join; +mod reduce; + +use serde::{Deserialize, Serialize}; + +pub(crate) use self::reduce::{AccumulablePlan, KeyValPlan, ReducePlan}; +use crate::expr::{ + AggregateExpr, EvalError, Id, LocalId, MapFilterProject, SafeMfpPlan, ScalarExpr, +}; +use crate::plan::join::JoinPlan; +use crate::repr::{DiffRow, RelationType}; + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Deserialize, Serialize)] +pub struct TypedPlan { + /// output type of the relation + pub typ: RelationType, + pub plan: Plan, +} + +/// TODO(discord9): support `TableFunc`（by define FlatMap that map 1 to n) +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Deserialize, Serialize)] +pub enum Plan { + /// A constant collection of rows. + Constant { rows: Vec }, + /// Get CDC data from an source, be it external reference to an existing source or an internal + /// reference to a `Let` identifier + Get { id: Id }, + /// Create a temporary collection from given `value``, and make this bind only available + /// in scope of `body` + Let { + id: LocalId, + value: Box, + body: Box, + }, + /// Map, Filter, and Project operators. + Mfp { + /// The input collection. + input: Box, + /// Linear operator to apply to each record. + mfp: MapFilterProject, + }, + /// Reduce operator, aggregation by key assembled from KeyValPlan + Reduce { + /// The input collection. + input: Box, + /// A plan for changing input records into key, value pairs. + key_val_plan: KeyValPlan, + /// A plan for performing the reduce. + /// + /// The implementation of reduction has several different strategies based + /// on the properties of the reduction, and the input itself. + reduce_plan: ReducePlan, + }, + /// A multiway relational equijoin, with fused map, filter, and projection. + /// + /// This stage performs a multiway join among `inputs`, using the equality + /// constraints expressed in `plan`. The plan also describes the implementation + /// strategy we will use, and any pushed down per-record work. + Join { + /// An ordered list of inputs that will be joined. + inputs: Vec, + /// Detailed information about the implementation of the join. + /// + /// This includes information about the implementation strategy, but also + /// any map, filter, project work that we might follow the join with, but + /// potentially pushed down into the implementation of the join. + plan: JoinPlan, + }, + /// Adds the contents of the input collections. + /// + /// Importantly, this is *multiset* union, so the multiplicities of records will + /// add. This is in contrast to *set* union, where the multiplicities would be + /// capped at one. A set union can be formed with `Union` followed by `Reduce` + /// implementing the "distinct" operator. + Union { + /// The input collections + inputs: Vec, + /// Whether to consolidate the output, e.g., cancel negated records. + consolidate_output: bool, + }, +} diff --git a/src/flow/src/plan/join.rs b/src/flow/src/plan/join.rs new file mode 100644 index 0000000000..13bb95f511 --- /dev/null +++ b/src/flow/src/plan/join.rs @@ -0,0 +1,78 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use serde::{Deserialize, Serialize}; + +use crate::expr::ScalarExpr; +use crate::plan::SafeMfpPlan; + +/// TODO(discord9): consider impl more join strategies +#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd)] +pub enum JoinPlan { + Linear(LinearJoinPlan), +} + +/// Determine if a given row should stay in the output. And apply a map filter project before output the row +#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd)] +pub struct JoinFilter { + /// each element in the outer vector will check if each expr in itself can be eval to same value + /// if not, the row will be filtered out. Useful for equi-join(join based on equality of some columns) + pub ready_equivalences: Vec>, + /// Apply a map filter project before output the row + pub before: SafeMfpPlan, +} + +/// A plan for the execution of a linear join. +/// +/// A linear join is a sequence of stages, each of which introduces +/// a new collection. Each stage is represented by a [LinearStagePlan]. +#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd)] +pub struct LinearJoinPlan { + /// The source relation from which we start the join. + pub source_relation: usize, + /// The arrangement to use for the source relation, if any + pub source_key: Option>, + /// An initial closure to apply before any stages. + /// + /// Values of `None` indicate the identity closure. + pub initial_closure: Option, + /// A *sequence* of stages to apply one after the other. + pub stage_plans: Vec, + /// A concluding filter to apply after the last stage. + /// + /// Values of `None` indicate the identity closure. + pub final_closure: Option, +} + +/// A plan for the execution of one stage of a linear join. +/// +/// Each stage is a binary join between the current accumulated +/// join results, and a new collection. The former is referred to +/// as the "stream" and the latter the "lookup". +#[derive(Clone, Debug, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd)] +pub struct LinearStagePlan { + /// The index of the relation into which we will look up. + pub lookup_relation: usize, + /// The key expressions to use for the stream relation. + pub stream_key: Vec, + /// Columns to retain from the stream relation. + /// These columns are those that are not redundant with `stream_key`, + /// and cannot be read out of the key component of an arrangement. + pub stream_thinning: Vec, + /// The key expressions to use for the lookup relation. + pub lookup_key: Vec, + /// The closure to apply to the concatenation of the key columns, + /// the stream value columns, and the lookup value colunms. + pub closure: JoinFilter, +} diff --git a/src/flow/src/plan/reduce.rs b/src/flow/src/plan/reduce.rs new file mode 100644 index 0000000000..52dd3a509d --- /dev/null +++ b/src/flow/src/plan/reduce.rs @@ -0,0 +1,50 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use serde::{Deserialize, Serialize}; + +use crate::expr::{AggregateExpr, Id, LocalId, MapFilterProject, SafeMfpPlan, ScalarExpr}; + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Deserialize, Serialize)] +pub struct KeyValPlan { + pub key_plan: SafeMfpPlan, + pub val_plan: SafeMfpPlan, +} + +/// TODO(discord9): def&impl of Hierarchical aggregates(for min/max with support to deletion) and +/// basic aggregates(for other aggregate functions) and mixed aggregate +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Deserialize, Serialize)] +pub enum ReducePlan { + /// Plan for not computing any aggregations, just determining the set of + /// distinct keys. + Distinct, + /// Plan for computing only accumulable aggregations. + /// Including simple functions like `sum`, `count`, `min/max`(without deletion) + Accumulable(AccumulablePlan), +} + +/// Accumulable plan for the execution of a reduction. +#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Deserialize, Serialize)] +pub struct AccumulablePlan { + /// All of the aggregations we were asked to compute, stored + /// in order. + pub full_aggrs: Vec, + /// All of the non-distinct accumulable aggregates. + /// Each element represents: + /// (index of aggr output, index of value among inputs, aggr expr) + /// These will all be rendered together in one dataflow fragment. + pub simple_aggrs: Vec<(usize, usize, AggregateExpr)>, + /// Same as above but for all of the `DISTINCT` accumulable aggregations. + pub distinct_aggrs: Vec<(usize, usize, AggregateExpr)>, +} From 9afe327bca0ee236d62c2d807c4a95ebe8c6633c Mon Sep 17 00:00:00 2001 From: "Lei, HUANG" <6406592+v0y4g3r@users.noreply.github.com> Date: Tue, 12 Mar 2024 20:00:38 +0800 Subject: [PATCH 11/17] feat: improve prom write requests decode performance (#3478) * feat: optimize decode performance * fix: some cr comments --- Cargo.lock | 1 + src/servers/Cargo.toml | 1 + src/servers/src/prom_row_builder.rs | 6 +- src/servers/src/proto.rs | 122 ++++++++++++++++++---------- 4 files changed, 86 insertions(+), 44 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4ae09096a9..c2b4dc0b5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9193,6 +9193,7 @@ dependencies = [ "derive_builder 0.12.0", "digest", "futures", + "hashbrown 0.14.3", "headers", "hex", "hostname", diff --git a/src/servers/Cargo.toml b/src/servers/Cargo.toml index a9363a0efe..04e226e3cd 100644 --- a/src/servers/Cargo.toml +++ b/src/servers/Cargo.toml @@ -49,6 +49,7 @@ datatypes.workspace = true derive_builder.workspace = true digest = "0.10" futures = "0.3" +hashbrown = "0.14" headers = "0.3" hex = { version = "0.4" } hostname = "0.3.1" diff --git a/src/servers/src/prom_row_builder.rs b/src/servers/src/prom_row_builder.rs index 20a049f472..0829973092 100644 --- a/src/servers/src/prom_row_builder.rs +++ b/src/servers/src/prom_row_builder.rs @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::hash_map::Entry; -use std::collections::HashMap; use std::string::ToString; use api::prom_store::remote::Sample; @@ -23,6 +21,8 @@ use api::v1::{ Value, }; use common_query::prelude::{GREPTIME_TIMESTAMP, GREPTIME_VALUE}; +use hashbrown::hash_map::Entry; +use hashbrown::HashMap; use crate::proto::PromLabel; use crate::repeated_field::Clear; @@ -85,7 +85,7 @@ impl Default for TableBuilder { impl TableBuilder { pub(crate) fn with_capacity(cols: usize, rows: usize) -> Self { - let mut col_indexes = HashMap::with_capacity(cols); + let mut col_indexes = HashMap::with_capacity_and_hasher(cols, Default::default()); col_indexes.insert(GREPTIME_TIMESTAMP.to_string(), 0); col_indexes.insert(GREPTIME_VALUE.to_string(), 1); diff --git a/src/servers/src/proto.rs b/src/servers/src/proto.rs index 1a96cd9ed8..adc9cc0ad3 100644 --- a/src/servers/src/proto.rs +++ b/src/servers/src/proto.rs @@ -13,12 +13,13 @@ // limitations under the License. use std::ops::Deref; +use std::slice; use api::prom_store::remote::Sample; use api::v1::RowInsertRequests; use bytes::{Buf, Bytes}; use prost::encoding::message::merge; -use prost::encoding::{decode_key, decode_varint, DecodeContext, WireType}; +use prost::encoding::{decode_key, decode_varint, WireType}; use prost::DecodeError; use crate::prom_row_builder::TablesBuilder; @@ -36,29 +37,22 @@ pub struct PromLabel { } impl Clear for PromLabel { - fn clear(&mut self) { - self.name.clear(); - self.value.clear(); - } + fn clear(&mut self) {} } impl PromLabel { - pub fn merge_field( + pub fn merge_field( &mut self, tag: u32, wire_type: WireType, - buf: &mut B, - ctx: DecodeContext, - ) -> Result<(), DecodeError> - where - B: Buf, - { + buf: &mut Bytes, + ) -> Result<(), DecodeError> { const STRUCT_NAME: &str = "PromLabel"; match tag { 1u32 => { // decode label name let value = &mut self.name; - prost::encoding::bytes::merge(wire_type, value, buf, ctx).map_err(|mut error| { + merge_bytes(value, buf).map_err(|mut error| { error.push(STRUCT_NAME, "name"); error }) @@ -66,16 +60,69 @@ impl PromLabel { 2u32 => { // decode label value let value = &mut self.value; - prost::encoding::bytes::merge(wire_type, value, buf, ctx).map_err(|mut error| { + merge_bytes(value, buf).map_err(|mut error| { error.push(STRUCT_NAME, "value"); error }) } - _ => prost::encoding::skip_field(wire_type, tag, buf, ctx), + _ => prost::encoding::skip_field(wire_type, tag, buf, Default::default()), } } } +#[inline(always)] +fn copy_to_bytes(data: &mut Bytes, len: usize) -> Bytes { + if len == data.remaining() { + std::mem::replace(data, Bytes::new()) + } else { + let ret = split_to(data, len); + data.advance(len); + ret + } +} + +/// Similar to `Bytes::split_to`, but directly operates on underlying memory region. +/// # Safety +/// This function is safe as long as `data` is backed by a consecutive region of memory, +/// for example `Vec` or `&[u8]`, and caller must ensure that `buf` outlives +/// the `Bytes` returned. +#[inline(always)] +fn split_to(buf: &mut Bytes, end: usize) -> Bytes { + let len = buf.len(); + assert!( + end <= len, + "range end out of bounds: {:?} <= {:?}", + end, + len, + ); + + if end == 0 { + return Bytes::new(); + } + + let ptr = buf.as_ptr(); + let x = unsafe { slice::from_raw_parts(ptr, end) }; + // `Bytes::drop` does nothing when it's built via `from_static`. + Bytes::from_static(x) +} + +/// Reads a variable-length encoded bytes field from `buf` and assign it to `value`. +/// # Safety +/// Callers must ensure `buf` outlives `value`. +#[inline(always)] +fn merge_bytes(value: &mut Bytes, buf: &mut Bytes) -> Result<(), DecodeError> { + let len = decode_varint(buf)?; + if len > buf.remaining() as u64 { + return Err(DecodeError::new(format!( + "buffer underflow, len: {}, remaining: {}", + len, + buf.remaining() + ))); + } + *value = copy_to_bytes(buf, len as usize); + Ok(()) +} + #[derive(Default)] pub struct PromTimeSeries { pub table_name: String, @@ -92,16 +139,12 @@ impl Clear for PromTimeSeries { } impl PromTimeSeries { - pub fn merge_field( + pub fn merge_field( &mut self, tag: u32, wire_type: WireType, - buf: &mut B, - ctx: DecodeContext, - ) -> Result<(), DecodeError> - where - B: Buf, - { + buf: &mut Bytes, + ) -> Result<(), DecodeError> { const STRUCT_NAME: &str = "PromTimeSeries"; match tag { 1u32 => { @@ -120,7 +163,7 @@ impl PromTimeSeries { let limit = remaining - len as usize; while buf.remaining() > limit { let (tag, wire_type) = decode_key(buf)?; - label.merge_field(tag, wire_type, buf, ctx.clone())?; + label.merge_field(tag, wire_type, buf)?; } if buf.remaining() != limit { return Err(DecodeError::new("delimited length exceeded")); @@ -135,15 +178,17 @@ impl PromTimeSeries { } 2u32 => { let sample = self.samples.push_default(); - merge(WireType::LengthDelimited, sample, buf, ctx).map_err(|mut error| { - error.push(STRUCT_NAME, "samples"); - error - })?; + merge(WireType::LengthDelimited, sample, buf, Default::default()).map_err( + |mut error| { + error.push(STRUCT_NAME, "samples"); + error + }, + )?; Ok(()) } - // skip exemplars - 3u32 => prost::encoding::skip_field(wire_type, tag, buf, ctx), - _ => prost::encoding::skip_field(wire_type, tag, buf, ctx), + // todo(hl): exemplars are skipped temporarily + 3u32 => prost::encoding::skip_field(wire_type, tag, buf, Default::default()), + _ => prost::encoding::skip_field(wire_type, tag, buf, Default::default()), } } @@ -178,13 +223,9 @@ impl PromWriteRequest { self.table_data.as_insert_requests() } - pub fn merge(&mut self, mut buf: B) -> Result<(), DecodeError> - where - B: Buf, - Self: Sized, - { + // todo(hl): maybe use &[u8] can reduce the overhead introduced with Bytes. + pub fn merge(&mut self, mut buf: Bytes) -> Result<(), DecodeError> { const STRUCT_NAME: &str = "PromWriteRequest"; - let ctx = DecodeContext::default(); while buf.has_remaining() { let (tag, wire_type) = decode_key(&mut buf)?; assert_eq!(WireType::LengthDelimited, wire_type); @@ -203,8 +244,7 @@ impl PromWriteRequest { let limit = remaining - len as usize; while buf.remaining() > limit { let (tag, wire_type) = decode_key(&mut buf)?; - self.series - .merge_field(tag, wire_type, &mut buf, ctx.clone())?; + self.series.merge_field(tag, wire_type, &mut buf)?; } if buf.remaining() != limit { return Err(DecodeError::new("delimited length exceeded")); @@ -212,10 +252,10 @@ impl PromWriteRequest { self.series.add_to_table_data(&mut self.table_data); } 3u32 => { - // we can ignore metadata for now. - prost::encoding::skip_field(wire_type, tag, &mut buf, ctx.clone())?; + // todo(hl): metadata are skipped. + prost::encoding::skip_field(wire_type, tag, &mut buf, Default::default())?; } - _ => prost::encoding::skip_field(wire_type, tag, &mut buf, ctx.clone())?, + _ => prost::encoding::skip_field(wire_type, tag, &mut buf, Default::default())?, } } Ok(()) From 7c895e2605d729eb0dc8500f33da1354cd998364 Mon Sep 17 00:00:00 2001 From: Yingwen Date: Tue, 12 Mar 2024 20:02:58 +0800 Subject: [PATCH 12/17] perf: more benchmarks for memtables (#3491) * chore: remove duplicate bench * refactor: rename bench * perf: add full scan bench for memtable * feat: filter bench and add time series to bench group * chore: comment * refactor: rename * style: fix clippy --- src/mito2/Cargo.toml | 3 +- src/mito2/README.md | 6 + src/mito2/benches/bench_merge_tree.rs | 21 -- src/mito2/benches/memtable_bench.rs | 352 +++++++++++++++++++++++ src/mito2/benches/merge_tree_bench.rs | 36 --- src/mito2/src/test_util/memtable_util.rs | 33 ++- 6 files changed, 379 insertions(+), 72 deletions(-) delete mode 100644 src/mito2/benches/bench_merge_tree.rs create mode 100644 src/mito2/benches/memtable_bench.rs delete mode 100644 src/mito2/benches/merge_tree_bench.rs diff --git a/src/mito2/Cargo.toml b/src/mito2/Cargo.toml index 5a9a117fe2..8a93e20b57 100644 --- a/src/mito2/Cargo.toml +++ b/src/mito2/Cargo.toml @@ -79,5 +79,6 @@ rand.workspace = true toml.workspace = true [[bench]] -name = "bench_merge_tree" +name = "memtable_bench" harness = false +required-features = ["test"] diff --git a/src/mito2/README.md b/src/mito2/README.md index ae58d6d3d2..1e2accd939 100644 --- a/src/mito2/README.md +++ b/src/mito2/README.md @@ -7,3 +7,9 @@ The Alfa Romeo [MiTo](https://en.wikipedia.org/wiki/Alfa_Romeo_MiTo) is a front- > "You can't be a true petrolhead until you've owned an Alfa Romeo." >
-- by Jeremy Clarkson
+ +## Benchmarks +Run benchmarks in this crate: +```bash +cargo bench -p mito2 -F test +``` diff --git a/src/mito2/benches/bench_merge_tree.rs b/src/mito2/benches/bench_merge_tree.rs deleted file mode 100644 index febc648dae..0000000000 --- a/src/mito2/benches/bench_merge_tree.rs +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -mod merge_tree_bench; - -use criterion::criterion_main; - -criterion_main! { - merge_tree_bench::benches -} diff --git a/src/mito2/benches/memtable_bench.rs b/src/mito2/benches/memtable_bench.rs new file mode 100644 index 0000000000..30701fe2ba --- /dev/null +++ b/src/mito2/benches/memtable_bench.rs @@ -0,0 +1,352 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use api::v1::value::ValueData; +use api::v1::{Row, Rows, SemanticType}; +use criterion::{criterion_group, criterion_main, Criterion}; +use datafusion_common::Column; +use datafusion_expr::{lit, Expr}; +use datatypes::data_type::ConcreteDataType; +use datatypes::schema::ColumnSchema; +use mito2::memtable::merge_tree::{MergeTreeConfig, MergeTreeMemtable}; +use mito2::memtable::time_series::TimeSeriesMemtable; +use mito2::memtable::{KeyValues, Memtable}; +use mito2::test_util::memtable_util::{self, region_metadata_to_row_schema}; +use rand::rngs::ThreadRng; +use rand::seq::SliceRandom; +use rand::Rng; +use store_api::metadata::{ + ColumnMetadata, RegionMetadata, RegionMetadataBuilder, RegionMetadataRef, +}; +use store_api::storage::RegionId; +use table::predicate::Predicate; + +/// Writes rows. +fn write_rows(c: &mut Criterion) { + let metadata = memtable_util::metadata_with_primary_key(vec![1, 0], true); + let timestamps = (0..100).collect::>(); + + // Note that this test only generate one time series. + let mut group = c.benchmark_group("write"); + group.bench_function("merge_tree", |b| { + let memtable = + MergeTreeMemtable::new(1, metadata.clone(), None, &MergeTreeConfig::default()); + let kvs = + memtable_util::build_key_values(&metadata, "hello".to_string(), 42, ×tamps, 1); + b.iter(|| { + memtable.write(&kvs).unwrap(); + }); + }); + group.bench_function("time_series", |b| { + let memtable = TimeSeriesMemtable::new(metadata.clone(), 1, None); + let kvs = + memtable_util::build_key_values(&metadata, "hello".to_string(), 42, ×tamps, 1); + b.iter(|| { + memtable.write(&kvs).unwrap(); + }); + }); +} + +/// Scans all rows. +fn full_scan(c: &mut Criterion) { + let metadata = Arc::new(cpu_metadata()); + let config = MergeTreeConfig::default(); + let start_sec = 1710043200; + let generator = CpuDataGenerator::new(metadata.clone(), 4000, start_sec, start_sec + 3600 * 2); + + let mut group = c.benchmark_group("full_scan"); + group.sample_size(10); + group.bench_function("merge_tree", |b| { + let memtable = MergeTreeMemtable::new(1, metadata.clone(), None, &config); + for kvs in generator.iter() { + memtable.write(&kvs).unwrap(); + } + + b.iter(|| { + let iter = memtable.iter(None, None).unwrap(); + for batch in iter { + let _batch = batch.unwrap(); + } + }); + }); + group.bench_function("time_series", |b| { + let memtable = TimeSeriesMemtable::new(metadata.clone(), 1, None); + for kvs in generator.iter() { + memtable.write(&kvs).unwrap(); + } + + b.iter(|| { + let iter = memtable.iter(None, None).unwrap(); + for batch in iter { + let _batch = batch.unwrap(); + } + }); + }); +} + +/// Filters 1 host. +fn filter_1_host(c: &mut Criterion) { + let metadata = Arc::new(cpu_metadata()); + let config = MergeTreeConfig::default(); + let start_sec = 1710043200; + let generator = CpuDataGenerator::new(metadata.clone(), 4000, start_sec, start_sec + 3600 * 2); + + let mut group = c.benchmark_group("filter_1_host"); + group.sample_size(10); + group.bench_function("merge_tree", |b| { + let memtable = MergeTreeMemtable::new(1, metadata.clone(), None, &config); + for kvs in generator.iter() { + memtable.write(&kvs).unwrap(); + } + let predicate = generator.random_host_filter(); + + b.iter(|| { + let iter = memtable.iter(None, Some(predicate.clone())).unwrap(); + for batch in iter { + let _batch = batch.unwrap(); + } + }); + }); + group.bench_function("time_series", |b| { + let memtable = TimeSeriesMemtable::new(metadata.clone(), 1, None); + for kvs in generator.iter() { + memtable.write(&kvs).unwrap(); + } + let predicate = generator.random_host_filter(); + + b.iter(|| { + let iter = memtable.iter(None, Some(predicate.clone())).unwrap(); + for batch in iter { + let _batch = batch.unwrap(); + } + }); + }); +} + +struct Host { + hostname: String, + region: String, + datacenter: String, + rack: String, + os: String, + arch: String, + team: String, + service: String, + service_version: String, + service_environment: String, +} + +impl Host { + fn random_with_id(id: usize) -> Host { + let mut rng = rand::thread_rng(); + let region = format!("ap-southeast-{}", rng.gen_range(0..10)); + let datacenter = format!( + "{}{}", + region, + ['a', 'b', 'c', 'd', 'e'].choose(&mut rng).unwrap() + ); + Host { + hostname: format!("host_{id}"), + region, + datacenter, + rack: rng.gen_range(0..100).to_string(), + os: "Ubuntu16.04LTS".to_string(), + arch: "x86".to_string(), + team: "CHI".to_string(), + service: rng.gen_range(0..100).to_string(), + service_version: rng.gen_range(0..10).to_string(), + service_environment: "test".to_string(), + } + } + + fn fill_values(&self, values: &mut Vec) { + let tags = [ + api::v1::Value { + value_data: Some(ValueData::StringValue(self.hostname.clone())), + }, + api::v1::Value { + value_data: Some(ValueData::StringValue(self.region.clone())), + }, + api::v1::Value { + value_data: Some(ValueData::StringValue(self.datacenter.clone())), + }, + api::v1::Value { + value_data: Some(ValueData::StringValue(self.rack.clone())), + }, + api::v1::Value { + value_data: Some(ValueData::StringValue(self.os.clone())), + }, + api::v1::Value { + value_data: Some(ValueData::StringValue(self.arch.clone())), + }, + api::v1::Value { + value_data: Some(ValueData::StringValue(self.team.clone())), + }, + api::v1::Value { + value_data: Some(ValueData::StringValue(self.service.clone())), + }, + api::v1::Value { + value_data: Some(ValueData::StringValue(self.service_version.clone())), + }, + api::v1::Value { + value_data: Some(ValueData::StringValue(self.service_environment.clone())), + }, + ]; + for tag in tags { + values.push(tag); + } + } +} + +struct CpuDataGenerator { + metadata: RegionMetadataRef, + column_schemas: Vec, + hosts: Vec, + start_sec: i64, + end_sec: i64, +} + +impl CpuDataGenerator { + fn new(metadata: RegionMetadataRef, num_hosts: usize, start_sec: i64, end_sec: i64) -> Self { + let column_schemas = region_metadata_to_row_schema(&metadata); + Self { + metadata, + column_schemas, + hosts: Self::generate_hosts(num_hosts), + start_sec, + end_sec, + } + } + + fn iter(&self) -> impl Iterator + '_ { + // point per 10s. + (self.start_sec..self.end_sec) + .step_by(10) + .enumerate() + .map(|(seq, ts)| self.build_key_values(seq, ts)) + } + + fn build_key_values(&self, seq: usize, current_sec: i64) -> KeyValues { + let rows = self + .hosts + .iter() + .map(|host| { + let mut rng = rand::thread_rng(); + let mut values = Vec::with_capacity(21); + values.push(api::v1::Value { + value_data: Some(ValueData::TimestampMillisecondValue(current_sec * 1000)), + }); + host.fill_values(&mut values); + for _ in 0..10 { + values.push(api::v1::Value { + value_data: Some(ValueData::F64Value(Self::random_f64(&mut rng))), + }); + } + Row { values } + }) + .collect(); + let mutation = api::v1::Mutation { + op_type: api::v1::OpType::Put as i32, + sequence: seq as u64, + rows: Some(Rows { + schema: self.column_schemas.clone(), + rows, + }), + }; + + KeyValues::new(&self.metadata, mutation).unwrap() + } + + fn random_host_filter(&self) -> Predicate { + let host = self.random_hostname(); + let expr = Expr::Column(Column::from_name("hostname")).eq(lit(host)); + Predicate::new(vec![expr.into()]) + } + + fn random_hostname(&self) -> String { + let mut rng = rand::thread_rng(); + self.hosts.choose(&mut rng).unwrap().hostname.clone() + } + + fn random_f64(rng: &mut ThreadRng) -> f64 { + let base: u32 = rng.gen_range(30..95); + base as f64 + } + + fn generate_hosts(num_hosts: usize) -> Vec { + (0..num_hosts).map(Host::random_with_id).collect() + } +} + +/// Creates a metadata for TSBS cpu-like table. +fn cpu_metadata() -> RegionMetadata { + let mut builder = RegionMetadataBuilder::new(RegionId::new(1, 1)); + builder.push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new( + "ts", + ConcreteDataType::timestamp_millisecond_datatype(), + false, + ), + semantic_type: SemanticType::Timestamp, + column_id: 0, + }); + let mut column_id = 1; + let tags = [ + "hostname", + "region", + "datacenter", + "rack", + "os", + "arch", + "team", + "service", + "service_version", + "service_environment", + ]; + for tag in tags { + builder.push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new(tag, ConcreteDataType::string_datatype(), true), + semantic_type: SemanticType::Tag, + column_id, + }); + column_id += 1; + } + let fields = [ + "usage_user", + "usage_system", + "usage_idle", + "usage_nice", + "usage_iowait", + "usage_irq", + "usage_softirq", + "usage_steal", + "usage_guest", + "usage_guest_nice", + ]; + for field in fields { + builder.push_column_metadata(ColumnMetadata { + column_schema: ColumnSchema::new(field, ConcreteDataType::float64_datatype(), true), + semantic_type: SemanticType::Field, + column_id, + }); + column_id += 1; + } + builder.primary_key(vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]); + builder.build().unwrap() +} + +criterion_group!(benches, write_rows, full_scan, filter_1_host); +criterion_main!(benches); diff --git a/src/mito2/benches/merge_tree_bench.rs b/src/mito2/benches/merge_tree_bench.rs deleted file mode 100644 index 3ad86c2d9e..0000000000 --- a/src/mito2/benches/merge_tree_bench.rs +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2023 Greptime Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use criterion::{criterion_group, criterion_main, Criterion}; -use mito2::memtable::merge_tree::{MergeTreeConfig, MergeTreeMemtable}; -use mito2::memtable::Memtable; -use mito2::test_util::memtable_util; - -fn bench_merge_tree_memtable(c: &mut Criterion) { - let metadata = memtable_util::metadata_with_primary_key(vec![1, 0], true); - let timestamps = (0..100).collect::>(); - - let memtable = MergeTreeMemtable::new(1, metadata.clone(), None, &MergeTreeConfig::default()); - - let _ = c.bench_function("MergeTreeMemtable", |b| { - let kvs = - memtable_util::build_key_values(&metadata, "hello".to_string(), 42, ×tamps, 1); - b.iter(|| { - memtable.write(&kvs).unwrap(); - }); - }); -} - -criterion_group!(benches, bench_merge_tree_memtable); -criterion_main!(benches); diff --git a/src/mito2/src/test_util/memtable_util.rs b/src/mito2/src/test_util/memtable_util.rs index 4f3d9cfd3d..38108dff3c 100644 --- a/src/mito2/src/test_util/memtable_util.rs +++ b/src/mito2/src/test_util/memtable_util.rs @@ -219,25 +219,14 @@ pub(crate) fn extract_data_batch(batch: &DataBatch) -> (u16, Vec<(i64, u64)>) { /// Builds key values with timestamps (ms) and sequences for test. pub(crate) fn build_key_values_with_ts_seq_values( - schema: &RegionMetadataRef, + metadata: &RegionMetadataRef, k0: String, k1: u32, timestamps: impl Iterator, values: impl Iterator>, sequence: SequenceNumber, ) -> KeyValues { - let column_schema = schema - .column_metadatas - .iter() - .map(|c| api::v1::ColumnSchema { - column_name: c.column_schema.name.clone(), - datatype: ColumnDataTypeWrapper::try_from(c.column_schema.data_type.clone()) - .unwrap() - .datatype() as i32, - semantic_type: c.semantic_type as i32, - ..Default::default() - }) - .collect(); + let column_schema = region_metadata_to_row_schema(metadata); let rows = timestamps .zip(values) @@ -269,7 +258,23 @@ pub(crate) fn build_key_values_with_ts_seq_values( rows, }), }; - KeyValues::new(schema.as_ref(), mutation).unwrap() + KeyValues::new(metadata.as_ref(), mutation).unwrap() +} + +/// Converts the region metadata to column schemas for a row. +pub fn region_metadata_to_row_schema(metadata: &RegionMetadataRef) -> Vec { + metadata + .column_metadatas + .iter() + .map(|c| api::v1::ColumnSchema { + column_name: c.column_schema.name.clone(), + datatype: ColumnDataTypeWrapper::try_from(c.column_schema.data_type.clone()) + .unwrap() + .datatype() as i32, + semantic_type: c.semantic_type as i32, + ..Default::default() + }) + .collect() } /// Encode keys. From cafb4708ce00dbae0216ece3ab86e9fe02aa993e Mon Sep 17 00:00:00 2001 From: tison Date: Tue, 12 Mar 2024 21:09:34 +0800 Subject: [PATCH 13/17] refactor: validate constraints eagerly (#3472) * chore: validate constraints eagerly Signed-off-by: tison * use timestamp column Signed-off-by: tison * fixup Signed-off-by: tison * lint Signed-off-by: tison * compile Signed-off-by: tison --------- Signed-off-by: tison --- src/sql/src/parsers/create_parser.rs | 44 +++++++++----------- src/sql/src/statements/create.rs | 4 +- tests-integration/src/lib.rs | 1 + tests-integration/src/tests/instance_test.rs | 3 +- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/sql/src/parsers/create_parser.rs b/src/sql/src/parsers/create_parser.rs index b154ee48ab..b84c965aa5 100644 --- a/src/sql/src/parsers/create_parser.rs +++ b/src/sql/src/parsers/create_parser.rs @@ -62,17 +62,21 @@ impl<'a> ParserContext<'a> { let _ = self.parser.next_token(); self.parser .expect_keyword(Keyword::TABLE) - .context(error::SyntaxSnafu)?; + .context(SyntaxSnafu)?; let if_not_exists = self.parser .parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]); let table_name = self.intern_parse_table_name()?; let (columns, constraints) = self.parse_columns()?; + if !columns.is_empty() { + validate_time_index(&columns, &constraints)?; + } + let engine = self.parse_table_engine(common_catalog::consts::FILE_ENGINE)?; let options = self .parser .parse_options(Keyword::WITH) - .context(error::SyntaxSnafu)? + .context(SyntaxSnafu)? .into_iter() .filter_map(|option| { if let Some(v) = parse_option_string(option.value) { @@ -140,8 +144,12 @@ impl<'a> ParserContext<'a> { } let (columns, constraints) = self.parse_columns()?; + validate_time_index(&columns, &constraints)?; let partitions = self.parse_partitions()?; + if let Some(partitions) = &partitions { + validate_partitions(&columns, partitions)?; + } let engine = self.parse_table_engine(default_engine())?; let options = self @@ -168,7 +176,6 @@ impl<'a> ParserContext<'a> { table_id: 0, // table id is assigned by catalog manager partitions, }; - validate_create(&create_table)?; Ok(Statement::CreateTable(create_table)) } @@ -553,18 +560,8 @@ impl<'a> ParserContext<'a> { } } -fn validate_create(create_table: &CreateTable) -> Result<()> { - if let Some(partitions) = &create_table.partitions { - validate_partitions(&create_table.columns, partitions)?; - } - validate_time_index(create_table)?; - - Ok(()) -} - -fn validate_time_index(create_table: &CreateTable) -> Result<()> { - let time_index_constraints: Vec<_> = create_table - .constraints +fn validate_time_index(columns: &[ColumnDef], constraints: &[TableConstraint]) -> Result<()> { + let time_index_constraints: Vec<_> = constraints .iter() .filter_map(|c| { if let TableConstraint::Unique { @@ -605,8 +602,7 @@ fn validate_time_index(create_table: &CreateTable) -> Result<()> { // It's safe to use time_index_constraints[0][0], // we already check the bound above. let time_index_column_ident = &time_index_constraints[0][0]; - let time_index_column = create_table - .columns + let time_index_column = columns .iter() .find(|c| c.name.value == *time_index_column_ident.value) .with_context(|| InvalidTimeIndexSnafu { @@ -753,7 +749,7 @@ mod tests { fn test_validate_external_table_options() { let sql = "CREATE EXTERNAL TABLE city ( host string, - ts int64, + ts timestamp, cpu float64 default 0, memory float64, TIME INDEX (ts), @@ -836,7 +832,7 @@ mod tests { fn test_parse_create_external_table_with_schema() { let sql = "CREATE EXTERNAL TABLE city ( host string, - ts int64, + ts timestamp, cpu float32 default 0, memory float64, TIME INDEX (ts), @@ -859,7 +855,7 @@ mod tests { let columns = &c.columns; assert_column_def(&columns[0], "host", "STRING"); - assert_column_def(&columns[1], "ts", "BIGINT"); + assert_column_def(&columns[1], "ts", "TIMESTAMP"); assert_column_def(&columns[2], "cpu", "FLOAT"); assert_column_def(&columns[3], "memory", "DOUBLE"); @@ -938,7 +934,7 @@ ENGINE=mito"; let _ = result.unwrap(); let sql = r" -CREATE TABLE rcx ( a INT, b STRING, c INT ) +CREATE TABLE rcx ( ts TIMESTAMP TIME INDEX, a INT, b STRING, c INT ) PARTITION ON COLUMNS(x) () ENGINE=mito"; let result = @@ -1326,7 +1322,7 @@ ENGINE=mito"; #[test] fn test_parse_partitions_with_error_syntax() { let sql = r" -CREATE TABLE rcx ( a INT, b STRING, c INT ) +CREATE TABLE rcx ( ts TIMESTAMP TIME INDEX, a INT, b STRING, c INT ) PARTITION COLUMNS(c, a) ( a < 10, a > 10 AND a < 20, @@ -1355,7 +1351,7 @@ ENGINE=mito"; #[test] fn test_parse_partitions_unreferenced_column() { let sql = r" -CREATE TABLE rcx ( a INT, b STRING, c INT ) +CREATE TABLE rcx ( ts TIMESTAMP TIME INDEX, a INT, b STRING, c INT ) PARTITION ON COLUMNS(c, a) ( b = 'foo' ) @@ -1371,7 +1367,7 @@ ENGINE=mito"; #[test] fn test_parse_partitions_not_binary_expr() { let sql = r" -CREATE TABLE rcx ( a INT, b STRING, c INT ) +CREATE TABLE rcx ( ts TIMESTAMP TIME INDEX, a INT, b STRING, c INT ) PARTITION ON COLUMNS(c, a) ( b ) diff --git a/src/sql/src/statements/create.rs b/src/sql/src/statements/create.rs index 8bbf64d86e..e665ef2577 100644 --- a/src/sql/src/statements/create.rs +++ b/src/sql/src/statements/create.rs @@ -229,6 +229,8 @@ pub struct CreateTableLike { #[cfg(test)] mod tests { + use std::assert_matches::assert_matches; + use crate::dialect::GreptimeDbDialect; use crate::error::Error; use crate::parser::{ParseOptions, ParserContext}; @@ -378,6 +380,6 @@ ENGINE=mito "; let result = ParserContext::create_with_dialect(sql, &GreptimeDbDialect {}, ParseOptions::default()); - assert!(matches!(result, Err(Error::InvalidTableOption { .. }))); + assert_matches!(result, Err(Error::InvalidTableOption { .. })) } } diff --git a/tests-integration/src/lib.rs b/tests-integration/src/lib.rs index 730694b8c6..edb88136d9 100644 --- a/tests-integration/src/lib.rs +++ b/tests-integration/src/lib.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![feature(assert_matches)] pub mod cluster; mod grpc; mod influxdb; diff --git a/tests-integration/src/tests/instance_test.rs b/tests-integration/src/tests/instance_test.rs index 5084b5b015..0c8aeaf09b 100644 --- a/tests-integration/src/tests/instance_test.rs +++ b/tests-integration/src/tests/instance_test.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::assert_matches::assert_matches; use std::env; use std::sync::Arc; @@ -639,7 +640,7 @@ async fn test_execute_external_create_without_ts(instance: Arc ), ) .await; - assert!(matches!(result, Err(Error::TableOperation { .. }))); + assert_matches!(result, Err(Error::ParseSql { .. })); } #[apply(both_instances_cases)] From 904484b52589b2d1cf704ca88b9b1c987148c05f Mon Sep 17 00:00:00 2001 From: Zhenchi Date: Wed, 13 Mar 2024 10:46:00 +0800 Subject: [PATCH 14/17] fix(common-time): allow building nanos timestamp from parts split from i64::MIN (#3493) Signed-off-by: Zhenchi --- src/common/time/src/timestamp.rs | 90 +++++++++++++++++++++++++++++--- 1 file changed, 83 insertions(+), 7 deletions(-) diff --git a/src/common/time/src/timestamp.rs b/src/common/time/src/timestamp.rs index 8fa01cd8be..7010ff8293 100644 --- a/src/common/time/src/timestamp.rs +++ b/src/common/time/src/timestamp.rs @@ -36,7 +36,7 @@ use crate::{error, Interval}; /// - for [TimeUnit::Second]: [-262144-01-01 00:00:00, +262143-12-31 23:59:59] /// - for [TimeUnit::Millisecond]: [-262144-01-01 00:00:00.000, +262143-12-31 23:59:59.999] /// - for [TimeUnit::Microsecond]: [-262144-01-01 00:00:00.000000, +262143-12-31 23:59:59.999999] -/// - for [TimeUnit::Nanosecond]: [1677-09-21 00:12:43.145225, 2262-04-11 23:47:16.854775807] +/// - for [TimeUnit::Nanosecond]: [1677-09-21 00:12:43.145224192, 2262-04-11 23:47:16.854775807] /// /// # Note: /// For values out of range, you can still store these timestamps, but while performing arithmetic @@ -187,28 +187,28 @@ impl Timestamp { Self { unit, value } } - pub fn new_second(value: i64) -> Self { + pub const fn new_second(value: i64) -> Self { Self { value, unit: TimeUnit::Second, } } - pub fn new_millisecond(value: i64) -> Self { + pub const fn new_millisecond(value: i64) -> Self { Self { value, unit: TimeUnit::Millisecond, } } - pub fn new_microsecond(value: i64) -> Self { + pub const fn new_microsecond(value: i64) -> Self { Self { value, unit: TimeUnit::Microsecond, } } - pub fn new_nanosecond(value: i64) -> Self { + pub const fn new_nanosecond(value: i64) -> Self { Self { value, unit: TimeUnit::Nanosecond, @@ -281,8 +281,26 @@ impl Timestamp { .and_then(|v| v.checked_add(micros as i64)) .map(Timestamp::new_microsecond) } else { + // Refer to + // + // subsec nanos are always non-negative, however the timestamp itself (both in seconds and in nanos) can be + // negative. Now i64::MIN is NOT dividable by 1_000_000_000, so + // + // (sec * 1_000_000_000) + nsec + // + // may underflow (even when in theory we COULD represent the datetime as i64) because we add the non-negative + // nanos AFTER the multiplication. This is fixed by converting the negative case to + // + // ((sec + 1) * 1_000_000_000) + (nsec - 1_000_000_000) + let mut sec = sec; + let mut nsec = nsec as i64; + if sec < 0 && nsec > 0 { + nsec -= 1_000_000_000; + sec += 1; + } + sec.checked_mul(1_000_000_000) - .and_then(|v| v.checked_add(nsec as i64)) + .and_then(|v| v.checked_add(nsec)) .map(Timestamp::new_nanosecond) } } @@ -425,6 +443,20 @@ impl Timestamp { } } +impl Timestamp { + pub const MIN_SECOND: Self = Self::new_second(-8_334_601_228_800); + pub const MAX_SECOND: Self = Self::new_second(8_210_266_876_799); + + pub const MIN_MILLISECOND: Self = Self::new_millisecond(-8_334_601_228_800_000); + pub const MAX_MILLISECOND: Self = Self::new_millisecond(8_210_266_876_799_999); + + pub const MIN_MICROSECOND: Self = Self::new_microsecond(-8_334_601_228_800_000_000); + pub const MAX_MICROSECOND: Self = Self::new_microsecond(8_210_266_876_799_999_999); + + pub const MIN_NANOSECOND: Self = Self::new_nanosecond(i64::MIN); + pub const MAX_NANOSECOND: Self = Self::new_nanosecond(i64::MAX); +} + /// Converts the naive datetime (which has no specific timezone) to a /// nanosecond epoch timestamp in UTC. fn naive_datetime_to_timestamp( @@ -586,6 +618,7 @@ impl Hash for Timestamp { mod tests { use std::collections::hash_map::DefaultHasher; + use chrono_tz::Tz; use rand::Rng; use serde_json::Value; @@ -1297,7 +1330,7 @@ mod tests { "+262142-12-31 23:59:59Z", "+262142-12-31 23:59:59.999Z", "+262142-12-31 23:59:59.999999Z", - "1677-09-21 00:12:43.145225Z", + "1677-09-21 00:12:43.145224192Z", "2262-04-11 23:47:16.854775807Z", "+100000-01-01 00:00:01.5Z", ]; @@ -1306,4 +1339,47 @@ mod tests { Timestamp::from_str_utc(s).unwrap(); } } + + #[test] + fn test_min_nanos_roundtrip() { + let (sec, nsec) = Timestamp::MIN_NANOSECOND.split(); + let ts = Timestamp::from_splits(sec, nsec).unwrap(); + assert_eq!(Timestamp::MIN_NANOSECOND, ts); + } + + #[test] + fn test_timestamp_bound_format() { + assert_eq!( + "1677-09-21 00:12:43.145224192", + Timestamp::MIN_NANOSECOND.to_timezone_aware_string(Some(&Timezone::Named(Tz::UTC))) + ); + assert_eq!( + "2262-04-11 23:47:16.854775807", + Timestamp::MAX_NANOSECOND.to_timezone_aware_string(Some(&Timezone::Named(Tz::UTC))) + ); + assert_eq!( + "-262143-01-01 00:00:00", + Timestamp::MIN_MICROSECOND.to_timezone_aware_string(Some(&Timezone::Named(Tz::UTC))) + ); + assert_eq!( + "+262142-12-31 23:59:59.999999", + Timestamp::MAX_MICROSECOND.to_timezone_aware_string(Some(&Timezone::Named(Tz::UTC))) + ); + assert_eq!( + "-262143-01-01 00:00:00", + Timestamp::MIN_MILLISECOND.to_timezone_aware_string(Some(&Timezone::Named(Tz::UTC))) + ); + assert_eq!( + "+262142-12-31 23:59:59.999", + Timestamp::MAX_MILLISECOND.to_timezone_aware_string(Some(&Timezone::Named(Tz::UTC))) + ); + assert_eq!( + "-262143-01-01 00:00:00", + Timestamp::MIN_SECOND.to_timezone_aware_string(Some(&Timezone::Named(Tz::UTC))) + ); + assert_eq!( + "+262142-12-31 23:59:59", + Timestamp::MAX_SECOND.to_timezone_aware_string(Some(&Timezone::Named(Tz::UTC))) + ); + } } From fb4da05f250a90466d213d1b15fd154afcfebf43 Mon Sep 17 00:00:00 2001 From: WU Jingdi Date: Wed, 13 Mar 2024 17:20:34 +0800 Subject: [PATCH 15/17] fix: adjust fill behavior of range query (#3489) --- src/query/src/range_select/plan.rs | 324 ++++++++++++++---- src/query/src/range_select/plan_rewrite.rs | 40 ++- tests/cases/standalone/common/range/by.result | 62 ++-- .../standalone/common/range/calculate.result | 240 ++++++------- .../standalone/common/range/error.result | 8 +- tests/cases/standalone/common/range/error.sql | 2 - .../cases/standalone/common/range/fill.result | 291 +++++++++++++--- tests/cases/standalone/common/range/fill.sql | 49 ++- .../standalone/common/range/interval.result | 32 +- .../cases/standalone/common/range/nest.result | 24 +- .../standalone/common/range/precisions.result | 28 +- .../common/range/special_aggr.result | 290 ++++++++-------- tests/cases/standalone/common/range/to.result | 126 +++---- 13 files changed, 963 insertions(+), 553 deletions(-) diff --git a/src/query/src/range_select/plan.rs b/src/query/src/range_select/plan.rs index c32164790b..bff30d4789 100644 --- a/src/query/src/range_select/plan.rs +++ b/src/query/src/range_select/plan.rs @@ -14,8 +14,8 @@ use std::any::Any; use std::cmp::Ordering; -use std::collections::hash_map::Entry; -use std::collections::HashMap; +use std::collections::btree_map::Entry; +use std::collections::{BTreeMap, HashMap}; use std::fmt::Display; use std::pin::Pin; use std::sync::Arc; @@ -218,14 +218,15 @@ impl Display for Fill { } impl Fill { - pub fn try_from_str(value: &str, datatype: &DataType) -> DfResult { + pub fn try_from_str(value: &str, datatype: &DataType) -> DfResult> { let s = value.to_uppercase(); match s.as_str() { - "NULL" | "" => Ok(Self::Null), - "PREV" => Ok(Self::Prev), + "" => Ok(None), + "NULL" => Ok(Some(Self::Null)), + "PREV" => Ok(Some(Self::Prev)), "LINEAR" => { if datatype.is_numeric() { - Ok(Self::Linear) + Ok(Some(Self::Linear)) } else { Err(DataFusionError::Plan(format!( "Use FILL LINEAR on Non-numeric DataType {}", @@ -240,13 +241,17 @@ impl Fill { s, err )) }) - .map(Fill::Const), + .map(|x| Some(Fill::Const(x))), } } /// The input `data` contains data on a complete time series. /// If the filling strategy is `PREV` or `LINEAR`, caller must be ensured that the incoming `ts`&`data` is ascending time order. pub fn apply_fill_strategy(&self, ts: &[i64], data: &mut [ScalarValue]) -> DfResult<()> { + // No calculation need in `Fill::Null` + if matches!(self, Fill::Null) { + return Ok(()); + } let len = data.len(); if *self == Fill::Linear { return Self::fill_linear(ts, data); @@ -254,7 +259,6 @@ impl Fill { for i in 0..len { if data[i].is_null() { match self { - Fill::Null => continue, Fill::Prev => { if i != 0 { data[i] = data[i - 1].clone() @@ -262,7 +266,8 @@ impl Fill { } // The calculation of linear interpolation is relatively complicated. // `Self::fill_linear` is used to dispose `Fill::Linear`. - Fill::Linear => unreachable!(), + // No calculation need in `Fill::Null` + Fill::Linear | Fill::Null => unreachable!(), Fill::Const(v) => data[i] = v.clone(), } } @@ -359,12 +364,12 @@ fn linear_interpolation( #[derive(Eq, Clone, Debug)] pub struct RangeFn { - /// with format like `max(a) RANGE 300s FILL NULL` + /// with format like `max(a) RANGE 300s [FILL NULL]` pub name: String, pub data_type: DataType, pub expr: Expr, pub range: Duration, - pub fill: Fill, + pub fill: Option, /// If the `FIll` strategy is `Linear` and the output is an integer, /// it is possible to calculate a floating point number. /// So for `FILL==LINEAR`, the entire data will be implicitly converted to Float type @@ -465,7 +470,7 @@ impl RangeSelect { name, data_type.clone(), // Only when data fill with Const option, the data can't be null - !matches!(fill, Fill::Const(..)), + !matches!(fill, Some(Fill::Const(..))), )) }, ) @@ -810,10 +815,26 @@ struct RangeFnExec { pub expr: Arc, pub args: Vec>, pub range: Millisecond, - pub fill: Fill, + pub fill: Option, pub need_cast: Option, } +impl Display for RangeFnExec { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(fill) = &self.fill { + write!( + f, + "{} RANGE {}s FILL {}", + self.expr.name(), + self.range / 1000, + fill + ) + } else { + write!(f, "{} RANGE {}s", self.expr.name(), self.range / 1000) + } + } +} + #[derive(Debug)] pub struct RangeSelectExec { input: Arc, @@ -834,18 +855,8 @@ impl DisplayAs for RangeSelectExec { match t { DisplayFormatType::Default | DisplayFormatType::Verbose => { write!(f, "RangeSelectExec: ")?; - let range_expr_strs: Vec = self - .range_exec - .iter() - .map(|e| { - format!( - "{} RANGE {}s FILL {}", - e.expr.name(), - e.range / 1000, - e.fill - ) - }) - .collect(); + let range_expr_strs: Vec = + self.range_exec.iter().map(RangeFnExec::to_string).collect(); let by: Vec = self.by.iter().map(|e| e.to_string()).collect(); write!( f, @@ -939,7 +950,7 @@ impl ExecutionPlan for RangeSelectExec { by: self.by.clone(), series_map: HashMap::new(), exec_state: ExecutionState::ReadingInput, - output_num_rows: 0, + num_not_null_rows: 0, row_converter, modify_map: HashMap::new(), metric: baseline_metric, @@ -979,8 +990,8 @@ struct RangeSelectStream { /// value: `[row_ids]` /// It is used to record the data that needs to be aggregated in each time slot during the data update process modify_map: HashMap<(u64, Millisecond), Vec>, - /// The number of rows of the final output - output_num_rows: usize, + /// The number of rows of not null rows in the final output + num_not_null_rows: usize, metric: BaselineMetrics, schema_project: Option>, schema_before_project: SchemaRef, @@ -992,7 +1003,7 @@ struct SeriesState { row: OwnedRow, /// key: align_ts /// value: a vector, each element is a range_fn follow the order of `range_exec` - align_ts_accumulator: HashMap>>, + align_ts_accumulator: BTreeMap>>, } /// Use `align_to` as time origin. @@ -1108,7 +1119,7 @@ impl RangeSelectStream { let accumulators_map = self.series_map.entry(*hash).or_insert_with(|| SeriesState { row: by_rows.row(*row as usize).owned(), - align_ts_accumulator: HashMap::new(), + align_ts_accumulator: BTreeMap::new(), }); match accumulators_map.align_ts_accumulator.entry(*ts) { Entry::Occupied(mut e) => { @@ -1116,7 +1127,7 @@ impl RangeSelectStream { accumulators[i].update_batch(&sliced_arrays) } Entry::Vacant(e) => { - self.output_num_rows += 1; + self.num_not_null_rows += 1; let mut accumulators = self .range_exec .iter() @@ -1141,29 +1152,47 @@ impl RangeSelectStream { // 1 for time index column let mut columns: Vec> = Vec::with_capacity(1 + self.range_exec.len() + self.by.len()); - let mut ts_builder = TimestampMillisecondBuilder::with_capacity(self.output_num_rows); - let mut all_scalar = vec![Vec::with_capacity(self.output_num_rows); self.range_exec.len()]; - let mut by_rows = Vec::with_capacity(self.output_num_rows); + let mut ts_builder = TimestampMillisecondBuilder::with_capacity(self.num_not_null_rows); + let mut all_scalar = + vec![Vec::with_capacity(self.num_not_null_rows); self.range_exec.len()]; + let mut by_rows = Vec::with_capacity(self.num_not_null_rows); let mut start_index = 0; - // RangePlan is calculated on a row basis. If a column uses the PREV or LINEAR filling strategy, - // we must arrange the data in the entire data row to determine the NULL filling value. - let need_sort_output = self + // If any range expr need fill, we need fill both the missing align_ts and null value. + let need_fill_output = self.range_exec.iter().any(|range| range.fill.is_some()); + // The padding value for each accumulator + let padding_values = self .range_exec .iter() - .any(|range| range.fill == Fill::Linear || range.fill == Fill::Prev); + .map(|e| e.expr.create_accumulator()?.evaluate()) + .collect::>>()?; for SeriesState { row, align_ts_accumulator, } in self.series_map.values() { - // collect data on time series - let mut align_ts = align_ts_accumulator.keys().copied().collect::>(); - if need_sort_output { - align_ts.sort(); + // skip empty time series + if align_ts_accumulator.is_empty() { + continue; } + // find the first and last align_ts + let begin_ts = *align_ts_accumulator.first_key_value().unwrap().0; + let end_ts = *align_ts_accumulator.last_key_value().unwrap().0; + let align_ts = if need_fill_output { + // we need to fill empty align_ts which not data in that solt + (begin_ts..=end_ts).step_by(self.align as usize).collect() + } else { + align_ts_accumulator.keys().copied().collect::>() + }; for ts in &align_ts { - for (i, accumulator) in align_ts_accumulator.get(ts).unwrap().iter().enumerate() { - all_scalar[i].push(accumulator.evaluate()?); + if let Some(slot) = align_ts_accumulator.get(ts) { + for (column, acc) in all_scalar.iter_mut().zip(slot.iter()) { + column.push(acc.evaluate()?); + } + } else { + // fill null in empty time solt + for (column, padding) in all_scalar.iter_mut().zip(padding_values.iter()) { + column.push(padding.clone()) + } } } ts_builder.append_slice(&align_ts); @@ -1176,14 +1205,16 @@ impl RangeSelectStream { ) in self.range_exec.iter().enumerate() { let time_series_data = - &mut all_scalar[i][start_index..start_index + align_ts_accumulator.len()]; + &mut all_scalar[i][start_index..start_index + align_ts.len()]; if let Some(data_type) = need_cast { cast_scalar_values(time_series_data, data_type)?; } - fill.apply_fill_strategy(&align_ts, time_series_data)?; + if let Some(fill) = fill { + fill.apply_fill_strategy(&align_ts, time_series_data)?; + } } - by_rows.resize(by_rows.len() + align_ts_accumulator.len(), row.row()); - start_index += align_ts_accumulator.len(); + by_rows.resize(by_rows.len() + align_ts.len(), row.row()); + start_index += align_ts.len(); } for column_scalar in all_scalar { columns.push(ScalarValue::iter_to_array(column_scalar)?); @@ -1304,7 +1335,7 @@ mod test { const TIME_INDEX_COLUMN: &str = "timestamp"; - fn prepare_test_data(is_float: bool) -> MemoryExec { + fn prepare_test_data(is_float: bool, is_gap: bool) -> MemoryExec { let schema = Arc::new(Schema::new(vec![ Field::new(TIME_INDEX_COLUMN, TimestampMillisecondType::DATA_TYPE, true), Field::new( @@ -1318,16 +1349,23 @@ mod test { ), Field::new("host", DataType::Utf8, true), ])); - let timestamp_column: Arc = Arc::new(TimestampMillisecondArray::from(vec![ - 0, 5_000, 10_000, 15_000, 20_000, // host 1 every 5s - 0, 5_000, 10_000, 15_000, 20_000, // host 2 every 5s - ])) as _; - let mut host = vec!["host1"; 5]; - host.extend(vec!["host2"; 5]); - let value_column: Arc = if is_float { - Arc::new(nullable_array!(Float64; - 0.0, null, 1.0, null, 2.0, // data for host 1 - 3.0, null, 4.0, null, 5.0 // data for host 2 + let timestamp_column: Arc = if !is_gap { + Arc::new(TimestampMillisecondArray::from(vec![ + 0, 5_000, 10_000, 15_000, 20_000, // host 1 every 5s + 0, 5_000, 10_000, 15_000, 20_000, // host 2 every 5s + ])) as _ + } else { + Arc::new(TimestampMillisecondArray::from(vec![ + 0, 15_000, // host 1 every 5s, missing data on 5_000, 10_000 + 0, 15_000, // host 2 every 5s, missing data on 5_000, 10_000 + ])) as _ + }; + let mut host = vec!["host1"; timestamp_column.len() / 2]; + host.extend(vec!["host2"; timestamp_column.len() / 2]); + let mut value_column: Arc = if is_gap { + Arc::new(nullable_array!(Int64; + 0, 6, // data for host 1 + 6, 12 // data for host 2 )) as _ } else { Arc::new(nullable_array!(Int64; @@ -1335,6 +1373,11 @@ mod test { 3, null, 4, null, 5 // data for host 2 )) as _ }; + if is_float { + value_column = + cast_with_options(&value_column, &DataType::Float64, &CastOptions::default()) + .unwrap(); + } let host_column: Arc = Arc::new(StringArray::from(host)) as _; let data = RecordBatch::try_new( schema.clone(), @@ -1349,8 +1392,9 @@ mod test { range1: Millisecond, range2: Millisecond, align: Millisecond, - fill: Fill, + fill: Option, is_float: bool, + is_gap: bool, expected: String, ) { let data_type = if is_float { @@ -1358,13 +1402,13 @@ mod test { } else { DataType::Int64 }; - let (need_cast, schema_data_type) = if !is_float && fill == Fill::Linear { + let (need_cast, schema_data_type) = if !is_float && matches!(fill, Some(Fill::Linear)) { // data_type = DataType::Float64; (Some(DataType::Float64), DataType::Float64) } else { (None, data_type.clone()) }; - let memory_exec = Arc::new(prepare_test_data(is_float)); + let memory_exec = Arc::new(prepare_test_data(is_float, is_gap)); let schema = Arc::new(Schema::new(vec![ Field::new("MIN(value)", schema_data_type.clone(), true), Field::new("MAX(value)", schema_data_type, true), @@ -1449,7 +1493,16 @@ mod test { \n| 3.0 | 3.0 | 1970-01-01T00:00:00 | host2 |\ \n+------------+------------+---------------------+-------+", ); - do_range_select_test(10_000, 10_000, 1_000_000, Fill::Null, true, expected).await; + do_range_select_test( + 10_000, + 10_000, + 1_000_000, + Some(Fill::Null), + true, + false, + expected, + ) + .await; } #[tokio::test] @@ -1472,7 +1525,16 @@ mod test { \n| 5.0 | 5.0 | 1970-01-01T00:00:20 | host2 |\ \n+------------+------------+---------------------+-------+", ); - do_range_select_test(10_000, 5_000, 5_000, Fill::Null, true, expected).await; + do_range_select_test( + 10_000, + 5_000, + 5_000, + Some(Fill::Null), + true, + false, + expected, + ) + .await; } #[tokio::test] @@ -1495,7 +1557,16 @@ mod test { \n| 5.0 | 5.0 | 1970-01-01T00:00:20 | host2 |\ \n+------------+------------+---------------------+-------+", ); - do_range_select_test(10_000, 5_000, 5_000, Fill::Prev, true, expected).await; + do_range_select_test( + 10_000, + 5_000, + 5_000, + Some(Fill::Prev), + true, + false, + expected, + ) + .await; } #[tokio::test] @@ -1518,7 +1589,16 @@ mod test { \n| 5.0 | 5.0 | 1970-01-01T00:00:20 | host2 |\ \n+------------+------------+---------------------+-------+", ); - do_range_select_test(10_000, 5_000, 5_000, Fill::Linear, true, expected).await; + do_range_select_test( + 10_000, + 5_000, + 5_000, + Some(Fill::Linear), + true, + false, + expected, + ) + .await; } #[tokio::test] @@ -1541,7 +1621,16 @@ mod test { \n| 5.0 | 5.0 | 1970-01-01T00:00:20 | host2 |\ \n+------------+------------+---------------------+-------+", ); - do_range_select_test(10_000, 5_000, 5_000, Fill::Linear, false, expected).await; + do_range_select_test( + 10_000, + 5_000, + 5_000, + Some(Fill::Linear), + false, + false, + expected, + ) + .await; } #[tokio::test] @@ -1568,7 +1657,101 @@ mod test { 10_000, 5_000, 5_000, - Fill::Const(ScalarValue::Float64(Some(6.6))), + Some(Fill::Const(ScalarValue::Float64(Some(6.6)))), + true, + false, + expected, + ) + .await; + } + + #[tokio::test] + async fn range_fill_gap() { + let expected = String::from( + "+------------+------------+---------------------+-------+\ + \n| MIN(value) | MAX(value) | timestamp | host |\ + \n+------------+------------+---------------------+-------+\ + \n| 0.0 | 0.0 | 1970-01-01T00:00:00 | host1 |\ + \n| 6.0 | 6.0 | 1970-01-01T00:00:15 | host1 |\ + \n| 6.0 | 6.0 | 1970-01-01T00:00:00 | host2 |\ + \n| 12.0 | 12.0 | 1970-01-01T00:00:15 | host2 |\ + \n+------------+------------+---------------------+-------+", + ); + do_range_select_test(5_000, 5_000, 5_000, None, true, true, expected).await; + let expected = String::from( + "+------------+------------+---------------------+-------+\ + \n| MIN(value) | MAX(value) | timestamp | host |\ + \n+------------+------------+---------------------+-------+\ + \n| 0.0 | 0.0 | 1970-01-01T00:00:00 | host1 |\ + \n| | | 1970-01-01T00:00:05 | host1 |\ + \n| | | 1970-01-01T00:00:10 | host1 |\ + \n| 6.0 | 6.0 | 1970-01-01T00:00:15 | host1 |\ + \n| 6.0 | 6.0 | 1970-01-01T00:00:00 | host2 |\ + \n| | | 1970-01-01T00:00:05 | host2 |\ + \n| | | 1970-01-01T00:00:10 | host2 |\ + \n| 12.0 | 12.0 | 1970-01-01T00:00:15 | host2 |\ + \n+------------+------------+---------------------+-------+", + ); + do_range_select_test(5_000, 5_000, 5_000, Some(Fill::Null), true, true, expected).await; + let expected = String::from( + "+------------+------------+---------------------+-------+\ + \n| MIN(value) | MAX(value) | timestamp | host |\ + \n+------------+------------+---------------------+-------+\ + \n| 0.0 | 0.0 | 1970-01-01T00:00:00 | host1 |\ + \n| 0.0 | 0.0 | 1970-01-01T00:00:05 | host1 |\ + \n| 0.0 | 0.0 | 1970-01-01T00:00:10 | host1 |\ + \n| 6.0 | 6.0 | 1970-01-01T00:00:15 | host1 |\ + \n| 6.0 | 6.0 | 1970-01-01T00:00:00 | host2 |\ + \n| 6.0 | 6.0 | 1970-01-01T00:00:05 | host2 |\ + \n| 6.0 | 6.0 | 1970-01-01T00:00:10 | host2 |\ + \n| 12.0 | 12.0 | 1970-01-01T00:00:15 | host2 |\ + \n+------------+------------+---------------------+-------+", + ); + do_range_select_test(5_000, 5_000, 5_000, Some(Fill::Prev), true, true, expected).await; + let expected = String::from( + "+------------+------------+---------------------+-------+\ + \n| MIN(value) | MAX(value) | timestamp | host |\ + \n+------------+------------+---------------------+-------+\ + \n| 0.0 | 0.0 | 1970-01-01T00:00:00 | host1 |\ + \n| 2.0 | 2.0 | 1970-01-01T00:00:05 | host1 |\ + \n| 4.0 | 4.0 | 1970-01-01T00:00:10 | host1 |\ + \n| 6.0 | 6.0 | 1970-01-01T00:00:15 | host1 |\ + \n| 6.0 | 6.0 | 1970-01-01T00:00:00 | host2 |\ + \n| 8.0 | 8.0 | 1970-01-01T00:00:05 | host2 |\ + \n| 10.0 | 10.0 | 1970-01-01T00:00:10 | host2 |\ + \n| 12.0 | 12.0 | 1970-01-01T00:00:15 | host2 |\ + \n+------------+------------+---------------------+-------+", + ); + do_range_select_test( + 5_000, + 5_000, + 5_000, + Some(Fill::Linear), + true, + true, + expected, + ) + .await; + let expected = String::from( + "+------------+------------+---------------------+-------+\ + \n| MIN(value) | MAX(value) | timestamp | host |\ + \n+------------+------------+---------------------+-------+\ + \n| 0.0 | 0.0 | 1970-01-01T00:00:00 | host1 |\ + \n| 6.0 | 6.0 | 1970-01-01T00:00:05 | host1 |\ + \n| 6.0 | 6.0 | 1970-01-01T00:00:10 | host1 |\ + \n| 6.0 | 6.0 | 1970-01-01T00:00:15 | host1 |\ + \n| 6.0 | 6.0 | 1970-01-01T00:00:00 | host2 |\ + \n| 6.0 | 6.0 | 1970-01-01T00:00:05 | host2 |\ + \n| 6.0 | 6.0 | 1970-01-01T00:00:10 | host2 |\ + \n| 12.0 | 12.0 | 1970-01-01T00:00:15 | host2 |\ + \n+------------+------------+---------------------+-------+", + ); + do_range_select_test( + 5_000, + 5_000, + 5_000, + Some(Fill::Const(ScalarValue::Float64(Some(6.0)))), + true, true, expected, ) @@ -1577,7 +1760,8 @@ mod test { #[test] fn fill_test() { - assert!(Fill::try_from_str("Linear", &DataType::UInt8).unwrap() == Fill::Linear); + assert!(Fill::try_from_str("", &DataType::UInt8).unwrap().is_none()); + assert!(Fill::try_from_str("Linear", &DataType::UInt8).unwrap() == Some(Fill::Linear)); assert_eq!( Fill::try_from_str("Linear", &DataType::Boolean) .unwrap_err() @@ -1598,7 +1782,7 @@ mod test { ); assert!( Fill::try_from_str("8", &DataType::UInt8).unwrap() - == Fill::Const(ScalarValue::UInt8(Some(8))) + == Some(Fill::Const(ScalarValue::UInt8(Some(8)))) ); let mut test1 = vec![ ScalarValue::UInt8(Some(8)), diff --git a/src/query/src/range_select/plan_rewrite.rs b/src/query/src/range_select/plan_rewrite.rs index d55fe65786..bab81279e7 100644 --- a/src/query/src/range_select/plan_rewrite.rs +++ b/src/query/src/range_select/plan_rewrite.rs @@ -216,7 +216,7 @@ impl<'a> TreeNodeRewriter for RangeExprRewriter<'a> { let mut data_type = range_expr.get_type(self.input_plan.schema())?; let mut need_cast = false; let fill = Fill::try_from_str(parse_str_expr(&func.args, 2)?, &data_type)?; - if matches!(fill, Fill::Linear) && data_type.is_integer() { + if matches!(fill, Some(Fill::Linear)) && data_type.is_integer() { data_type = DataType::Float64; need_cast = true; } @@ -224,12 +224,20 @@ impl<'a> TreeNodeRewriter for RangeExprRewriter<'a> { inconsistent_check!(self.align, self.align != Duration::default()); inconsistent_check!(self.align_to, self.align_to != 0); let range_fn = RangeFn { - name: format!( - "{} RANGE {} FILL {}", - range_expr.display_name()?, - parse_expr_to_string(&func.args, 1)?, - fill - ), + name: if let Some(fill) = &fill { + format!( + "{} RANGE {} FILL {}", + range_expr.display_name()?, + parse_expr_to_string(&func.args, 1)?, + fill + ) + } else { + format!( + "{} RANGE {}", + range_expr.display_name()?, + parse_expr_to_string(&func.args, 1)?, + ) + }, data_type, expr: range_expr, range, @@ -551,7 +559,7 @@ mod test { async fn range_no_project() { let query = r#"SELECT timestamp, tag_0, tag_1, avg(field_0 + field_1) RANGE '5m' FROM test ALIGN '1h' by (tag_0,tag_1);"#; let expected = String::from( - "RangeSelect: range_exprs=[AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL], align=3600000ms, align_to=0ms, align_by=[test.tag_0, test.tag_1], time_index=timestamp [timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8, AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL:Float64;N]\ + "RangeSelect: range_exprs=[AVG(test.field_0 + test.field_1) RANGE 5m], align=3600000ms, align_to=0ms, align_by=[test.tag_0, test.tag_1], time_index=timestamp [timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8, AVG(test.field_0 + test.field_1) RANGE 5m:Float64;N]\ \n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]" ); query_plan_compare(query, expected).await; @@ -561,8 +569,8 @@ mod test { async fn range_expr_calculation() { let query = r#"SELECT (avg(field_0 + field_1)/4) RANGE '5m' FROM test ALIGN '1h' by (tag_0,tag_1);"#; let expected = String::from( - "Projection: AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL / Int64(4) [AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL / Int64(4):Float64;N]\ - \n RangeSelect: range_exprs=[AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL], align=3600000ms, align_to=0ms, align_by=[test.tag_0, test.tag_1], time_index=timestamp [AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\ + "Projection: AVG(test.field_0 + test.field_1) RANGE 5m / Int64(4) [AVG(test.field_0 + test.field_1) RANGE 5m / Int64(4):Float64;N]\ + \n RangeSelect: range_exprs=[AVG(test.field_0 + test.field_1) RANGE 5m], align=3600000ms, align_to=0ms, align_by=[test.tag_0, test.tag_1], time_index=timestamp [AVG(test.field_0 + test.field_1) RANGE 5m:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\ \n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]" ); query_plan_compare(query, expected).await; @@ -573,8 +581,8 @@ mod test { let query = r#"SELECT (covar(field_0 + field_1, field_1)/4) RANGE '5m' FROM test ALIGN '1h';"#; let expected = String::from( - "Projection: COVARIANCE(test.field_0 + test.field_1,test.field_1) RANGE 5m FILL NULL / Int64(4) [COVARIANCE(test.field_0 + test.field_1,test.field_1) RANGE 5m FILL NULL / Int64(4):Float64;N]\ - \n RangeSelect: range_exprs=[COVARIANCE(test.field_0 + test.field_1,test.field_1) RANGE 5m FILL NULL], align=3600000ms, align_to=0ms, align_by=[test.tag_0, test.tag_1, test.tag_2, test.tag_3, test.tag_4], time_index=timestamp [COVARIANCE(test.field_0 + test.field_1,test.field_1) RANGE 5m FILL NULL:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8]\ + "Projection: COVARIANCE(test.field_0 + test.field_1,test.field_1) RANGE 5m / Int64(4) [COVARIANCE(test.field_0 + test.field_1,test.field_1) RANGE 5m / Int64(4):Float64;N]\ + \n RangeSelect: range_exprs=[COVARIANCE(test.field_0 + test.field_1,test.field_1) RANGE 5m], align=3600000ms, align_to=0ms, align_by=[test.tag_0, test.tag_1, test.tag_2, test.tag_3, test.tag_4], time_index=timestamp [COVARIANCE(test.field_0 + test.field_1,test.field_1) RANGE 5m:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8]\ \n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]" ); query_plan_compare(query, expected).await; @@ -621,8 +629,8 @@ mod test { async fn range_in_expr() { let query = r#"SELECT sin(avg(field_0 + field_1) RANGE '5m' + 1) FROM test ALIGN '1h' by (tag_0,tag_1);"#; let expected = String::from( - "Projection: sin(AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL + Int64(1)) [sin(AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL + Int64(1)):Float64;N]\ - \n RangeSelect: range_exprs=[AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL], align=3600000ms, align_to=0ms, align_by=[test.tag_0, test.tag_1], time_index=timestamp [AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\ + "Projection: sin(AVG(test.field_0 + test.field_1) RANGE 5m + Int64(1)) [sin(AVG(test.field_0 + test.field_1) RANGE 5m + Int64(1)):Float64;N]\ + \n RangeSelect: range_exprs=[AVG(test.field_0 + test.field_1) RANGE 5m], align=3600000ms, align_to=0ms, align_by=[test.tag_0, test.tag_1], time_index=timestamp [AVG(test.field_0 + test.field_1) RANGE 5m:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\ \n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]" ); query_plan_compare(query, expected).await; @@ -643,8 +651,8 @@ mod test { async fn deep_nest_range_expr() { let query = r#"SELECT round(sin(avg(field_0 + field_1) RANGE '5m' + 1)) FROM test ALIGN '1h' by (tag_0,tag_1);"#; let expected = String::from( - "Projection: round(sin(AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL + Int64(1))) [round(sin(AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL + Int64(1))):Float64;N]\ - \n RangeSelect: range_exprs=[AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL], align=3600000ms, align_to=0ms, align_by=[test.tag_0, test.tag_1], time_index=timestamp [AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\ + "Projection: round(sin(AVG(test.field_0 + test.field_1) RANGE 5m + Int64(1))) [round(sin(AVG(test.field_0 + test.field_1) RANGE 5m + Int64(1))):Float64;N]\ + \n RangeSelect: range_exprs=[AVG(test.field_0 + test.field_1) RANGE 5m], align=3600000ms, align_to=0ms, align_by=[test.tag_0, test.tag_1], time_index=timestamp [AVG(test.field_0 + test.field_1) RANGE 5m:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\ \n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]" ); query_plan_compare(query, expected).await; diff --git a/tests/cases/standalone/common/range/by.result b/tests/cases/standalone/common/range/by.result index 250cc6dd16..1665eec32e 100644 --- a/tests/cases/standalone/common/range/by.result +++ b/tests/cases/standalone/common/range/by.result @@ -23,47 +23,47 @@ Affected Rows: 10 -- Test by calculate SELECT ts, length(host), max(val) RANGE '5s' FROM host ALIGN '20s' BY (length(host)) ORDER BY ts; -+---------------------+-----------------------------+----------------------------------+ -| ts | character_length(host.host) | MAX(host.val) RANGE 5s FILL NULL | -+---------------------+-----------------------------+----------------------------------+ -| 1970-01-01T00:00:00 | 5 | 3 | -| 1970-01-01T00:00:20 | 5 | 5 | -+---------------------+-----------------------------+----------------------------------+ ++---------------------+-----------------------------+------------------------+ +| ts | character_length(host.host) | MAX(host.val) RANGE 5s | ++---------------------+-----------------------------+------------------------+ +| 1970-01-01T00:00:00 | 5 | 3 | +| 1970-01-01T00:00:20 | 5 | 5 | ++---------------------+-----------------------------+------------------------+ SELECT ts, max(val) RANGE '5s' FROM host ALIGN '20s' BY (2) ORDER BY ts; -+---------------------+----------------------------------+ -| ts | MAX(host.val) RANGE 5s FILL NULL | -+---------------------+----------------------------------+ -| 1970-01-01T00:00:00 | 3 | -| 1970-01-01T00:00:20 | 5 | -+---------------------+----------------------------------+ ++---------------------+------------------------+ +| ts | MAX(host.val) RANGE 5s | ++---------------------+------------------------+ +| 1970-01-01T00:00:00 | 3 | +| 1970-01-01T00:00:20 | 5 | ++---------------------+------------------------+ -- The user explicitly specifies that the aggregation key is empty. In this case, there is no aggregation key. All data will be aggregated into a group. -- Implement by rewrite `BY()` to `BY(1)` automatically through sqlparser. They are semantically equivalent. SELECT ts, max(val) RANGE '5s' FROM host ALIGN '20s' BY () ORDER BY ts; -+---------------------+----------------------------------+ -| ts | MAX(host.val) RANGE 5s FILL NULL | -+---------------------+----------------------------------+ -| 1970-01-01T00:00:00 | 3 | -| 1970-01-01T00:00:20 | 5 | -+---------------------+----------------------------------+ ++---------------------+------------------------+ +| ts | MAX(host.val) RANGE 5s | ++---------------------+------------------------+ +| 1970-01-01T00:00:00 | 3 | +| 1970-01-01T00:00:20 | 5 | ++---------------------+------------------------+ SELECT ts, length(host)::INT64 + 2, max(val) RANGE '5s' FROM host ALIGN '20s' BY (length(host)::INT64 + 2) ORDER BY ts; -+---------------------+----------------------------------------+----------------------------------+ -| ts | character_length(host.host) + Int64(2) | MAX(host.val) RANGE 5s FILL NULL | -+---------------------+----------------------------------------+----------------------------------+ -| 1970-01-01T00:00:00 | 7 | 3 | -| 1970-01-01T00:00:20 | 7 | 5 | -+---------------------+----------------------------------------+----------------------------------+ ++---------------------+----------------------------------------+------------------------+ +| ts | character_length(host.host) + Int64(2) | MAX(host.val) RANGE 5s | ++---------------------+----------------------------------------+------------------------+ +| 1970-01-01T00:00:00 | 7 | 3 | +| 1970-01-01T00:00:20 | 7 | 5 | ++---------------------+----------------------------------------+------------------------+ -- Test error -- project non-aggregation key SELECT ts, host, max(val) RANGE '5s' FROM host ALIGN '20s' BY () ORDER BY ts; -Error: 3001(EngineExecuteQuery), DataFusion error: No field named host.host. Valid fields are "MAX(host.val) RANGE 5s FILL NULL", host.ts, "Int64(1)". +Error: 3001(EngineExecuteQuery), DataFusion error: No field named host.host. Valid fields are "MAX(host.val) RANGE 5s", host.ts, "Int64(1)". DROP TABLE host; @@ -94,12 +94,12 @@ Affected Rows: 10 SELECT ts, max(val) RANGE '5s' FROM host ALIGN '20s' ORDER BY ts; -+---------------------+----------------------------------+ -| ts | MAX(host.val) RANGE 5s FILL NULL | -+---------------------+----------------------------------+ -| 1970-01-01T00:00:00 | 3 | -| 1970-01-01T00:00:20 | 5 | -+---------------------+----------------------------------+ ++---------------------+------------------------+ +| ts | MAX(host.val) RANGE 5s | ++---------------------+------------------------+ +| 1970-01-01T00:00:00 | 3 | +| 1970-01-01T00:00:20 | 5 | ++---------------------+------------------------+ DROP TABLE host; diff --git a/tests/cases/standalone/common/range/calculate.result b/tests/cases/standalone/common/range/calculate.result index e94a3228ba..a753a9268f 100644 --- a/tests/cases/standalone/common/range/calculate.result +++ b/tests/cases/standalone/common/range/calculate.result @@ -23,151 +23,151 @@ Affected Rows: 10 -- Test range expr calculate SELECT ts, host, covar(val::DOUBLE, val::DOUBLE) RANGE '20s' FROM host ALIGN '10s' ORDER BY host, ts; -+---------------------+-------+---------------------------------------------------+ -| ts | host | COVARIANCE(host.val,host.val) RANGE 20s FILL NULL | -+---------------------+-------+---------------------------------------------------+ -| 1969-12-31T23:59:50 | host1 | | -| 1970-01-01T00:00:00 | host1 | 0.5 | -| 1970-01-01T00:00:10 | host1 | 0.5 | -| 1970-01-01T00:00:20 | host1 | | -| 1969-12-31T23:59:50 | host2 | | -| 1970-01-01T00:00:00 | host2 | 0.5 | -| 1970-01-01T00:00:10 | host2 | 0.5 | -| 1970-01-01T00:00:20 | host2 | | -+---------------------+-------+---------------------------------------------------+ ++---------------------+-------+-----------------------------------------+ +| ts | host | COVARIANCE(host.val,host.val) RANGE 20s | ++---------------------+-------+-----------------------------------------+ +| 1969-12-31T23:59:50 | host1 | | +| 1970-01-01T00:00:00 | host1 | 0.5 | +| 1970-01-01T00:00:10 | host1 | 0.5 | +| 1970-01-01T00:00:20 | host1 | | +| 1969-12-31T23:59:50 | host2 | | +| 1970-01-01T00:00:00 | host2 | 0.5 | +| 1970-01-01T00:00:10 | host2 | 0.5 | +| 1970-01-01T00:00:20 | host2 | | ++---------------------+-------+-----------------------------------------+ SELECT ts, host, 2 * min(val) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+---------------------------------------------+ -| ts | host | Int64(2) * MIN(host.val) RANGE 5s FILL NULL | -+---------------------+-------+---------------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 0 | -| 1970-01-01T00:00:05 | host1 | | -| 1970-01-01T00:00:10 | host1 | 2 | -| 1970-01-01T00:00:15 | host1 | | -| 1970-01-01T00:00:20 | host1 | 4 | -| 1970-01-01T00:00:00 | host2 | 6 | -| 1970-01-01T00:00:05 | host2 | | -| 1970-01-01T00:00:10 | host2 | 8 | -| 1970-01-01T00:00:15 | host2 | | -| 1970-01-01T00:00:20 | host2 | 10 | -+---------------------+-------+---------------------------------------------+ ++---------------------+-------+-----------------------------------+ +| ts | host | Int64(2) * MIN(host.val) RANGE 5s | ++---------------------+-------+-----------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | +| 1970-01-01T00:00:05 | host1 | | +| 1970-01-01T00:00:10 | host1 | 2 | +| 1970-01-01T00:00:15 | host1 | | +| 1970-01-01T00:00:20 | host1 | 4 | +| 1970-01-01T00:00:00 | host2 | 6 | +| 1970-01-01T00:00:05 | host2 | | +| 1970-01-01T00:00:10 | host2 | 8 | +| 1970-01-01T00:00:15 | host2 | | +| 1970-01-01T00:00:20 | host2 | 10 | ++---------------------+-------+-----------------------------------+ SELECT ts, host, min(val * 2) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+---------------------------------------------+ -| ts | host | MIN(host.val * Int64(2)) RANGE 5s FILL NULL | -+---------------------+-------+---------------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 0 | -| 1970-01-01T00:00:05 | host1 | | -| 1970-01-01T00:00:10 | host1 | 2 | -| 1970-01-01T00:00:15 | host1 | | -| 1970-01-01T00:00:20 | host1 | 4 | -| 1970-01-01T00:00:00 | host2 | 6 | -| 1970-01-01T00:00:05 | host2 | | -| 1970-01-01T00:00:10 | host2 | 8 | -| 1970-01-01T00:00:15 | host2 | | -| 1970-01-01T00:00:20 | host2 | 10 | -+---------------------+-------+---------------------------------------------+ ++---------------------+-------+-----------------------------------+ +| ts | host | MIN(host.val * Int64(2)) RANGE 5s | ++---------------------+-------+-----------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | +| 1970-01-01T00:00:05 | host1 | | +| 1970-01-01T00:00:10 | host1 | 2 | +| 1970-01-01T00:00:15 | host1 | | +| 1970-01-01T00:00:20 | host1 | 4 | +| 1970-01-01T00:00:00 | host2 | 6 | +| 1970-01-01T00:00:05 | host2 | | +| 1970-01-01T00:00:10 | host2 | 8 | +| 1970-01-01T00:00:15 | host2 | | +| 1970-01-01T00:00:20 | host2 | 10 | ++---------------------+-------+-----------------------------------+ SELECT ts, host, min(val::DOUBLE) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+----------------------------------+ -| ts | host | MIN(host.val) RANGE 5s FILL NULL | -+---------------------+-------+----------------------------------+ -| 1970-01-01T00:00:00 | host1 | 0.0 | -| 1970-01-01T00:00:05 | host1 | | -| 1970-01-01T00:00:10 | host1 | 1.0 | -| 1970-01-01T00:00:15 | host1 | | -| 1970-01-01T00:00:20 | host1 | 2.0 | -| 1970-01-01T00:00:00 | host2 | 3.0 | -| 1970-01-01T00:00:05 | host2 | | -| 1970-01-01T00:00:10 | host2 | 4.0 | -| 1970-01-01T00:00:15 | host2 | | -| 1970-01-01T00:00:20 | host2 | 5.0 | -+---------------------+-------+----------------------------------+ ++---------------------+-------+------------------------+ +| ts | host | MIN(host.val) RANGE 5s | ++---------------------+-------+------------------------+ +| 1970-01-01T00:00:00 | host1 | 0.0 | +| 1970-01-01T00:00:05 | host1 | | +| 1970-01-01T00:00:10 | host1 | 1.0 | +| 1970-01-01T00:00:15 | host1 | | +| 1970-01-01T00:00:20 | host1 | 2.0 | +| 1970-01-01T00:00:00 | host2 | 3.0 | +| 1970-01-01T00:00:05 | host2 | | +| 1970-01-01T00:00:10 | host2 | 4.0 | +| 1970-01-01T00:00:15 | host2 | | +| 1970-01-01T00:00:20 | host2 | 5.0 | ++---------------------+-------+------------------------+ SELECT ts, host, min(floor(val::DOUBLE)) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+-----------------------------------------+ -| ts | host | MIN(floor(host.val)) RANGE 5s FILL NULL | -+---------------------+-------+-----------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 0.0 | -| 1970-01-01T00:00:05 | host1 | | -| 1970-01-01T00:00:10 | host1 | 1.0 | -| 1970-01-01T00:00:15 | host1 | | -| 1970-01-01T00:00:20 | host1 | 2.0 | -| 1970-01-01T00:00:00 | host2 | 3.0 | -| 1970-01-01T00:00:05 | host2 | | -| 1970-01-01T00:00:10 | host2 | 4.0 | -| 1970-01-01T00:00:15 | host2 | | -| 1970-01-01T00:00:20 | host2 | 5.0 | -+---------------------+-------+-----------------------------------------+ ++---------------------+-------+-------------------------------+ +| ts | host | MIN(floor(host.val)) RANGE 5s | ++---------------------+-------+-------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0.0 | +| 1970-01-01T00:00:05 | host1 | | +| 1970-01-01T00:00:10 | host1 | 1.0 | +| 1970-01-01T00:00:15 | host1 | | +| 1970-01-01T00:00:20 | host1 | 2.0 | +| 1970-01-01T00:00:00 | host2 | 3.0 | +| 1970-01-01T00:00:05 | host2 | | +| 1970-01-01T00:00:10 | host2 | 4.0 | +| 1970-01-01T00:00:15 | host2 | | +| 1970-01-01T00:00:20 | host2 | 5.0 | ++---------------------+-------+-------------------------------+ SELECT ts, host, floor(min(val) RANGE '5s') FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+-----------------------------------------+ -| ts | host | floor(MIN(host.val) RANGE 5s FILL NULL) | -+---------------------+-------+-----------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 0.0 | -| 1970-01-01T00:00:05 | host1 | | -| 1970-01-01T00:00:10 | host1 | 1.0 | -| 1970-01-01T00:00:15 | host1 | | -| 1970-01-01T00:00:20 | host1 | 2.0 | -| 1970-01-01T00:00:00 | host2 | 3.0 | -| 1970-01-01T00:00:05 | host2 | | -| 1970-01-01T00:00:10 | host2 | 4.0 | -| 1970-01-01T00:00:15 | host2 | | -| 1970-01-01T00:00:20 | host2 | 5.0 | -+---------------------+-------+-----------------------------------------+ ++---------------------+-------+-------------------------------+ +| ts | host | floor(MIN(host.val) RANGE 5s) | ++---------------------+-------+-------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0.0 | +| 1970-01-01T00:00:05 | host1 | | +| 1970-01-01T00:00:10 | host1 | 1.0 | +| 1970-01-01T00:00:15 | host1 | | +| 1970-01-01T00:00:20 | host1 | 2.0 | +| 1970-01-01T00:00:00 | host2 | 3.0 | +| 1970-01-01T00:00:05 | host2 | | +| 1970-01-01T00:00:10 | host2 | 4.0 | +| 1970-01-01T00:00:15 | host2 | | +| 1970-01-01T00:00:20 | host2 | 5.0 | ++---------------------+-------+-------------------------------+ -- Test complex range expr calculate SELECT ts, host, (min(val) + max(val)) RANGE '20s' + 1.0 FROM host ALIGN '10s' ORDER BY host, ts; -+---------------------+-------+------------------------------------------------------------------------------------+ -| ts | host | MIN(host.val) RANGE 20s FILL NULL + MAX(host.val) RANGE 20s FILL NULL + Float64(1) | -+---------------------+-------+------------------------------------------------------------------------------------+ -| 1969-12-31T23:59:50 | host1 | 1.0 | -| 1970-01-01T00:00:00 | host1 | 2.0 | -| 1970-01-01T00:00:10 | host1 | 4.0 | -| 1970-01-01T00:00:20 | host1 | 5.0 | -| 1969-12-31T23:59:50 | host2 | 7.0 | -| 1970-01-01T00:00:00 | host2 | 8.0 | -| 1970-01-01T00:00:10 | host2 | 10.0 | -| 1970-01-01T00:00:20 | host2 | 11.0 | -+---------------------+-------+------------------------------------------------------------------------------------+ ++---------------------+-------+----------------------------------------------------------------+ +| ts | host | MIN(host.val) RANGE 20s + MAX(host.val) RANGE 20s + Float64(1) | ++---------------------+-------+----------------------------------------------------------------+ +| 1969-12-31T23:59:50 | host1 | 1.0 | +| 1970-01-01T00:00:00 | host1 | 2.0 | +| 1970-01-01T00:00:10 | host1 | 4.0 | +| 1970-01-01T00:00:20 | host1 | 5.0 | +| 1969-12-31T23:59:50 | host2 | 7.0 | +| 1970-01-01T00:00:00 | host2 | 8.0 | +| 1970-01-01T00:00:10 | host2 | 10.0 | +| 1970-01-01T00:00:20 | host2 | 11.0 | ++---------------------+-------+----------------------------------------------------------------+ SELECT ts, host, covar(ceil(val::DOUBLE), floor(val::DOUBLE)) RANGE '20s' FROM host ALIGN '10s' ORDER BY host, ts; -+---------------------+-------+----------------------------------------------------------------+ -| ts | host | COVARIANCE(ceil(host.val),floor(host.val)) RANGE 20s FILL NULL | -+---------------------+-------+----------------------------------------------------------------+ -| 1969-12-31T23:59:50 | host1 | | -| 1970-01-01T00:00:00 | host1 | 0.5 | -| 1970-01-01T00:00:10 | host1 | 0.5 | -| 1970-01-01T00:00:20 | host1 | | -| 1969-12-31T23:59:50 | host2 | | -| 1970-01-01T00:00:00 | host2 | 0.5 | -| 1970-01-01T00:00:10 | host2 | 0.5 | -| 1970-01-01T00:00:20 | host2 | | -+---------------------+-------+----------------------------------------------------------------+ ++---------------------+-------+------------------------------------------------------+ +| ts | host | COVARIANCE(ceil(host.val),floor(host.val)) RANGE 20s | ++---------------------+-------+------------------------------------------------------+ +| 1969-12-31T23:59:50 | host1 | | +| 1970-01-01T00:00:00 | host1 | 0.5 | +| 1970-01-01T00:00:10 | host1 | 0.5 | +| 1970-01-01T00:00:20 | host1 | | +| 1969-12-31T23:59:50 | host2 | | +| 1970-01-01T00:00:00 | host2 | 0.5 | +| 1970-01-01T00:00:10 | host2 | 0.5 | +| 1970-01-01T00:00:20 | host2 | | ++---------------------+-------+------------------------------------------------------+ SELECT ts, host, floor(cos(ceil(sin(min(val) RANGE '5s')))) FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+---------------------------------------------------------+ -| ts | host | floor(cos(ceil(sin(MIN(host.val) RANGE 5s FILL NULL)))) | -+---------------------+-------+---------------------------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 1.0 | -| 1970-01-01T00:00:05 | host1 | | -| 1970-01-01T00:00:10 | host1 | 0.0 | -| 1970-01-01T00:00:15 | host1 | | -| 1970-01-01T00:00:20 | host1 | 0.0 | -| 1970-01-01T00:00:00 | host2 | 0.0 | -| 1970-01-01T00:00:05 | host2 | | -| 1970-01-01T00:00:10 | host2 | 1.0 | -| 1970-01-01T00:00:15 | host2 | | -| 1970-01-01T00:00:20 | host2 | 1.0 | -+---------------------+-------+---------------------------------------------------------+ ++---------------------+-------+-----------------------------------------------+ +| ts | host | floor(cos(ceil(sin(MIN(host.val) RANGE 5s)))) | ++---------------------+-------+-----------------------------------------------+ +| 1970-01-01T00:00:00 | host1 | 1.0 | +| 1970-01-01T00:00:05 | host1 | | +| 1970-01-01T00:00:10 | host1 | 0.0 | +| 1970-01-01T00:00:15 | host1 | | +| 1970-01-01T00:00:20 | host1 | 0.0 | +| 1970-01-01T00:00:00 | host2 | 0.0 | +| 1970-01-01T00:00:05 | host2 | | +| 1970-01-01T00:00:10 | host2 | 1.0 | +| 1970-01-01T00:00:15 | host2 | | +| 1970-01-01T00:00:20 | host2 | 1.0 | ++---------------------+-------+-----------------------------------------------+ SELECT ts, host, gcd(CAST(max(floor(val::DOUBLE)) RANGE '10s' FILL PREV as INT64) * 4, max(val * 4) RANGE '10s' FILL PREV) * length(host) + 1 FROM host ALIGN '5s' ORDER BY host, ts; diff --git a/tests/cases/standalone/common/range/error.result b/tests/cases/standalone/common/range/error.result index cb86b0c462..a33558589b 100644 --- a/tests/cases/standalone/common/range/error.result +++ b/tests/cases/standalone/common/range/error.result @@ -41,7 +41,7 @@ Error: 2000(InvalidSyntax), sql parser error: Illegal Range select, no RANGE key SELECT min(val) RANGE '10s', max(val) FROM host ALIGN '5s'; -Error: 3001(EngineExecuteQuery), DataFusion error: No field named "MAX(host.val)". Valid fields are "MIN(host.val) RANGE 10s FILL NULL", host.ts, host.host. +Error: 3001(EngineExecuteQuery), DataFusion error: No field named "MAX(host.val)". Valid fields are "MIN(host.val) RANGE 10s", host.ts, host.host. SELECT min(val) * 2 RANGE '10s' FROM host ALIGN '5s'; @@ -73,10 +73,6 @@ SELECT rank() OVER (PARTITION BY host ORDER BY ts DESC) RANGE '10s' FROM host AL Error: 2000(InvalidSyntax), Range Query: Window functions is not allowed in Range Query -- 2.6 invalid fill -SELECT min(val) RANGE '5s', min(val) RANGE '5s' FILL NULL FROM host ALIGN '5s'; - -Error: 3001(EngineExecuteQuery), DataFusion error: Schema contains duplicate unqualified field name "MIN(host.val) RANGE 5s FILL NULL" - SELECT min(val) RANGE '5s' FROM host ALIGN '5s' FILL 3.0; Error: 3000(PlanQuery), DataFusion error: Error during planning: 3.0 is not a valid fill option, fail to convert to a const value. { Arrow error: Cast error: Cannot cast string '3.0' to value of Int64 type } @@ -96,7 +92,7 @@ Error: 2000(InvalidSyntax), Range Query: Can't use 0 as align in Range Query SELECT min(val) RANGE (INTERVAL '0' day) FROM host ALIGN '5s'; -Error: 2000(InvalidSyntax), Range Query: Invalid Range expr `MIN(host.val) RANGE IntervalMonthDayNano("0") FILL NULL`, Can't use 0 as range in Range Query +Error: 2000(InvalidSyntax), Range Query: Invalid Range expr `MIN(host.val) RANGE IntervalMonthDayNano("0")`, Can't use 0 as range in Range Query DROP TABLE host; diff --git a/tests/cases/standalone/common/range/error.sql b/tests/cases/standalone/common/range/error.sql index cda3569f1f..ba3d1f63e2 100644 --- a/tests/cases/standalone/common/range/error.sql +++ b/tests/cases/standalone/common/range/error.sql @@ -54,8 +54,6 @@ SELECT rank() OVER (PARTITION BY host ORDER BY ts DESC) RANGE '10s' FROM host AL -- 2.6 invalid fill -SELECT min(val) RANGE '5s', min(val) RANGE '5s' FILL NULL FROM host ALIGN '5s'; - SELECT min(val) RANGE '5s' FROM host ALIGN '5s' FILL 3.0; -- 2.7 zero align/range diff --git a/tests/cases/standalone/common/range/fill.result b/tests/cases/standalone/common/range/fill.result index 00a371b0fd..739aec6dd6 100644 --- a/tests/cases/standalone/common/range/fill.result +++ b/tests/cases/standalone/common/range/fill.result @@ -20,23 +20,23 @@ INSERT INTO TABLE host VALUES Affected Rows: 10 --- Test Fill +-- Test Fill when aggregate result is null SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+----------------------------------+ -| ts | host | MIN(host.val) RANGE 5s FILL NULL | -+---------------------+-------+----------------------------------+ -| 1970-01-01T00:00:00 | host1 | 0 | -| 1970-01-01T00:00:05 | host1 | | -| 1970-01-01T00:00:10 | host1 | 1 | -| 1970-01-01T00:00:15 | host1 | | -| 1970-01-01T00:00:20 | host1 | 2 | -| 1970-01-01T00:00:00 | host2 | 3 | -| 1970-01-01T00:00:05 | host2 | | -| 1970-01-01T00:00:10 | host2 | 4 | -| 1970-01-01T00:00:15 | host2 | | -| 1970-01-01T00:00:20 | host2 | 5 | -+---------------------+-------+----------------------------------+ ++---------------------+-------+------------------------+ +| ts | host | MIN(host.val) RANGE 5s | ++---------------------+-------+------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | +| 1970-01-01T00:00:05 | host1 | | +| 1970-01-01T00:00:10 | host1 | 1 | +| 1970-01-01T00:00:15 | host1 | | +| 1970-01-01T00:00:20 | host1 | 2 | +| 1970-01-01T00:00:00 | host2 | 3 | +| 1970-01-01T00:00:05 | host2 | | +| 1970-01-01T00:00:10 | host2 | 4 | +| 1970-01-01T00:00:15 | host2 | | +| 1970-01-01T00:00:20 | host2 | 5 | ++---------------------+-------+------------------------+ SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s' FILL NULL ORDER BY host, ts; @@ -57,54 +57,231 @@ SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s' FILL NULL ORDER BY hos SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL 6 FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+----------------------------------+-------------------------------+ -| ts | host | MIN(host.val) RANGE 5s FILL NULL | MIN(host.val) RANGE 5s FILL 6 | -+---------------------+-------+----------------------------------+-------------------------------+ -| 1970-01-01T00:00:00 | host1 | 0 | 0 | -| 1970-01-01T00:00:05 | host1 | | 6 | -| 1970-01-01T00:00:10 | host1 | 1 | 1 | -| 1970-01-01T00:00:15 | host1 | | 6 | -| 1970-01-01T00:00:20 | host1 | 2 | 2 | -| 1970-01-01T00:00:00 | host2 | 3 | 3 | -| 1970-01-01T00:00:05 | host2 | | 6 | -| 1970-01-01T00:00:10 | host2 | 4 | 4 | -| 1970-01-01T00:00:15 | host2 | | 6 | -| 1970-01-01T00:00:20 | host2 | 5 | 5 | -+---------------------+-------+----------------------------------+-------------------------------+ ++---------------------+-------+------------------------+-------------------------------+ +| ts | host | MIN(host.val) RANGE 5s | MIN(host.val) RANGE 5s FILL 6 | ++---------------------+-------+------------------------+-------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | 0 | +| 1970-01-01T00:00:05 | host1 | | 6 | +| 1970-01-01T00:00:10 | host1 | 1 | 1 | +| 1970-01-01T00:00:15 | host1 | | 6 | +| 1970-01-01T00:00:20 | host1 | 2 | 2 | +| 1970-01-01T00:00:00 | host2 | 3 | 3 | +| 1970-01-01T00:00:05 | host2 | | 6 | +| 1970-01-01T00:00:10 | host2 | 4 | 4 | +| 1970-01-01T00:00:15 | host2 | | 6 | +| 1970-01-01T00:00:20 | host2 | 5 | 5 | ++---------------------+-------+------------------------+-------------------------------+ SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL PREV FROM host ALIGN '5s'ORDER BY host, ts; -+---------------------+-------+----------------------------------+----------------------------------+ -| ts | host | MIN(host.val) RANGE 5s FILL NULL | MIN(host.val) RANGE 5s FILL PREV | -+---------------------+-------+----------------------------------+----------------------------------+ -| 1970-01-01T00:00:00 | host1 | 0 | 0 | -| 1970-01-01T00:00:05 | host1 | | 0 | -| 1970-01-01T00:00:10 | host1 | 1 | 1 | -| 1970-01-01T00:00:15 | host1 | | 1 | -| 1970-01-01T00:00:20 | host1 | 2 | 2 | -| 1970-01-01T00:00:00 | host2 | 3 | 3 | -| 1970-01-01T00:00:05 | host2 | | 3 | -| 1970-01-01T00:00:10 | host2 | 4 | 4 | -| 1970-01-01T00:00:15 | host2 | | 4 | -| 1970-01-01T00:00:20 | host2 | 5 | 5 | -+---------------------+-------+----------------------------------+----------------------------------+ ++---------------------+-------+------------------------+----------------------------------+ +| ts | host | MIN(host.val) RANGE 5s | MIN(host.val) RANGE 5s FILL PREV | ++---------------------+-------+------------------------+----------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | 0 | +| 1970-01-01T00:00:05 | host1 | | 0 | +| 1970-01-01T00:00:10 | host1 | 1 | 1 | +| 1970-01-01T00:00:15 | host1 | | 1 | +| 1970-01-01T00:00:20 | host1 | 2 | 2 | +| 1970-01-01T00:00:00 | host2 | 3 | 3 | +| 1970-01-01T00:00:05 | host2 | | 3 | +| 1970-01-01T00:00:10 | host2 | 4 | 4 | +| 1970-01-01T00:00:15 | host2 | | 4 | +| 1970-01-01T00:00:20 | host2 | 5 | 5 | ++---------------------+-------+------------------------+----------------------------------+ SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL LINEAR FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+----------------------------------+------------------------------------+ -| ts | host | MIN(host.val) RANGE 5s FILL NULL | MIN(host.val) RANGE 5s FILL LINEAR | -+---------------------+-------+----------------------------------+------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 0 | 0.0 | -| 1970-01-01T00:00:05 | host1 | | 0.5 | -| 1970-01-01T00:00:10 | host1 | 1 | 1.0 | -| 1970-01-01T00:00:15 | host1 | | 1.5 | -| 1970-01-01T00:00:20 | host1 | 2 | 2.0 | -| 1970-01-01T00:00:00 | host2 | 3 | 3.0 | -| 1970-01-01T00:00:05 | host2 | | 3.5 | -| 1970-01-01T00:00:10 | host2 | 4 | 4.0 | -| 1970-01-01T00:00:15 | host2 | | 4.5 | -| 1970-01-01T00:00:20 | host2 | 5 | 5.0 | -+---------------------+-------+----------------------------------+------------------------------------+ ++---------------------+-------+------------------------+------------------------------------+ +| ts | host | MIN(host.val) RANGE 5s | MIN(host.val) RANGE 5s FILL LINEAR | ++---------------------+-------+------------------------+------------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | 0.0 | +| 1970-01-01T00:00:05 | host1 | | 0.5 | +| 1970-01-01T00:00:10 | host1 | 1 | 1.0 | +| 1970-01-01T00:00:15 | host1 | | 1.5 | +| 1970-01-01T00:00:20 | host1 | 2 | 2.0 | +| 1970-01-01T00:00:00 | host2 | 3 | 3.0 | +| 1970-01-01T00:00:05 | host2 | | 3.5 | +| 1970-01-01T00:00:10 | host2 | 4 | 4.0 | +| 1970-01-01T00:00:15 | host2 | | 4.5 | +| 1970-01-01T00:00:20 | host2 | 5 | 5.0 | ++---------------------+-------+------------------------+------------------------------------+ + +DROP TABLE host; + +Affected Rows: 0 + +-- Test Fill when time slot data is missing +CREATE TABLE host ( + ts timestamp(3) time index, + host STRING PRIMARY KEY, + val BIGINT, +); + +Affected Rows: 0 + +INSERT INTO TABLE host VALUES + (0, 'host1', 0), + (1000, 'host1', 1), + (2000, 'host1', 2), + -- missing data for 5000, 10000 + (15000, 'host1', 6), + (16000, 'host1', 7), + (17000, 'host1', 8), + (0, 'host2', 6), + (1000, 'host2', 7), + (2000, 'host2', 8), + -- missing data for 5000, 10000 + (15000, 'host2', 12), + (16000, 'host2', 13), + (17000, 'host2', 14); + +Affected Rows: 12 + +SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; + ++---------------------+-------+------------------------+ +| ts | host | MIN(host.val) RANGE 5s | ++---------------------+-------+------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | +| 1970-01-01T00:00:15 | host1 | 6 | +| 1970-01-01T00:00:00 | host2 | 6 | +| 1970-01-01T00:00:15 | host2 | 12 | ++---------------------+-------+------------------------+ + +SELECT ts, host, min(val) RANGE '5s' FILL NULL FROM host ALIGN '5s' ORDER BY host, ts; + ++---------------------+-------+----------------------------------+ +| ts | host | MIN(host.val) RANGE 5s FILL NULL | ++---------------------+-------+----------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | +| 1970-01-01T00:00:05 | host1 | | +| 1970-01-01T00:00:10 | host1 | | +| 1970-01-01T00:00:15 | host1 | 6 | +| 1970-01-01T00:00:00 | host2 | 6 | +| 1970-01-01T00:00:05 | host2 | | +| 1970-01-01T00:00:10 | host2 | | +| 1970-01-01T00:00:15 | host2 | 12 | ++---------------------+-------+----------------------------------+ + +SELECT ts, host, min(val) RANGE '5s' FILL PREV FROM host ALIGN '5s' ORDER BY host, ts; + ++---------------------+-------+----------------------------------+ +| ts | host | MIN(host.val) RANGE 5s FILL PREV | ++---------------------+-------+----------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | +| 1970-01-01T00:00:05 | host1 | 0 | +| 1970-01-01T00:00:10 | host1 | 0 | +| 1970-01-01T00:00:15 | host1 | 6 | +| 1970-01-01T00:00:00 | host2 | 6 | +| 1970-01-01T00:00:05 | host2 | 6 | +| 1970-01-01T00:00:10 | host2 | 6 | +| 1970-01-01T00:00:15 | host2 | 12 | ++---------------------+-------+----------------------------------+ + +SELECT ts, host, min(val) RANGE '5s' FILL LINEAR FROM host ALIGN '5s' ORDER BY host, ts; + ++---------------------+-------+------------------------------------+ +| ts | host | MIN(host.val) RANGE 5s FILL LINEAR | ++---------------------+-------+------------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0.0 | +| 1970-01-01T00:00:05 | host1 | 2.0 | +| 1970-01-01T00:00:10 | host1 | 4.0 | +| 1970-01-01T00:00:15 | host1 | 6.0 | +| 1970-01-01T00:00:00 | host2 | 6.0 | +| 1970-01-01T00:00:05 | host2 | 8.0 | +| 1970-01-01T00:00:10 | host2 | 10.0 | +| 1970-01-01T00:00:15 | host2 | 12.0 | ++---------------------+-------+------------------------------------+ + +SELECT ts, host, min(val) RANGE '5s' FILL 6 FROM host ALIGN '5s' ORDER BY host, ts; + ++---------------------+-------+-------------------------------+ +| ts | host | MIN(host.val) RANGE 5s FILL 6 | ++---------------------+-------+-------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | +| 1970-01-01T00:00:05 | host1 | 6 | +| 1970-01-01T00:00:10 | host1 | 6 | +| 1970-01-01T00:00:15 | host1 | 6 | +| 1970-01-01T00:00:00 | host2 | 6 | +| 1970-01-01T00:00:05 | host2 | 6 | +| 1970-01-01T00:00:10 | host2 | 6 | +| 1970-01-01T00:00:15 | host2 | 12 | ++---------------------+-------+-------------------------------+ + +SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s' FILL NULL ORDER BY host, ts; + ++---------------------+-------+----------------------------------+ +| ts | host | MIN(host.val) RANGE 5s FILL NULL | ++---------------------+-------+----------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | +| 1970-01-01T00:00:05 | host1 | | +| 1970-01-01T00:00:10 | host1 | | +| 1970-01-01T00:00:15 | host1 | 6 | +| 1970-01-01T00:00:00 | host2 | 6 | +| 1970-01-01T00:00:05 | host2 | | +| 1970-01-01T00:00:10 | host2 | | +| 1970-01-01T00:00:15 | host2 | 12 | ++---------------------+-------+----------------------------------+ + +SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL NULL FROM host ALIGN '5s' ORDER BY host, ts; + ++---------------------+-------+------------------------+----------------------------------+ +| ts | host | MIN(host.val) RANGE 5s | MIN(host.val) RANGE 5s FILL NULL | ++---------------------+-------+------------------------+----------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | 0 | +| 1970-01-01T00:00:05 | host1 | | | +| 1970-01-01T00:00:10 | host1 | | | +| 1970-01-01T00:00:15 | host1 | 6 | 6 | +| 1970-01-01T00:00:00 | host2 | 6 | 6 | +| 1970-01-01T00:00:05 | host2 | | | +| 1970-01-01T00:00:10 | host2 | | | +| 1970-01-01T00:00:15 | host2 | 12 | 12 | ++---------------------+-------+------------------------+----------------------------------+ + +SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL 6 FROM host ALIGN '5s' ORDER BY host, ts; + ++---------------------+-------+------------------------+-------------------------------+ +| ts | host | MIN(host.val) RANGE 5s | MIN(host.val) RANGE 5s FILL 6 | ++---------------------+-------+------------------------+-------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | 0 | +| 1970-01-01T00:00:05 | host1 | | 6 | +| 1970-01-01T00:00:10 | host1 | | 6 | +| 1970-01-01T00:00:15 | host1 | 6 | 6 | +| 1970-01-01T00:00:00 | host2 | 6 | 6 | +| 1970-01-01T00:00:05 | host2 | | 6 | +| 1970-01-01T00:00:10 | host2 | | 6 | +| 1970-01-01T00:00:15 | host2 | 12 | 12 | ++---------------------+-------+------------------------+-------------------------------+ + +SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL PREV FROM host ALIGN '5s'ORDER BY host, ts; + ++---------------------+-------+------------------------+----------------------------------+ +| ts | host | MIN(host.val) RANGE 5s | MIN(host.val) RANGE 5s FILL PREV | ++---------------------+-------+------------------------+----------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | 0 | +| 1970-01-01T00:00:05 | host1 | | 0 | +| 1970-01-01T00:00:10 | host1 | | 0 | +| 1970-01-01T00:00:15 | host1 | 6 | 6 | +| 1970-01-01T00:00:00 | host2 | 6 | 6 | +| 1970-01-01T00:00:05 | host2 | | 6 | +| 1970-01-01T00:00:10 | host2 | | 6 | +| 1970-01-01T00:00:15 | host2 | 12 | 12 | ++---------------------+-------+------------------------+----------------------------------+ + +SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL LINEAR FROM host ALIGN '5s' ORDER BY host, ts; + ++---------------------+-------+------------------------+------------------------------------+ +| ts | host | MIN(host.val) RANGE 5s | MIN(host.val) RANGE 5s FILL LINEAR | ++---------------------+-------+------------------------+------------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | 0.0 | +| 1970-01-01T00:00:05 | host1 | | 2.0 | +| 1970-01-01T00:00:10 | host1 | | 4.0 | +| 1970-01-01T00:00:15 | host1 | 6 | 6.0 | +| 1970-01-01T00:00:00 | host2 | 6 | 6.0 | +| 1970-01-01T00:00:05 | host2 | | 8.0 | +| 1970-01-01T00:00:10 | host2 | | 10.0 | +| 1970-01-01T00:00:15 | host2 | 12 | 12.0 | ++---------------------+-------+------------------------+------------------------------------+ DROP TABLE host; diff --git a/tests/cases/standalone/common/range/fill.sql b/tests/cases/standalone/common/range/fill.sql index 66ae5f247c..9136237749 100644 --- a/tests/cases/standalone/common/range/fill.sql +++ b/tests/cases/standalone/common/range/fill.sql @@ -16,7 +16,7 @@ INSERT INTO TABLE host VALUES (15000, 'host2', null), (20000, 'host2', 5); --- Test Fill +-- Test Fill when aggregate result is null SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; @@ -29,3 +29,50 @@ SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL PREV FROM host AL SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL LINEAR FROM host ALIGN '5s' ORDER BY host, ts; DROP TABLE host; + +-- Test Fill when time slot data is missing + +CREATE TABLE host ( + ts timestamp(3) time index, + host STRING PRIMARY KEY, + val BIGINT, +); + +INSERT INTO TABLE host VALUES + (0, 'host1', 0), + (1000, 'host1', 1), + (2000, 'host1', 2), + -- missing data for 5000, 10000 + (15000, 'host1', 6), + (16000, 'host1', 7), + (17000, 'host1', 8), + + (0, 'host2', 6), + (1000, 'host2', 7), + (2000, 'host2', 8), + -- missing data for 5000, 10000 + (15000, 'host2', 12), + (16000, 'host2', 13), + (17000, 'host2', 14); + +SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; + +SELECT ts, host, min(val) RANGE '5s' FILL NULL FROM host ALIGN '5s' ORDER BY host, ts; + +SELECT ts, host, min(val) RANGE '5s' FILL PREV FROM host ALIGN '5s' ORDER BY host, ts; + +SELECT ts, host, min(val) RANGE '5s' FILL LINEAR FROM host ALIGN '5s' ORDER BY host, ts; + +SELECT ts, host, min(val) RANGE '5s' FILL 6 FROM host ALIGN '5s' ORDER BY host, ts; + +SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s' FILL NULL ORDER BY host, ts; + +SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL NULL FROM host ALIGN '5s' ORDER BY host, ts; + +SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL 6 FROM host ALIGN '5s' ORDER BY host, ts; + +SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL PREV FROM host ALIGN '5s'ORDER BY host, ts; + +SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL LINEAR FROM host ALIGN '5s' ORDER BY host, ts; + +DROP TABLE host; diff --git a/tests/cases/standalone/common/range/interval.result b/tests/cases/standalone/common/range/interval.result index 99247049ad..10da6cf90d 100644 --- a/tests/cases/standalone/common/range/interval.result +++ b/tests/cases/standalone/common/range/interval.result @@ -20,25 +20,25 @@ Affected Rows: 8 SELECT ts, host, min(val) RANGE (INTERVAL '1 year') FROM host ALIGN (INTERVAL '1 year') ORDER BY host, ts; -+---------------------+-------+--------------------------------------------------------------------------------------+ -| ts | host | MIN(host.val) RANGE IntervalMonthDayNano("950737950171172051122527404032") FILL NULL | -+---------------------+-------+--------------------------------------------------------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 0 | -| 1970-12-27T00:00:00 | host1 | 2 | -| 1970-01-01T00:00:00 | host2 | 4 | -| 1970-12-27T00:00:00 | host2 | 6 | -+---------------------+-------+--------------------------------------------------------------------------------------+ ++---------------------+-------+----------------------------------------------------------------------------+ +| ts | host | MIN(host.val) RANGE IntervalMonthDayNano("950737950171172051122527404032") | ++---------------------+-------+----------------------------------------------------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | +| 1970-12-27T00:00:00 | host1 | 2 | +| 1970-01-01T00:00:00 | host2 | 4 | +| 1970-12-27T00:00:00 | host2 | 6 | ++---------------------+-------+----------------------------------------------------------------------------+ SELECT ts, host, min(val) RANGE (INTERVAL '1' year) FROM host ALIGN (INTERVAL '1' year) ORDER BY host, ts; -+---------------------+-------+--------------------------------------------------------------------------------------+ -| ts | host | MIN(host.val) RANGE IntervalMonthDayNano("950737950171172051122527404032") FILL NULL | -+---------------------+-------+--------------------------------------------------------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 0 | -| 1970-12-27T00:00:00 | host1 | 2 | -| 1970-01-01T00:00:00 | host2 | 4 | -| 1970-12-27T00:00:00 | host2 | 6 | -+---------------------+-------+--------------------------------------------------------------------------------------+ ++---------------------+-------+----------------------------------------------------------------------------+ +| ts | host | MIN(host.val) RANGE IntervalMonthDayNano("950737950171172051122527404032") | ++---------------------+-------+----------------------------------------------------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | +| 1970-12-27T00:00:00 | host1 | 2 | +| 1970-01-01T00:00:00 | host2 | 4 | +| 1970-12-27T00:00:00 | host2 | 6 | ++---------------------+-------+----------------------------------------------------------------------------+ DROP TABLE host; diff --git a/tests/cases/standalone/common/range/nest.result b/tests/cases/standalone/common/range/nest.result index bf749adc81..4c5adc8032 100644 --- a/tests/cases/standalone/common/range/nest.result +++ b/tests/cases/standalone/common/range/nest.result @@ -35,15 +35,15 @@ SELECT ts, host, foo FROM (SELECT ts, host, min(val) RANGE '5s' AS foo FROM host SELECT ts, b, min(c) RANGE '5s' FROM (SELECT ts, host AS b, val AS c FROM host WHERE host = 'host1') ALIGN '5s' BY (b) ORDER BY b, ts; -+---------------------+-------+---------------------------+ -| ts | b | MIN(c) RANGE 5s FILL NULL | -+---------------------+-------+---------------------------+ -| 1970-01-01T00:00:00 | host1 | 0 | -| 1970-01-01T00:00:05 | host1 | | -| 1970-01-01T00:00:10 | host1 | 1 | -| 1970-01-01T00:00:15 | host1 | | -| 1970-01-01T00:00:20 | host1 | 2 | -+---------------------+-------+---------------------------+ ++---------------------+-------+-----------------+ +| ts | b | MIN(c) RANGE 5s | ++---------------------+-------+-----------------+ +| 1970-01-01T00:00:00 | host1 | 0 | +| 1970-01-01T00:00:05 | host1 | | +| 1970-01-01T00:00:10 | host1 | 1 | +| 1970-01-01T00:00:15 | host1 | | +| 1970-01-01T00:00:20 | host1 | 2 | ++---------------------+-------+-----------------+ -- Test EXPLAIN and ANALYZE -- SQLNESS REPLACE (-+) - @@ -55,9 +55,9 @@ EXPLAIN SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s'; +-+-+ | plan_type_| plan_| +-+-+ -| logical_plan_| RangeSelect: range_exprs=[MIN(host.val) RANGE 5s FILL NULL], align=5000ms, align_to=0ms, align_by=[host.host], time_index=ts | +| logical_plan_| RangeSelect: range_exprs=[MIN(host.val) RANGE 5s], align=5000ms, align_to=0ms, align_by=[host.host], time_index=ts | |_|_MergeScan [is_placeholder=false]_| -| physical_plan | RangeSelectExec: range_expr=[MIN(host.val) RANGE 5s FILL NULL], align=5000ms, align_to=0ms, align_by=[host@1], time_index=ts | +| physical_plan | RangeSelectExec: range_expr=[MIN(host.val) RANGE 5s], align=5000ms, align_to=0ms, align_by=[host@1], time_index=ts | |_|_MergeScanExec: REDACTED |_|_| +-+-+ @@ -71,7 +71,7 @@ EXPLAIN ANALYZE SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s'; +-+-+ | plan_type_| plan_| +-+-+ -| Plan with Metrics | RangeSelectExec: range_expr=[MIN(host.val) RANGE 5s FILL NULL], align=5000ms, align_to=0ms, align_by=[host@1], time_index=ts, REDACTED +| Plan with Metrics | RangeSelectExec: range_expr=[MIN(host.val) RANGE 5s], align=5000ms, align_to=0ms, align_by=[host@1], time_index=ts, REDACTED |_|_MergeScanExec: REDACTED |_|_| +-+-+ diff --git a/tests/cases/standalone/common/range/precisions.result b/tests/cases/standalone/common/range/precisions.result index e70f4a5c0a..1b54091c17 100644 --- a/tests/cases/standalone/common/range/precisions.result +++ b/tests/cases/standalone/common/range/precisions.result @@ -23,20 +23,20 @@ Affected Rows: 10 -- Test on Timestamps of different precisions SELECT ts, host, min(val) RANGE '5s' FROM host_sec ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+--------------------------------------+ -| ts | host | MIN(host_sec.val) RANGE 5s FILL NULL | -+---------------------+-------+--------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 0.0 | -| 1970-01-01T00:00:05 | host1 | | -| 1970-01-01T00:00:10 | host1 | 1.0 | -| 1970-01-01T00:00:15 | host1 | | -| 1970-01-01T00:00:20 | host1 | 2.0 | -| 1970-01-01T00:00:00 | host2 | 3.0 | -| 1970-01-01T00:00:05 | host2 | | -| 1970-01-01T00:00:10 | host2 | 4.0 | -| 1970-01-01T00:00:15 | host2 | | -| 1970-01-01T00:00:20 | host2 | 5.0 | -+---------------------+-------+--------------------------------------+ ++---------------------+-------+----------------------------+ +| ts | host | MIN(host_sec.val) RANGE 5s | ++---------------------+-------+----------------------------+ +| 1970-01-01T00:00:00 | host1 | 0.0 | +| 1970-01-01T00:00:05 | host1 | | +| 1970-01-01T00:00:10 | host1 | 1.0 | +| 1970-01-01T00:00:15 | host1 | | +| 1970-01-01T00:00:20 | host1 | 2.0 | +| 1970-01-01T00:00:00 | host2 | 3.0 | +| 1970-01-01T00:00:05 | host2 | | +| 1970-01-01T00:00:10 | host2 | 4.0 | +| 1970-01-01T00:00:15 | host2 | | +| 1970-01-01T00:00:20 | host2 | 5.0 | ++---------------------+-------+----------------------------+ DROP TABLE host_sec; diff --git a/tests/cases/standalone/common/range/special_aggr.result b/tests/cases/standalone/common/range/special_aggr.result index b3eff8f5a9..6fab9998d4 100644 --- a/tests/cases/standalone/common/range/special_aggr.result +++ b/tests/cases/standalone/common/range/special_aggr.result @@ -43,173 +43,173 @@ Affected Rows: 30 SELECT ts, host, first_value(val) RANGE '5s', last_value(val) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+------------------------------------------+-----------------------------------------+ -| ts | host | FIRST_VALUE(host.val) RANGE 5s FILL NULL | LAST_VALUE(host.val) RANGE 5s FILL NULL | -+---------------------+-------+------------------------------------------+-----------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 0 | 2 | -| 1970-01-01T00:00:05 | host1 | | | -| 1970-01-01T00:00:10 | host1 | | 5 | -| 1970-01-01T00:00:15 | host1 | 6 | 7 | -| 1970-01-01T00:00:20 | host1 | 8 | | -| 1970-01-01T00:00:00 | host2 | 0 | 2 | -| 1970-01-01T00:00:05 | host2 | | | -| 1970-01-01T00:00:10 | host2 | | 5 | -| 1970-01-01T00:00:15 | host2 | 6 | 7 | -| 1970-01-01T00:00:20 | host2 | 8 | | -+---------------------+-------+------------------------------------------+-----------------------------------------+ ++---------------------+-------+--------------------------------+-------------------------------+ +| ts | host | FIRST_VALUE(host.val) RANGE 5s | LAST_VALUE(host.val) RANGE 5s | ++---------------------+-------+--------------------------------+-------------------------------+ +| 1970-01-01T00:00:00 | host1 | 0 | 2 | +| 1970-01-01T00:00:05 | host1 | | | +| 1970-01-01T00:00:10 | host1 | | 5 | +| 1970-01-01T00:00:15 | host1 | 6 | 7 | +| 1970-01-01T00:00:20 | host1 | 8 | | +| 1970-01-01T00:00:00 | host2 | 0 | 2 | +| 1970-01-01T00:00:05 | host2 | | | +| 1970-01-01T00:00:10 | host2 | | 5 | +| 1970-01-01T00:00:15 | host2 | 6 | 7 | +| 1970-01-01T00:00:20 | host2 | 8 | | ++---------------------+-------+--------------------------------+-------------------------------+ SELECT ts, host, first_value(addon ORDER BY val DESC) RANGE '5s', last_value(addon ORDER BY val DESC) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+---------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ -| ts | host | FIRST_VALUE(host.addon) ORDER BY [host.val DESC NULLS FIRST] RANGE 5s FILL NULL | LAST_VALUE(host.addon) ORDER BY [host.val DESC NULLS FIRST] RANGE 5s FILL NULL | -+---------------------+-------+---------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 3 | 1 | -| 1970-01-01T00:00:05 | host1 | 4 | 4 | -| 1970-01-01T00:00:10 | host1 | 7 | 8 | -| 1970-01-01T00:00:15 | host1 | 11 | 10 | -| 1970-01-01T00:00:20 | host1 | 15 | 13 | -| 1970-01-01T00:00:00 | host2 | 18 | 16 | -| 1970-01-01T00:00:05 | host2 | 19 | 19 | -| 1970-01-01T00:00:10 | host2 | 22 | 23 | -| 1970-01-01T00:00:15 | host2 | 26 | 25 | -| 1970-01-01T00:00:20 | host2 | 30 | 28 | -+---------------------+-------+---------------------------------------------------------------------------------+--------------------------------------------------------------------------------+ ++---------------------+-------+-----------------------------------------------------------------------+----------------------------------------------------------------------+ +| ts | host | FIRST_VALUE(host.addon) ORDER BY [host.val DESC NULLS FIRST] RANGE 5s | LAST_VALUE(host.addon) ORDER BY [host.val DESC NULLS FIRST] RANGE 5s | ++---------------------+-------+-----------------------------------------------------------------------+----------------------------------------------------------------------+ +| 1970-01-01T00:00:00 | host1 | 3 | 1 | +| 1970-01-01T00:00:05 | host1 | 4 | 4 | +| 1970-01-01T00:00:10 | host1 | 7 | 8 | +| 1970-01-01T00:00:15 | host1 | 11 | 10 | +| 1970-01-01T00:00:20 | host1 | 15 | 13 | +| 1970-01-01T00:00:00 | host2 | 18 | 16 | +| 1970-01-01T00:00:05 | host2 | 19 | 19 | +| 1970-01-01T00:00:10 | host2 | 22 | 23 | +| 1970-01-01T00:00:15 | host2 | 26 | 25 | +| 1970-01-01T00:00:20 | host2 | 30 | 28 | ++---------------------+-------+-----------------------------------------------------------------------+----------------------------------------------------------------------+ SELECT ts, host, first_value(addon ORDER BY val DESC NULLS LAST) RANGE '5s', last_value(addon ORDER BY val DESC NULLS LAST) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+--------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| ts | host | FIRST_VALUE(host.addon) ORDER BY [host.val DESC NULLS LAST] RANGE 5s FILL NULL | LAST_VALUE(host.addon) ORDER BY [host.val DESC NULLS LAST] RANGE 5s FILL NULL | -+---------------------+-------+--------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 3 | 1 | -| 1970-01-01T00:00:05 | host1 | 4 | 4 | -| 1970-01-01T00:00:10 | host1 | 9 | 7 | -| 1970-01-01T00:00:15 | host1 | 12 | 11 | -| 1970-01-01T00:00:20 | host1 | 14 | 15 | -| 1970-01-01T00:00:00 | host2 | 18 | 16 | -| 1970-01-01T00:00:05 | host2 | 19 | 19 | -| 1970-01-01T00:00:10 | host2 | 24 | 22 | -| 1970-01-01T00:00:15 | host2 | 27 | 26 | -| 1970-01-01T00:00:20 | host2 | 29 | 30 | -+---------------------+-------+--------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ ++---------------------+-------+----------------------------------------------------------------------+---------------------------------------------------------------------+ +| ts | host | FIRST_VALUE(host.addon) ORDER BY [host.val DESC NULLS LAST] RANGE 5s | LAST_VALUE(host.addon) ORDER BY [host.val DESC NULLS LAST] RANGE 5s | ++---------------------+-------+----------------------------------------------------------------------+---------------------------------------------------------------------+ +| 1970-01-01T00:00:00 | host1 | 3 | 1 | +| 1970-01-01T00:00:05 | host1 | 4 | 4 | +| 1970-01-01T00:00:10 | host1 | 9 | 7 | +| 1970-01-01T00:00:15 | host1 | 12 | 11 | +| 1970-01-01T00:00:20 | host1 | 14 | 15 | +| 1970-01-01T00:00:00 | host2 | 18 | 16 | +| 1970-01-01T00:00:05 | host2 | 19 | 19 | +| 1970-01-01T00:00:10 | host2 | 24 | 22 | +| 1970-01-01T00:00:15 | host2 | 27 | 26 | +| 1970-01-01T00:00:20 | host2 | 29 | 30 | ++---------------------+-------+----------------------------------------------------------------------+---------------------------------------------------------------------+ SELECT ts, host, first_value(addon ORDER BY val ASC) RANGE '5s', last_value(addon ORDER BY val ASC) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+-------------------------------------------------------------------------------+------------------------------------------------------------------------------+ -| ts | host | FIRST_VALUE(host.addon) ORDER BY [host.val ASC NULLS LAST] RANGE 5s FILL NULL | LAST_VALUE(host.addon) ORDER BY [host.val ASC NULLS LAST] RANGE 5s FILL NULL | -+---------------------+-------+-------------------------------------------------------------------------------+------------------------------------------------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 1 | 3 | -| 1970-01-01T00:00:05 | host1 | 4 | 4 | -| 1970-01-01T00:00:10 | host1 | 8 | 7 | -| 1970-01-01T00:00:15 | host1 | 10 | 11 | -| 1970-01-01T00:00:20 | host1 | 13 | 15 | -| 1970-01-01T00:00:00 | host2 | 16 | 18 | -| 1970-01-01T00:00:05 | host2 | 19 | 19 | -| 1970-01-01T00:00:10 | host2 | 23 | 22 | -| 1970-01-01T00:00:15 | host2 | 25 | 26 | -| 1970-01-01T00:00:20 | host2 | 28 | 30 | -+---------------------+-------+-------------------------------------------------------------------------------+------------------------------------------------------------------------------+ ++---------------------+-------+---------------------------------------------------------------------+--------------------------------------------------------------------+ +| ts | host | FIRST_VALUE(host.addon) ORDER BY [host.val ASC NULLS LAST] RANGE 5s | LAST_VALUE(host.addon) ORDER BY [host.val ASC NULLS LAST] RANGE 5s | ++---------------------+-------+---------------------------------------------------------------------+--------------------------------------------------------------------+ +| 1970-01-01T00:00:00 | host1 | 1 | 3 | +| 1970-01-01T00:00:05 | host1 | 4 | 4 | +| 1970-01-01T00:00:10 | host1 | 8 | 7 | +| 1970-01-01T00:00:15 | host1 | 10 | 11 | +| 1970-01-01T00:00:20 | host1 | 13 | 15 | +| 1970-01-01T00:00:00 | host2 | 16 | 18 | +| 1970-01-01T00:00:05 | host2 | 19 | 19 | +| 1970-01-01T00:00:10 | host2 | 23 | 22 | +| 1970-01-01T00:00:15 | host2 | 25 | 26 | +| 1970-01-01T00:00:20 | host2 | 28 | 30 | ++---------------------+-------+---------------------------------------------------------------------+--------------------------------------------------------------------+ SELECT ts, host, first_value(addon ORDER BY val ASC NULLS FIRST) RANGE '5s', last_value(addon ORDER BY val ASC NULLS FIRST) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+--------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| ts | host | FIRST_VALUE(host.addon) ORDER BY [host.val ASC NULLS FIRST] RANGE 5s FILL NULL | LAST_VALUE(host.addon) ORDER BY [host.val ASC NULLS FIRST] RANGE 5s FILL NULL | -+---------------------+-------+--------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 1 | 3 | -| 1970-01-01T00:00:05 | host1 | 4 | 4 | -| 1970-01-01T00:00:10 | host1 | 7 | 9 | -| 1970-01-01T00:00:15 | host1 | 11 | 12 | -| 1970-01-01T00:00:20 | host1 | 15 | 14 | -| 1970-01-01T00:00:00 | host2 | 16 | 18 | -| 1970-01-01T00:00:05 | host2 | 19 | 19 | -| 1970-01-01T00:00:10 | host2 | 22 | 24 | -| 1970-01-01T00:00:15 | host2 | 26 | 27 | -| 1970-01-01T00:00:20 | host2 | 30 | 29 | -+---------------------+-------+--------------------------------------------------------------------------------+-------------------------------------------------------------------------------+ ++---------------------+-------+----------------------------------------------------------------------+---------------------------------------------------------------------+ +| ts | host | FIRST_VALUE(host.addon) ORDER BY [host.val ASC NULLS FIRST] RANGE 5s | LAST_VALUE(host.addon) ORDER BY [host.val ASC NULLS FIRST] RANGE 5s | ++---------------------+-------+----------------------------------------------------------------------+---------------------------------------------------------------------+ +| 1970-01-01T00:00:00 | host1 | 1 | 3 | +| 1970-01-01T00:00:05 | host1 | 4 | 4 | +| 1970-01-01T00:00:10 | host1 | 7 | 9 | +| 1970-01-01T00:00:15 | host1 | 11 | 12 | +| 1970-01-01T00:00:20 | host1 | 15 | 14 | +| 1970-01-01T00:00:00 | host2 | 16 | 18 | +| 1970-01-01T00:00:05 | host2 | 19 | 19 | +| 1970-01-01T00:00:10 | host2 | 22 | 24 | +| 1970-01-01T00:00:15 | host2 | 26 | 27 | +| 1970-01-01T00:00:20 | host2 | 30 | 29 | ++---------------------+-------+----------------------------------------------------------------------+---------------------------------------------------------------------+ SELECT ts, host, first_value(addon ORDER BY val ASC, ts ASC) RANGE '5s', last_value(addon ORDER BY val ASC, ts ASC) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+-------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+ -| ts | host | FIRST_VALUE(host.addon) ORDER BY [host.val ASC NULLS LAST, host.ts ASC NULLS LAST] RANGE 5s FILL NULL | LAST_VALUE(host.addon) ORDER BY [host.val ASC NULLS LAST, host.ts ASC NULLS LAST] RANGE 5s FILL NULL | -+---------------------+-------+-------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 1 | 3 | -| 1970-01-01T00:00:05 | host1 | 4 | 6 | -| 1970-01-01T00:00:10 | host1 | 8 | 7 | -| 1970-01-01T00:00:15 | host1 | 10 | 11 | -| 1970-01-01T00:00:20 | host1 | 13 | 15 | -| 1970-01-01T00:00:00 | host2 | 16 | 18 | -| 1970-01-01T00:00:05 | host2 | 19 | 21 | -| 1970-01-01T00:00:10 | host2 | 23 | 22 | -| 1970-01-01T00:00:15 | host2 | 25 | 26 | -| 1970-01-01T00:00:20 | host2 | 28 | 30 | -+---------------------+-------+-------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------+ ++---------------------+-------+---------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------+ +| ts | host | FIRST_VALUE(host.addon) ORDER BY [host.val ASC NULLS LAST, host.ts ASC NULLS LAST] RANGE 5s | LAST_VALUE(host.addon) ORDER BY [host.val ASC NULLS LAST, host.ts ASC NULLS LAST] RANGE 5s | ++---------------------+-------+---------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------+ +| 1970-01-01T00:00:00 | host1 | 1 | 3 | +| 1970-01-01T00:00:05 | host1 | 4 | 6 | +| 1970-01-01T00:00:10 | host1 | 8 | 7 | +| 1970-01-01T00:00:15 | host1 | 10 | 11 | +| 1970-01-01T00:00:20 | host1 | 13 | 15 | +| 1970-01-01T00:00:00 | host2 | 16 | 18 | +| 1970-01-01T00:00:05 | host2 | 19 | 21 | +| 1970-01-01T00:00:10 | host2 | 23 | 22 | +| 1970-01-01T00:00:15 | host2 | 25 | 26 | +| 1970-01-01T00:00:20 | host2 | 28 | 30 | ++---------------------+-------+---------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------+ SELECT ts, host, count(val) RANGE '5s'FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+------------------------------------+ -| ts | host | COUNT(host.val) RANGE 5s FILL NULL | -+---------------------+-------+------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 3 | -| 1970-01-01T00:00:05 | host1 | 0 | -| 1970-01-01T00:00:10 | host1 | 2 | -| 1970-01-01T00:00:15 | host1 | 2 | -| 1970-01-01T00:00:20 | host1 | 2 | -| 1970-01-01T00:00:00 | host2 | 3 | -| 1970-01-01T00:00:05 | host2 | 0 | -| 1970-01-01T00:00:10 | host2 | 2 | -| 1970-01-01T00:00:15 | host2 | 2 | -| 1970-01-01T00:00:20 | host2 | 2 | -+---------------------+-------+------------------------------------+ ++---------------------+-------+--------------------------+ +| ts | host | COUNT(host.val) RANGE 5s | ++---------------------+-------+--------------------------+ +| 1970-01-01T00:00:00 | host1 | 3 | +| 1970-01-01T00:00:05 | host1 | 0 | +| 1970-01-01T00:00:10 | host1 | 2 | +| 1970-01-01T00:00:15 | host1 | 2 | +| 1970-01-01T00:00:20 | host1 | 2 | +| 1970-01-01T00:00:00 | host2 | 3 | +| 1970-01-01T00:00:05 | host2 | 0 | +| 1970-01-01T00:00:10 | host2 | 2 | +| 1970-01-01T00:00:15 | host2 | 2 | +| 1970-01-01T00:00:20 | host2 | 2 | ++---------------------+-------+--------------------------+ SELECT ts, host, count(distinct val) RANGE '5s'FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+---------------------------------------------+ -| ts | host | COUNT(DISTINCT host.val) RANGE 5s FILL NULL | -+---------------------+-------+---------------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 3 | -| 1970-01-01T00:00:05 | host1 | 0 | -| 1970-01-01T00:00:10 | host1 | 2 | -| 1970-01-01T00:00:15 | host1 | 2 | -| 1970-01-01T00:00:20 | host1 | 2 | -| 1970-01-01T00:00:00 | host2 | 3 | -| 1970-01-01T00:00:05 | host2 | 0 | -| 1970-01-01T00:00:10 | host2 | 2 | -| 1970-01-01T00:00:15 | host2 | 2 | -| 1970-01-01T00:00:20 | host2 | 2 | -+---------------------+-------+---------------------------------------------+ ++---------------------+-------+-----------------------------------+ +| ts | host | COUNT(DISTINCT host.val) RANGE 5s | ++---------------------+-------+-----------------------------------+ +| 1970-01-01T00:00:00 | host1 | 3 | +| 1970-01-01T00:00:05 | host1 | 0 | +| 1970-01-01T00:00:10 | host1 | 2 | +| 1970-01-01T00:00:15 | host1 | 2 | +| 1970-01-01T00:00:20 | host1 | 2 | +| 1970-01-01T00:00:00 | host2 | 3 | +| 1970-01-01T00:00:05 | host2 | 0 | +| 1970-01-01T00:00:10 | host2 | 2 | +| 1970-01-01T00:00:15 | host2 | 2 | +| 1970-01-01T00:00:20 | host2 | 2 | ++---------------------+-------+-----------------------------------+ SELECT ts, host, count(*) RANGE '5s'FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+-----------------------------+ -| ts | host | COUNT(*) RANGE 5s FILL NULL | -+---------------------+-------+-----------------------------+ -| 1970-01-01T00:00:00 | host1 | 3 | -| 1970-01-01T00:00:05 | host1 | 3 | -| 1970-01-01T00:00:10 | host1 | 3 | -| 1970-01-01T00:00:15 | host1 | 3 | -| 1970-01-01T00:00:20 | host1 | 3 | -| 1970-01-01T00:00:00 | host2 | 3 | -| 1970-01-01T00:00:05 | host2 | 3 | -| 1970-01-01T00:00:10 | host2 | 3 | -| 1970-01-01T00:00:15 | host2 | 3 | -| 1970-01-01T00:00:20 | host2 | 3 | -+---------------------+-------+-----------------------------+ ++---------------------+-------+-------------------+ +| ts | host | COUNT(*) RANGE 5s | ++---------------------+-------+-------------------+ +| 1970-01-01T00:00:00 | host1 | 3 | +| 1970-01-01T00:00:05 | host1 | 3 | +| 1970-01-01T00:00:10 | host1 | 3 | +| 1970-01-01T00:00:15 | host1 | 3 | +| 1970-01-01T00:00:20 | host1 | 3 | +| 1970-01-01T00:00:00 | host2 | 3 | +| 1970-01-01T00:00:05 | host2 | 3 | +| 1970-01-01T00:00:10 | host2 | 3 | +| 1970-01-01T00:00:15 | host2 | 3 | +| 1970-01-01T00:00:20 | host2 | 3 | ++---------------------+-------+-------------------+ SELECT ts, host, count(distinct *) RANGE '5s'FROM host ALIGN '5s' ORDER BY host, ts; -+---------------------+-------+--------------------------------------+ -| ts | host | COUNT(DISTINCT *) RANGE 5s FILL NULL | -+---------------------+-------+--------------------------------------+ -| 1970-01-01T00:00:00 | host1 | 3 | -| 1970-01-01T00:00:05 | host1 | 3 | -| 1970-01-01T00:00:10 | host1 | 3 | -| 1970-01-01T00:00:15 | host1 | 3 | -| 1970-01-01T00:00:20 | host1 | 3 | -| 1970-01-01T00:00:00 | host2 | 3 | -| 1970-01-01T00:00:05 | host2 | 3 | -| 1970-01-01T00:00:10 | host2 | 3 | -| 1970-01-01T00:00:15 | host2 | 3 | -| 1970-01-01T00:00:20 | host2 | 3 | -+---------------------+-------+--------------------------------------+ ++---------------------+-------+----------------------------+ +| ts | host | COUNT(DISTINCT *) RANGE 5s | ++---------------------+-------+----------------------------+ +| 1970-01-01T00:00:00 | host1 | 3 | +| 1970-01-01T00:00:05 | host1 | 3 | +| 1970-01-01T00:00:10 | host1 | 3 | +| 1970-01-01T00:00:15 | host1 | 3 | +| 1970-01-01T00:00:20 | host1 | 3 | +| 1970-01-01T00:00:00 | host2 | 3 | +| 1970-01-01T00:00:05 | host2 | 3 | +| 1970-01-01T00:00:10 | host2 | 3 | +| 1970-01-01T00:00:15 | host2 | 3 | +| 1970-01-01T00:00:20 | host2 | 3 | ++---------------------+-------+----------------------------+ -- Test error first_value/last_value SELECT ts, host, first_value(val, val) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts; @@ -241,11 +241,11 @@ Affected Rows: 3 SELECT ts, first_value(val ORDER BY addon ASC) RANGE '5s', last_value(val ORDER BY addon ASC) RANGE '5s' FROM host ALIGN '5s'; -+---------------------+-------------------------------------------------------------------------------+------------------------------------------------------------------------------+ -| ts | FIRST_VALUE(host.val) ORDER BY [host.addon ASC NULLS LAST] RANGE 5s FILL NULL | LAST_VALUE(host.val) ORDER BY [host.addon ASC NULLS LAST] RANGE 5s FILL NULL | -+---------------------+-------------------------------------------------------------------------------+------------------------------------------------------------------------------+ -| 1970-01-01T00:00:00 | 2 | 0 | -+---------------------+-------------------------------------------------------------------------------+------------------------------------------------------------------------------+ ++---------------------+---------------------------------------------------------------------+--------------------------------------------------------------------+ +| ts | FIRST_VALUE(host.val) ORDER BY [host.addon ASC NULLS LAST] RANGE 5s | LAST_VALUE(host.val) ORDER BY [host.addon ASC NULLS LAST] RANGE 5s | ++---------------------+---------------------------------------------------------------------+--------------------------------------------------------------------+ +| 1970-01-01T00:00:00 | 2 | 0 | ++---------------------+---------------------------------------------------------------------+--------------------------------------------------------------------+ DROP TABLE host; diff --git a/tests/cases/standalone/common/range/to.result b/tests/cases/standalone/common/range/to.result index dffb2da519..a2bfb3de4c 100644 --- a/tests/cases/standalone/common/range/to.result +++ b/tests/cases/standalone/common/range/to.result @@ -20,14 +20,14 @@ Affected Rows: 8 SELECT ts, host, min(val) RANGE '1d' FROM host ALIGN '1d' ORDER BY host, ts; -+---------------------+-------+----------------------------------+ -| ts | host | MIN(host.val) RANGE 1d FILL NULL | -+---------------------+-------+----------------------------------+ -| 2024-01-23T00:00:00 | host1 | 0 | -| 2024-01-24T00:00:00 | host1 | 2 | -| 2024-01-23T00:00:00 | host2 | 4 | -| 2024-01-24T00:00:00 | host2 | 6 | -+---------------------+-------+----------------------------------+ ++---------------------+-------+------------------------+ +| ts | host | MIN(host.val) RANGE 1d | ++---------------------+-------+------------------------+ +| 2024-01-23T00:00:00 | host1 | 0 | +| 2024-01-24T00:00:00 | host1 | 2 | +| 2024-01-23T00:00:00 | host2 | 4 | +| 2024-01-24T00:00:00 | host2 | 6 | ++---------------------+-------+------------------------+ SELECT ts, host, min(val) RANGE '1d' FROM host ALIGN '1d' TO UNKNOWN ORDER BY host, ts; @@ -35,52 +35,52 @@ Error: 3000(PlanQuery), DataFusion error: Error during planning: Illegal `align SELECT ts, host, min(val) RANGE '1d' FROM host ALIGN '1d' TO '1900-01-01T00:00:00+01:00' ORDER BY host, ts; -+---------------------+-------+----------------------------------+ -| ts | host | MIN(host.val) RANGE 1d FILL NULL | -+---------------------+-------+----------------------------------+ -| 2024-01-22T23:00:00 | host1 | 0 | -| 2024-01-23T23:00:00 | host1 | 1 | -| 2024-01-24T23:00:00 | host1 | 3 | -| 2024-01-22T23:00:00 | host2 | 4 | -| 2024-01-23T23:00:00 | host2 | 5 | -| 2024-01-24T23:00:00 | host2 | 7 | -+---------------------+-------+----------------------------------+ ++---------------------+-------+------------------------+ +| ts | host | MIN(host.val) RANGE 1d | ++---------------------+-------+------------------------+ +| 2024-01-22T23:00:00 | host1 | 0 | +| 2024-01-23T23:00:00 | host1 | 1 | +| 2024-01-24T23:00:00 | host1 | 3 | +| 2024-01-22T23:00:00 | host2 | 4 | +| 2024-01-23T23:00:00 | host2 | 5 | +| 2024-01-24T23:00:00 | host2 | 7 | ++---------------------+-------+------------------------+ SELECT ts, host, min(val) RANGE '1d' FROM host ALIGN '1d' TO '2024-01-23T00:00:00+01:00' ORDER BY host, ts; -+---------------------+-------+----------------------------------+ -| ts | host | MIN(host.val) RANGE 1d FILL NULL | -+---------------------+-------+----------------------------------+ -| 2024-01-22T23:00:00 | host1 | 0 | -| 2024-01-23T23:00:00 | host1 | 1 | -| 2024-01-24T23:00:00 | host1 | 3 | -| 2024-01-22T23:00:00 | host2 | 4 | -| 2024-01-23T23:00:00 | host2 | 5 | -| 2024-01-24T23:00:00 | host2 | 7 | -+---------------------+-------+----------------------------------+ ++---------------------+-------+------------------------+ +| ts | host | MIN(host.val) RANGE 1d | ++---------------------+-------+------------------------+ +| 2024-01-22T23:00:00 | host1 | 0 | +| 2024-01-23T23:00:00 | host1 | 1 | +| 2024-01-24T23:00:00 | host1 | 3 | +| 2024-01-22T23:00:00 | host2 | 4 | +| 2024-01-23T23:00:00 | host2 | 5 | +| 2024-01-24T23:00:00 | host2 | 7 | ++---------------------+-------+------------------------+ SELECT ts, host, min(val) RANGE '1d' FROM host ALIGN '1d' TO '2023-01-01T00:00:00+01:00' ORDER BY host, ts; -+---------------------+-------+----------------------------------+ -| ts | host | MIN(host.val) RANGE 1d FILL NULL | -+---------------------+-------+----------------------------------+ -| 2024-01-22T23:00:00 | host1 | 0 | -| 2024-01-23T23:00:00 | host1 | 1 | -| 2024-01-24T23:00:00 | host1 | 3 | -| 2024-01-22T23:00:00 | host2 | 4 | -| 2024-01-23T23:00:00 | host2 | 5 | -| 2024-01-24T23:00:00 | host2 | 7 | -+---------------------+-------+----------------------------------+ ++---------------------+-------+------------------------+ +| ts | host | MIN(host.val) RANGE 1d | ++---------------------+-------+------------------------+ +| 2024-01-22T23:00:00 | host1 | 0 | +| 2024-01-23T23:00:00 | host1 | 1 | +| 2024-01-24T23:00:00 | host1 | 3 | +| 2024-01-22T23:00:00 | host2 | 4 | +| 2024-01-23T23:00:00 | host2 | 5 | +| 2024-01-24T23:00:00 | host2 | 7 | ++---------------------+-------+------------------------+ SELECT ts, min(val) RANGE (INTERVAL '1' day) FROM host ALIGN (INTERVAL '1' day) TO '1900-01-01T00:00:00+01:00' by (1) ORDER BY ts; -+---------------------+----------------------------------------------------------------------------+ -| ts | MIN(host.val) RANGE IntervalMonthDayNano("18446744073709551616") FILL NULL | -+---------------------+----------------------------------------------------------------------------+ -| 2024-01-22T23:00:00 | 0 | -| 2024-01-23T23:00:00 | 1 | -| 2024-01-24T23:00:00 | 3 | -+---------------------+----------------------------------------------------------------------------+ ++---------------------+------------------------------------------------------------------+ +| ts | MIN(host.val) RANGE IntervalMonthDayNano("18446744073709551616") | ++---------------------+------------------------------------------------------------------+ +| 2024-01-22T23:00:00 | 0 | +| 2024-01-23T23:00:00 | 1 | +| 2024-01-24T23:00:00 | 3 | ++---------------------+------------------------------------------------------------------+ --- ALIGN TO with time zone --- set time_zone='Asia/Shanghai'; @@ -90,14 +90,14 @@ Affected Rows: 0 ---- align to 'Asia/Shanghai' unix epoch 0 ---- SELECT ts, host, min(val) RANGE '1d' FROM host ALIGN '1d' ORDER BY host, ts; -+---------------------+-------+----------------------------------+ -| ts | host | MIN(host.val) RANGE 1d FILL NULL | -+---------------------+-------+----------------------------------+ -| 2024-01-23T08:00:00 | host1 | 0 | -| 2024-01-24T08:00:00 | host1 | 2 | -| 2024-01-23T08:00:00 | host2 | 4 | -| 2024-01-24T08:00:00 | host2 | 6 | -+---------------------+-------+----------------------------------+ ++---------------------+-------+------------------------+ +| ts | host | MIN(host.val) RANGE 1d | ++---------------------+-------+------------------------+ +| 2024-01-23T08:00:00 | host1 | 0 | +| 2024-01-24T08:00:00 | host1 | 2 | +| 2024-01-23T08:00:00 | host2 | 4 | +| 2024-01-24T08:00:00 | host2 | 6 | ++---------------------+-------+------------------------+ set time_zone='+23:00'; @@ -106,16 +106,16 @@ Affected Rows: 0 ---- align to '+23:00' unix epoch 0 ---- SELECT ts, host, min(val) RANGE '1d' FROM host ALIGN '1d' ORDER BY host, ts; -+---------------------+-------+----------------------------------+ -| ts | host | MIN(host.val) RANGE 1d FILL NULL | -+---------------------+-------+----------------------------------+ -| 2024-01-22T23:00:00 | host1 | 0 | -| 2024-01-23T23:00:00 | host1 | 1 | -| 2024-01-24T23:00:00 | host1 | 3 | -| 2024-01-22T23:00:00 | host2 | 4 | -| 2024-01-23T23:00:00 | host2 | 5 | -| 2024-01-24T23:00:00 | host2 | 7 | -+---------------------+-------+----------------------------------+ ++---------------------+-------+------------------------+ +| ts | host | MIN(host.val) RANGE 1d | ++---------------------+-------+------------------------+ +| 2024-01-22T23:00:00 | host1 | 0 | +| 2024-01-23T23:00:00 | host1 | 1 | +| 2024-01-24T23:00:00 | host1 | 3 | +| 2024-01-22T23:00:00 | host2 | 4 | +| 2024-01-23T23:00:00 | host2 | 5 | +| 2024-01-24T23:00:00 | host2 | 7 | ++---------------------+-------+------------------------+ set time_zone='UTC'; From b55905cf66c150ce3720a29b6582b05f7ebdc795 Mon Sep 17 00:00:00 2001 From: Zhenchi Date: Wed, 13 Mar 2024 18:03:03 +0800 Subject: [PATCH 16/17] feat(fuzz): add insert target (#3499) * fix(common-time): allow building nanos timestamp from parts split from i64::MIN Signed-off-by: Zhenchi * feat(fuzz): add insert target Signed-off-by: Zhenchi * chore: cleanup cargo.toml and polish comments Signed-off-by: Zhenchi --------- Signed-off-by: Zhenchi --- Cargo.lock | 1 + tests-fuzz/Cargo.toml | 15 +- tests-fuzz/src/generator/insert_expr.rs | 61 +++++-- tests-fuzz/src/ir.rs | 75 +++++++- tests-fuzz/src/ir/insert_expr.rs | 29 +++- .../src/translator/mysql/insert_expr.rs | 60 +++++-- tests-fuzz/src/validator/column.rs | 2 +- tests-fuzz/targets/fuzz_insert.rs | 163 ++++++++++++++++++ 8 files changed, 360 insertions(+), 46 deletions(-) create mode 100644 tests-fuzz/targets/fuzz_insert.rs diff --git a/Cargo.lock b/Cargo.lock index c2b4dc0b5c..a8ca9d963b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10219,6 +10219,7 @@ dependencies = [ "common-query", "common-runtime", "common-telemetry", + "common-time", "datatypes", "derive_builder 0.12.0", "dotenv", diff --git a/tests-fuzz/Cargo.toml b/tests-fuzz/Cargo.toml index 5cf789eb54..a63dc60bab 100644 --- a/tests-fuzz/Cargo.toml +++ b/tests-fuzz/Cargo.toml @@ -18,6 +18,7 @@ common-macro = { workspace = true } common-query = { workspace = true } common-runtime = { workspace = true } common-telemetry = { workspace = true } +common-time = { workspace = true } datatypes = { workspace = true } derive_builder = { workspace = true } dotenv = "0.15" @@ -39,13 +40,6 @@ sqlx = { version = "0.6", features = [ ] } [dev-dependencies] -dotenv = "0.15" -sqlx = { version = "0.6", features = [ - "runtime-tokio-rustls", - "mysql", - "postgres", - "chrono", -] } tokio = { workspace = true } [[bin]] @@ -54,3 +48,10 @@ path = "targets/fuzz_create_table.rs" test = false bench = false doc = false + +[[bin]] +name = "fuzz_insert" +path = "targets/fuzz_insert.rs" +test = false +bench = false +doc = false diff --git a/tests-fuzz/src/generator/insert_expr.rs b/tests-fuzz/src/generator/insert_expr.rs index f626b236f3..ec4ba4f615 100644 --- a/tests-fuzz/src/generator/insert_expr.rs +++ b/tests-fuzz/src/generator/insert_expr.rs @@ -14,6 +14,7 @@ use std::marker::PhantomData; +use datatypes::value::Value; use derive_builder::Builder; use rand::seq::SliceRandom; use rand::Rng; @@ -22,7 +23,7 @@ use crate::context::TableContextRef; use crate::error::{Error, Result}; use crate::fake::WordGenerator; use crate::generator::{Generator, Random}; -use crate::ir::insert_expr::InsertIntoExpr; +use crate::ir::insert_expr::{InsertIntoExpr, RowValue}; use crate::ir::{generate_random_value, Ident}; /// Generates [InsertIntoExpr]. @@ -41,30 +42,64 @@ pub struct InsertExprGenerator { impl Generator for InsertExprGenerator { type Error = Error; - /// Generates the [CreateTableExpr]. + /// Generates the [InsertIntoExpr]. fn generate(&self, rng: &mut R) -> Result { - let mut columns = self.table_ctx.columns.clone(); - columns.shuffle(rng); + // Whether to omit all columns, i.e. INSERT INTO table_name VALUES (...) + let omit_column_list = rng.gen_bool(0.2); - let mut rows = Vec::with_capacity(self.rows); + let mut values_columns = vec![]; + if omit_column_list { + // If omit column list, then all columns are required in the values list + values_columns = self.table_ctx.columns.clone(); + } else { + for column in &self.table_ctx.columns { + let can_omit = column.is_nullable() || column.has_default_value(); + + // 50% chance to omit a column if it's not required + if !can_omit || rng.gen_bool(0.5) { + values_columns.push(column.clone()); + } + } + values_columns.shuffle(rng); + + // If all columns are omitted, pick a random column + if values_columns.is_empty() { + values_columns.push(self.table_ctx.columns.choose(rng).unwrap().clone()); + } + } + + let mut values_list = Vec::with_capacity(self.rows); for _ in 0..self.rows { - let mut row = Vec::with_capacity(columns.len()); - for column in &columns { - // TODO(weny): generates the special cases - row.push(generate_random_value( + let mut row = Vec::with_capacity(values_columns.len()); + for column in &values_columns { + if column.is_nullable() && rng.gen_bool(0.2) { + row.push(RowValue::Value(Value::Null)); + continue; + } + + if column.has_default_value() && rng.gen_bool(0.2) { + row.push(RowValue::Default); + continue; + } + + row.push(RowValue::Value(generate_random_value( rng, &column.column_type, Some(self.word_generator.as_ref()), - )); + ))); } - rows.push(row); + values_list.push(row); } Ok(InsertIntoExpr { table_name: self.table_ctx.name.to_string(), - columns, - rows, + columns: if omit_column_list { + vec![] + } else { + values_columns + }, + values_list, }) } } diff --git a/tests-fuzz/src/ir.rs b/tests-fuzz/src/ir.rs index d7acd503eb..d20df0fa33 100644 --- a/tests-fuzz/src/ir.rs +++ b/tests-fuzz/src/ir.rs @@ -22,10 +22,13 @@ pub(crate) mod select_expr; use core::fmt; pub use alter_expr::AlterTableExpr; +use common_time::{Date, DateTime, Timestamp}; pub use create_expr::CreateTableExpr; use datatypes::data_type::ConcreteDataType; +use datatypes::types::TimestampType; use datatypes::value::Value; use derive_builder::Builder; +pub use insert_expr::InsertIntoExpr; use lazy_static::lazy_static; use rand::seq::SliceRandom; use rand::Rng; @@ -91,14 +94,62 @@ pub fn generate_random_value( Some(random) => Value::from(random.gen(rng).value), None => Value::from(rng.gen::().to_string()), }, - ConcreteDataType::Date(_) => Value::from(rng.gen::()), - ConcreteDataType::DateTime(_) => Value::from(rng.gen::()), - &ConcreteDataType::Timestamp(_) => Value::from(rng.gen::()), + ConcreteDataType::Date(_) => generate_random_date(rng), + ConcreteDataType::DateTime(_) => generate_random_datetime(rng), + &ConcreteDataType::Timestamp(ts_type) => generate_random_timestamp(rng, ts_type), _ => unimplemented!("unsupported type: {datatype}"), } } +fn generate_random_timestamp(rng: &mut R, ts_type: TimestampType) -> Value { + let v = match ts_type { + TimestampType::Second(_) => { + let min = i64::from(Timestamp::MIN_SECOND); + let max = i64::from(Timestamp::MAX_SECOND); + let value = rng.gen_range(min..=max); + Timestamp::new_second(value) + } + TimestampType::Millisecond(_) => { + let min = i64::from(Timestamp::MIN_MILLISECOND); + let max = i64::from(Timestamp::MAX_MILLISECOND); + let value = rng.gen_range(min..=max); + Timestamp::new_millisecond(value) + } + TimestampType::Microsecond(_) => { + let min = i64::from(Timestamp::MIN_MICROSECOND); + let max = i64::from(Timestamp::MAX_MICROSECOND); + let value = rng.gen_range(min..=max); + Timestamp::new_microsecond(value) + } + TimestampType::Nanosecond(_) => { + let min = i64::from(Timestamp::MIN_NANOSECOND); + let max = i64::from(Timestamp::MAX_NANOSECOND); + let value = rng.gen_range(min..=max); + Timestamp::new_nanosecond(value) + } + }; + Value::from(v) +} + +fn generate_random_datetime(rng: &mut R) -> Value { + let min = i64::from(Timestamp::MIN_MILLISECOND); + let max = i64::from(Timestamp::MAX_MILLISECOND); + let value = rng.gen_range(min..=max); + let datetime = Timestamp::new_millisecond(value) + .to_chrono_datetime() + .unwrap(); + Value::from(DateTime::from(datetime)) +} + +fn generate_random_date(rng: &mut R) -> Value { + let min = i64::from(Timestamp::MIN_MILLISECOND); + let max = i64::from(Timestamp::MAX_MILLISECOND); + let value = rng.gen_range(min..=max); + let date = Timestamp::new_millisecond(value).to_chrono_date().unwrap(); + Value::from(Date::from(date)) +} + /// An identifier. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Eq, Ord, Hash)] pub struct Ident { @@ -181,6 +232,24 @@ impl Column { .iter() .any(|opt| opt == &ColumnOption::PrimaryKey) } + + /// Returns true if it's nullable. + pub fn is_nullable(&self) -> bool { + !self + .options + .iter() + .any(|opt| matches!(opt, ColumnOption::NotNull | ColumnOption::TimeIndex)) + } + + // Returns true if it has default value. + pub fn has_default_value(&self) -> bool { + self.options.iter().any(|opt| { + matches!( + opt, + ColumnOption::DefaultValue(_) | ColumnOption::DefaultFn(_) + ) + }) + } } /// Returns droppable columns. i.e., non-primary key columns, non-ts columns. diff --git a/tests-fuzz/src/ir/insert_expr.rs b/tests-fuzz/src/ir/insert_expr.rs index 8fc4113384..c7476c5170 100644 --- a/tests-fuzz/src/ir/insert_expr.rs +++ b/tests-fuzz/src/ir/insert_expr.rs @@ -12,14 +12,37 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::fmt::Display; + use datatypes::value::Value; use crate::ir::Column; -pub type RowValue = Vec; - pub struct InsertIntoExpr { pub table_name: String, pub columns: Vec, - pub rows: Vec, + pub values_list: Vec, +} + +pub type RowValues = Vec; + +pub enum RowValue { + Value(Value), + Default, +} + +impl Display for RowValue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + RowValue::Value(v) => match v { + Value::Null => write!(f, "NULL"), + v @ (Value::String(_) + | Value::Timestamp(_) + | Value::DateTime(_) + | Value::Date(_)) => write!(f, "'{}'", v), + v => write!(f, "{}", v), + }, + RowValue::Default => write!(f, "DEFAULT"), + } + } } diff --git a/tests-fuzz/src/translator/mysql/insert_expr.rs b/tests-fuzz/src/translator/mysql/insert_expr.rs index 269df9e6ee..49ff192afb 100644 --- a/tests-fuzz/src/translator/mysql/insert_expr.rs +++ b/tests-fuzz/src/translator/mysql/insert_expr.rs @@ -22,33 +22,42 @@ impl DslTranslator for InsertIntoExprTranslator { type Error = Error; fn translate(&self, input: &InsertIntoExpr) -> Result { - let columns = input - .columns - .iter() - .map(|c| c.name.to_string()) - .collect::>() - .join(", ") - .to_string(); - Ok(format!( - "INSERT INTO {} ({})\nVALUES\n{};", + "INSERT INTO {} {} VALUES\n{};", input.table_name, - columns, + Self::format_columns(input), Self::format_values(input) )) } } impl InsertIntoExprTranslator { + fn format_columns(input: &InsertIntoExpr) -> String { + if input.columns.is_empty() { + "".to_string() + } else { + let list = input + .columns + .iter() + .map(|c| c.name.to_string()) + .collect::>() + .join(", ") + .to_string(); + + format!("({})", list) + } + } + fn format_values(input: &InsertIntoExpr) -> String { input - .rows + .values_list .iter() - .map(|row| { + .map(|value| { format!( "({})", - row.iter() - .map(|v| format!("'{v}'")) + value + .iter() + .map(|v| v.to_string()) .collect::>() .join(", ") ) @@ -64,7 +73,7 @@ mod tests { use rand::SeedableRng; - use super::InsertIntoExprTranslator; + use super::*; use crate::generator::insert_expr::InsertExprGeneratorBuilder; use crate::generator::Generator; use crate::test_utils; @@ -84,10 +93,23 @@ mod tests { let insert_expr = insert_expr_generator.generate(&mut rng).unwrap(); let output = InsertIntoExprTranslator.translate(&insert_expr).unwrap(); - let expected = r#"INSERT INTO test (host, idc, memory_util, ts, cpu_util, disk_util) -VALUES -('adipisci', 'debitis', '0.5495312687894465', '15292064470292927036', '0.9354265029131291', '0.8037816422279636'), -('ut', 'sequi', '0.8807117723618908', '14214208091261382505', '0.5240550121500691', '0.350785883750684');"#; + let expected = r#"INSERT INTO test (ts, host, cpu_util) VALUES +('+199601-11-07 21:32:56.695+0000', 'corrupti', 0.051130243193075464), +('+40822-03-25 02:17:34.328+0000', NULL, 0.6552502332327004);"#; + assert_eq!(output, expected); + + let insert_expr = insert_expr_generator.generate(&mut rng).unwrap(); + let output = InsertIntoExprTranslator.translate(&insert_expr).unwrap(); + let expected = r#"INSERT INTO test (cpu_util, disk_util, ts) VALUES +(0.7074194466620976, 0.661288102315126, '-47252-05-08 07:33:49.567+0000'), +(0.8266101224213618, 0.7947724277743285, '-224292-12-07 02:51:53.371+0000');"#; + assert_eq!(output, expected); + + let insert_expr = insert_expr_generator.generate(&mut rng).unwrap(); + let output = InsertIntoExprTranslator.translate(&insert_expr).unwrap(); + let expected = r#"INSERT INTO test VALUES +('odio', NULL, 0.48809950435391647, 0.5228925709595407, 0.9091528874275897, '+241156-12-16 20:52:15.185+0000'), +('dignissimos', 'labore', NULL, 0.12983559048685023, 0.6362040919831425, '-30691-06-17 23:41:09.938+0000');"#; assert_eq!(output, expected); } } diff --git a/tests-fuzz/src/validator/column.rs b/tests-fuzz/src/validator/column.rs index 5b148f7f02..797834eec5 100644 --- a/tests-fuzz/src/validator/column.rs +++ b/tests-fuzz/src/validator/column.rs @@ -98,7 +98,7 @@ impl PartialEq for ColumnEntry { .iter() .any(|opt| matches!(opt, ColumnOption::NotNull)) { - debug!("ColumnOption::NotNull is not found"); + debug!("ColumnOption::NotNull is found"); return false; } } else { diff --git a/tests-fuzz/targets/fuzz_insert.rs b/tests-fuzz/targets/fuzz_insert.rs new file mode 100644 index 0000000000..7fc6d30a23 --- /dev/null +++ b/tests-fuzz/targets/fuzz_insert.rs @@ -0,0 +1,163 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![no_main] + +use std::sync::Arc; + +use common_telemetry::info; +use libfuzzer_sys::arbitrary::{Arbitrary, Unstructured}; +use libfuzzer_sys::fuzz_target; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaChaRng; +use snafu::{ensure, ResultExt}; +use sqlx::{Executor, MySql, Pool}; +use tests_fuzz::context::{TableContext, TableContextRef}; +use tests_fuzz::error::{self, Result}; +use tests_fuzz::fake::{ + merge_two_word_map_fn, random_capitalize_map, uppercase_and_keyword_backtick_map, + MappedGenerator, WordGenerator, +}; +use tests_fuzz::generator::create_expr::CreateTableExprGeneratorBuilder; +use tests_fuzz::generator::insert_expr::InsertExprGeneratorBuilder; +use tests_fuzz::generator::Generator; +use tests_fuzz::ir::{CreateTableExpr, InsertIntoExpr}; +use tests_fuzz::translator::mysql::create_expr::CreateTableExprTranslator; +use tests_fuzz::translator::mysql::insert_expr::InsertIntoExprTranslator; +use tests_fuzz::translator::DslTranslator; +use tests_fuzz::utils::{init_greptime_connections, Connections}; + +struct FuzzContext { + greptime: Pool, +} + +impl FuzzContext { + async fn close(self) { + self.greptime.close().await; + } +} + +#[derive(Copy, Clone, Debug)] +struct FuzzInput { + seed: u64, + columns: usize, + rows: usize, +} + +impl Arbitrary<'_> for FuzzInput { + fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result { + let seed = u.int_in_range(u64::MIN..=u64::MAX)?; + let mut rng = ChaChaRng::seed_from_u64(seed); + let columns = rng.gen_range(2..30); + let rows = rng.gen_range(1..4096); + Ok(FuzzInput { + columns, + rows, + seed, + }) + } +} + +fn generate_create_expr( + input: FuzzInput, + rng: &mut R, +) -> Result { + let create_table_generator = CreateTableExprGeneratorBuilder::default() + .name_generator(Box::new(MappedGenerator::new( + WordGenerator, + merge_two_word_map_fn(random_capitalize_map, uppercase_and_keyword_backtick_map), + ))) + .columns(input.columns) + .engine("mito") + .build() + .unwrap(); + create_table_generator.generate(rng) +} + +fn generate_insert_expr( + input: FuzzInput, + rng: &mut R, + table_ctx: TableContextRef, +) -> Result { + let insert_generator = InsertExprGeneratorBuilder::default() + .table_ctx(table_ctx) + .rows(input.rows) + .build() + .unwrap(); + insert_generator.generate(rng) +} + +async fn execute_insert(ctx: FuzzContext, input: FuzzInput) -> Result<()> { + info!("input: {input:?}"); + let mut rng = ChaChaRng::seed_from_u64(input.seed); + + let create_expr = generate_create_expr(input, &mut rng)?; + let translator = CreateTableExprTranslator; + let sql = translator.translate(&create_expr)?; + let _result = sqlx::query(&sql) + .execute(&ctx.greptime) + .await + .context(error::ExecuteQuerySnafu { sql: &sql })?; + + let table_ctx = Arc::new(TableContext::from(&create_expr)); + let insert_expr = generate_insert_expr(input, &mut rng, table_ctx)?; + let translator = InsertIntoExprTranslator; + let sql = translator.translate(&insert_expr)?; + let result = ctx + .greptime + // unprepared query, see + .execute(sql.as_str()) + .await + .context(error::ExecuteQuerySnafu { sql: &sql })?; + + ensure!( + result.rows_affected() == input.rows as u64, + error::AssertSnafu { + reason: format!( + "expected rows affected: {}, actual: {}", + input.rows, + result.rows_affected(), + ) + } + ); + + // TODO: Validate inserted rows + + // Cleans up + let sql = format!("DROP TABLE {}", create_expr.table_name); + let result = sqlx::query(&sql) + .execute(&ctx.greptime) + .await + .context(error::ExecuteQuerySnafu { sql })?; + info!( + "Drop table: {}\n\nResult: {result:?}\n\n", + create_expr.table_name + ); + ctx.close().await; + + Ok(()) +} + +fuzz_target!(|input: FuzzInput| { + common_telemetry::init_default_ut_logging(); + common_runtime::block_on_write(async { + let Connections { mysql } = init_greptime_connections().await; + let ctx = FuzzContext { + greptime: mysql.expect("mysql connection init must be succeed"), + }; + execute_insert(ctx, input) + .await + .unwrap_or_else(|err| panic!("fuzz test must be succeed: {err:?}")); + }) +}); From e4333969b48292e4b00f00f75a2657b92be0c98a Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Wed, 13 Mar 2024 22:11:47 +0800 Subject: [PATCH 17/17] feat(fuzz): add alter table target (#3503) * feat(fuzz): validate semantic type of column * feat(fuzz): add fuzz_alter_table target * feat(fuzz): validate columns * chore(ci): add fuzz_alter_table ci cfg --- .github/workflows/develop.yml | 2 +- tests-fuzz/Cargo.toml | 7 + tests-fuzz/src/context.rs | 189 +++++++++++++++++++++++- tests-fuzz/src/generator/alter_expr.rs | 47 +++++- tests-fuzz/src/ir/create_expr.rs | 2 +- tests-fuzz/src/validator/column.rs | 59 +++++++- tests-fuzz/targets/fuzz_alter_table.rs | 185 +++++++++++++++++++++++ tests-fuzz/targets/fuzz_create_table.rs | 6 +- 8 files changed, 480 insertions(+), 17 deletions(-) create mode 100644 tests-fuzz/targets/fuzz_alter_table.rs diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 94ada0fabf..1ce1d8c18c 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -123,7 +123,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - target: [ "fuzz_create_table" ] + target: [ "fuzz_create_table", "fuzz_alter_table" ] steps: - uses: actions/checkout@v4 - uses: arduino/setup-protoc@v3 diff --git a/tests-fuzz/Cargo.toml b/tests-fuzz/Cargo.toml index a63dc60bab..ce216de419 100644 --- a/tests-fuzz/Cargo.toml +++ b/tests-fuzz/Cargo.toml @@ -55,3 +55,10 @@ path = "targets/fuzz_insert.rs" test = false bench = false doc = false + +[[bin]] +name = "fuzz_alter_table" +path = "targets/fuzz_alter_table.rs" +test = false +bench = false +doc = false diff --git a/tests-fuzz/src/context.rs b/tests-fuzz/src/context.rs index a9fbcbd3aa..29536c853c 100644 --- a/tests-fuzz/src/context.rs +++ b/tests-fuzz/src/context.rs @@ -14,13 +14,20 @@ use std::sync::Arc; +use common_query::AddColumnLocation; use partition::partition::PartitionDef; +use rand::Rng; +use snafu::{ensure, OptionExt}; -use crate::ir::{Column, CreateTableExpr, Ident}; +use crate::error::{self, Result}; +use crate::generator::Random; +use crate::ir::alter_expr::AlterTableOperation; +use crate::ir::{AlterTableExpr, Column, CreateTableExpr, Ident}; pub type TableContextRef = Arc; /// TableContext stores table info. +#[derive(Debug, Clone)] pub struct TableContext { pub name: Ident, pub columns: Vec, @@ -48,3 +55,183 @@ impl From<&CreateTableExpr> for TableContext { } } } + +impl TableContext { + /// Applies the [AlterTableExpr]. + pub fn alter(mut self, expr: AlterTableExpr) -> Result { + match expr.alter_options { + AlterTableOperation::AddColumn { column, location } => { + ensure!( + !self.columns.iter().any(|col| col.name == column.name), + error::UnexpectedSnafu { + violated: format!("Column {} exists", column.name), + } + ); + match location { + Some(AddColumnLocation::First) => { + let mut columns = Vec::with_capacity(self.columns.len() + 1); + columns.push(column); + columns.extend(self.columns); + self.columns = columns; + } + Some(AddColumnLocation::After { column_name }) => { + let index = self + .columns + .iter() + // TODO(weny): find a better way? + .position(|col| col.name.to_string() == column_name) + .context(error::UnexpectedSnafu { + violated: format!("Column: {column_name} not found"), + })?; + self.columns.insert(index + 1, column); + } + None => self.columns.push(column), + } + // Re-generates the primary_keys + self.primary_keys = self + .columns + .iter() + .enumerate() + .flat_map(|(idx, col)| { + if col.is_primary_key() { + Some(idx) + } else { + None + } + }) + .collect(); + Ok(self) + } + AlterTableOperation::DropColumn { name } => { + self.columns.retain(|col| col.name != name); + // Re-generates the primary_keys + self.primary_keys = self + .columns + .iter() + .enumerate() + .flat_map(|(idx, col)| { + if col.is_primary_key() { + Some(idx) + } else { + None + } + }) + .collect(); + Ok(self) + } + AlterTableOperation::RenameTable { new_table_name } => { + ensure!( + new_table_name != self.name, + error::UnexpectedSnafu { + violated: "The new table name is equal the current name", + } + ); + self.name = new_table_name; + Ok(self) + } + } + } + + pub fn generate_unique_column_name( + &self, + rng: &mut R, + generator: &dyn Random, + ) -> Ident { + let mut name = generator.gen(rng); + while self.columns.iter().any(|col| col.name.value == name.value) { + name = generator.gen(rng); + } + name + } + + pub fn generate_unique_table_name( + &self, + rng: &mut R, + generator: &dyn Random, + ) -> Ident { + let mut name = generator.gen(rng); + while self.name.value == name.value { + name = generator.gen(rng); + } + name + } +} + +#[cfg(test)] +mod tests { + use common_query::AddColumnLocation; + use datatypes::data_type::ConcreteDataType; + + use super::TableContext; + use crate::ir::alter_expr::AlterTableOperation; + use crate::ir::create_expr::ColumnOption; + use crate::ir::{AlterTableExpr, Column, Ident}; + + #[test] + fn test_table_context_alter() { + let table_ctx = TableContext { + name: "foo".into(), + columns: vec![], + partition: None, + primary_keys: vec![], + }; + // Add a column + let expr = AlterTableExpr { + table_name: "foo".into(), + alter_options: AlterTableOperation::AddColumn { + column: Column { + name: "a".into(), + column_type: ConcreteDataType::timestamp_microsecond_datatype(), + options: vec![ColumnOption::PrimaryKey], + }, + location: None, + }, + }; + let table_ctx = table_ctx.alter(expr).unwrap(); + assert_eq!(table_ctx.columns[0].name, Ident::new("a")); + assert_eq!(table_ctx.primary_keys, vec![0]); + + // Add a column at first + let expr = AlterTableExpr { + table_name: "foo".into(), + alter_options: AlterTableOperation::AddColumn { + column: Column { + name: "b".into(), + column_type: ConcreteDataType::timestamp_microsecond_datatype(), + options: vec![ColumnOption::PrimaryKey], + }, + location: Some(AddColumnLocation::First), + }, + }; + let table_ctx = table_ctx.alter(expr).unwrap(); + assert_eq!(table_ctx.columns[0].name, Ident::new("b")); + assert_eq!(table_ctx.primary_keys, vec![0, 1]); + + // Add a column after "b" + let expr = AlterTableExpr { + table_name: "foo".into(), + alter_options: AlterTableOperation::AddColumn { + column: Column { + name: "c".into(), + column_type: ConcreteDataType::timestamp_microsecond_datatype(), + options: vec![ColumnOption::PrimaryKey], + }, + location: Some(AddColumnLocation::After { + column_name: "b".into(), + }), + }, + }; + let table_ctx = table_ctx.alter(expr).unwrap(); + assert_eq!(table_ctx.columns[1].name, Ident::new("c")); + assert_eq!(table_ctx.primary_keys, vec![0, 1, 2]); + + // Drop the column "b" + let expr = AlterTableExpr { + table_name: "foo".into(), + alter_options: AlterTableOperation::DropColumn { name: "b".into() }, + }; + let table_ctx = table_ctx.alter(expr).unwrap(); + assert_eq!(table_ctx.columns[1].name, Ident::new("a")); + assert_eq!(table_ctx.primary_keys, vec![0, 1]); + } +} diff --git a/tests-fuzz/src/generator/alter_expr.rs b/tests-fuzz/src/generator/alter_expr.rs index 03e823773d..1a9d4b965b 100644 --- a/tests-fuzz/src/generator/alter_expr.rs +++ b/tests-fuzz/src/generator/alter_expr.rs @@ -15,6 +15,7 @@ use std::marker::PhantomData; use common_query::AddColumnLocation; +use datatypes::data_type::ConcreteDataType; use derive_builder::Builder; use rand::Rng; use snafu::ensure; @@ -24,10 +25,38 @@ use crate::error::{self, Error, Result}; use crate::fake::WordGenerator; use crate::generator::{ColumnOptionGenerator, ConcreteDataTypeGenerator, Generator, Random}; use crate::ir::alter_expr::{AlterTableExpr, AlterTableOperation}; +use crate::ir::create_expr::ColumnOption; use crate::ir::{ - column_options_generator, droppable_columns, generate_columns, ColumnTypeGenerator, Ident, + droppable_columns, generate_columns, generate_random_value, ColumnTypeGenerator, Ident, }; +fn add_column_options_generator( + rng: &mut R, + column_type: &ConcreteDataType, +) -> Vec { + // 0 -> NULL + // 1 -> DEFAULT VALUE + // 2 -> PRIMARY KEY + DEFAULT VALUE + let idx = rng.gen_range(0..3); + match idx { + 0 => vec![ColumnOption::Null], + 1 => { + vec![ColumnOption::DefaultValue(generate_random_value( + rng, + column_type, + None, + ))] + } + 2 => { + vec![ + ColumnOption::PrimaryKey, + ColumnOption::DefaultValue(generate_random_value(rng, column_type, None)), + ] + } + _ => unreachable!(), + } +} + /// Generates the [AlterTableOperation::AddColumn] of [AlterTableExpr]. #[derive(Builder)] #[builder(pattern = "owned")] @@ -37,7 +66,7 @@ pub struct AlterExprAddColumnGenerator { location: bool, #[builder(default = "Box::new(WordGenerator)")] name_generator: Box>, - #[builder(default = "Box::new(column_options_generator)")] + #[builder(default = "Box::new(add_column_options_generator)")] column_options_generator: ColumnOptionGenerator, #[builder(default = "Box::new(ColumnTypeGenerator)")] column_type_generator: ConcreteDataTypeGenerator, @@ -65,7 +94,9 @@ impl Generator for AlterExprAddColumnGenera None }; - let name = self.name_generator.gen(rng); + let name = self + .table_ctx + .generate_unique_column_name(rng, self.name_generator.as_ref()); let column = generate_columns( rng, vec![name], @@ -116,7 +147,9 @@ impl Generator for AlterExprRenameGenerator { type Error = Error; fn generate(&self, rng: &mut R) -> Result { - let new_table_name = self.name_generator.gen(rng); + let new_table_name = self + .table_ctx + .generate_unique_table_name(rng, self.name_generator.as_ref()); Ok(AlterTableExpr { table_name: self.table_ctx.name.clone(), alter_options: AlterTableOperation::RenameTable { new_table_name }, @@ -153,7 +186,7 @@ mod tests { .generate(&mut rng) .unwrap(); let serialized = serde_json::to_string(&expr).unwrap(); - let expected = r#"{"table_name":{"value":"animI","quote_style":null},"alter_options":{"AddColumn":{"column":{"name":{"value":"velit","quote_style":null},"column_type":{"Int32":{}},"options":[{"DefaultValue":{"Int32":853246610}}]},"location":null}}}"#; + let expected = r#"{"table_name":{"value":"animI","quote_style":null},"alter_options":{"AddColumn":{"column":{"name":{"value":"velit","quote_style":null},"column_type":{"Int32":{}},"options":[{"DefaultValue":{"Int32":1606462472}}]},"location":null}}}"#; assert_eq!(expected, serialized); let expr = AlterExprRenameGeneratorBuilder::default() @@ -163,7 +196,7 @@ mod tests { .generate(&mut rng) .unwrap(); let serialized = serde_json::to_string(&expr).unwrap(); - let expected = r#"{"table_name":{"value":"animI","quote_style":null},"alter_options":{"RenameTable":{"new_table_name":{"value":"iure","quote_style":null}}}}"#; + let expected = r#"{"table_name":{"value":"animI","quote_style":null},"alter_options":{"RenameTable":{"new_table_name":{"value":"nihil","quote_style":null}}}}"#; assert_eq!(expected, serialized); let expr = AlterExprDropColumnGeneratorBuilder::default() @@ -173,7 +206,7 @@ mod tests { .generate(&mut rng) .unwrap(); let serialized = serde_json::to_string(&expr).unwrap(); - let expected = r#"{"table_name":{"value":"animI","quote_style":null},"alter_options":{"DropColumn":{"name":{"value":"toTAm","quote_style":null}}}}"#; + let expected = r#"{"table_name":{"value":"animI","quote_style":null},"alter_options":{"DropColumn":{"name":{"value":"cUmquE","quote_style":null}}}}"#; assert_eq!(expected, serialized); } } diff --git a/tests-fuzz/src/ir/create_expr.rs b/tests-fuzz/src/ir/create_expr.rs index 6ef151f825..1e6c165b5c 100644 --- a/tests-fuzz/src/ir/create_expr.rs +++ b/tests-fuzz/src/ir/create_expr.rs @@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize}; use crate::ir::{Column, Ident}; -// The column options +/// The column options #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)] pub enum ColumnOption { Null, diff --git a/tests-fuzz/src/validator/column.rs b/tests-fuzz/src/validator/column.rs index 797834eec5..0736bbb48d 100644 --- a/tests-fuzz/src/validator/column.rs +++ b/tests-fuzz/src/validator/column.rs @@ -37,6 +37,21 @@ fn is_nullable(str: &str) -> bool { str.to_uppercase() == "YES" } +enum SemanticType { + Timestamp, + Field, + Tag, +} + +fn semantic_type(str: &str) -> Option { + match str { + "TIMESTAMP" => Some(SemanticType::Timestamp), + "FIELD" => Some(SemanticType::Field), + "TAG" => Some(SemanticType::Tag), + _ => None, + } +} + impl PartialEq for ColumnEntry { fn eq(&self, other: &Column) -> bool { // Checks `table_name` @@ -108,11 +123,47 @@ impl PartialEq for ColumnEntry { .iter() .any(|opt| matches!(opt, ColumnOption::NotNull | ColumnOption::TimeIndex)) { - debug!("unexpected ColumnOption::NotNull or ColumnOption::TimeIndex"); + debug!("ColumnOption::NotNull or ColumnOption::TimeIndex is not found"); return false; } } //TODO: Checks `semantic_type` + match semantic_type(&self.semantic_type) { + Some(SemanticType::Tag) => { + if !other + .options + .iter() + .any(|opt| matches!(opt, ColumnOption::PrimaryKey)) + { + debug!("ColumnOption::PrimaryKey is not found"); + return false; + } + } + Some(SemanticType::Field) => { + if other + .options + .iter() + .any(|opt| matches!(opt, ColumnOption::PrimaryKey | ColumnOption::TimeIndex)) + { + debug!("unexpected ColumnOption::PrimaryKey or ColumnOption::TimeIndex"); + return false; + } + } + Some(SemanticType::Timestamp) => { + if !other + .options + .iter() + .any(|opt| matches!(opt, ColumnOption::TimeIndex)) + { + debug!("ColumnOption::TimeIndex is not found"); + return false; + } + } + None => { + debug!("unknown semantic type: {}", self.semantic_type); + return false; + } + }; true } @@ -186,7 +237,7 @@ mod tests { table_name: String::new(), column_name: "test".to_string(), data_type: ConcreteDataType::int8_datatype().name(), - semantic_type: String::new(), + semantic_type: "FIELD".to_string(), column_default: None, is_nullable: "Yes".to_string(), }; @@ -210,7 +261,7 @@ mod tests { table_name: String::new(), column_name: "test".to_string(), data_type: ConcreteDataType::int8_datatype().to_string(), - semantic_type: String::new(), + semantic_type: "FIELD".to_string(), column_default: Some("1".to_string()), is_nullable: "Yes".to_string(), }; @@ -226,7 +277,7 @@ mod tests { table_name: String::new(), column_name: "test".to_string(), data_type: ConcreteDataType::int8_datatype().to_string(), - semantic_type: String::new(), + semantic_type: "FIELD".to_string(), column_default: Some("Hello()".to_string()), is_nullable: "Yes".to_string(), }; diff --git a/tests-fuzz/targets/fuzz_alter_table.rs b/tests-fuzz/targets/fuzz_alter_table.rs new file mode 100644 index 0000000000..3d345c2f16 --- /dev/null +++ b/tests-fuzz/targets/fuzz_alter_table.rs @@ -0,0 +1,185 @@ +// Copyright 2023 Greptime Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![no_main] + +use std::sync::Arc; + +use arbitrary::{Arbitrary, Unstructured}; +use common_telemetry::info; +use libfuzzer_sys::fuzz_target; +use rand::{Rng, SeedableRng}; +use rand_chacha::ChaChaRng; +use snafu::ResultExt; +use sqlx::{MySql, Pool}; +use tests_fuzz::context::{TableContext, TableContextRef}; +use tests_fuzz::error::{self, Result}; +use tests_fuzz::fake::{ + merge_two_word_map_fn, random_capitalize_map, uppercase_and_keyword_backtick_map, + MappedGenerator, WordGenerator, +}; +use tests_fuzz::generator::alter_expr::{ + AlterExprAddColumnGeneratorBuilder, AlterExprDropColumnGeneratorBuilder, + AlterExprRenameGeneratorBuilder, +}; +use tests_fuzz::generator::create_expr::CreateTableExprGeneratorBuilder; +use tests_fuzz::generator::Generator; +use tests_fuzz::ir::{droppable_columns, AlterTableExpr, CreateTableExpr}; +use tests_fuzz::translator::mysql::alter_expr::AlterTableExprTranslator; +use tests_fuzz::translator::mysql::create_expr::CreateTableExprTranslator; +use tests_fuzz::translator::DslTranslator; +use tests_fuzz::utils::{init_greptime_connections, Connections}; +use tests_fuzz::validator; + +struct FuzzContext { + greptime: Pool, +} + +impl FuzzContext { + async fn close(self) { + self.greptime.close().await; + } +} + +#[derive(Clone, Debug)] +struct FuzzInput { + seed: u64, + actions: usize, +} + +fn generate_create_table_expr(rng: &mut R) -> Result { + let columns = rng.gen_range(2..30); + let create_table_generator = CreateTableExprGeneratorBuilder::default() + .name_generator(Box::new(MappedGenerator::new( + WordGenerator, + merge_two_word_map_fn(random_capitalize_map, uppercase_and_keyword_backtick_map), + ))) + .columns(columns) + .engine("mito") + .build() + .unwrap(); + create_table_generator.generate(rng) +} + +fn generate_alter_table_expr( + table_ctx: TableContextRef, + rng: &mut R, +) -> Result { + let rename = rng.gen_bool(0.2); + if rename { + let expr_generator = AlterExprRenameGeneratorBuilder::default() + .table_ctx(table_ctx) + .name_generator(Box::new(MappedGenerator::new( + WordGenerator, + merge_two_word_map_fn(random_capitalize_map, uppercase_and_keyword_backtick_map), + ))) + .build() + .unwrap(); + expr_generator.generate(rng) + } else { + let drop_column = rng.gen_bool(0.5) && !droppable_columns(&table_ctx.columns).is_empty(); + if drop_column { + let expr_generator = AlterExprDropColumnGeneratorBuilder::default() + .table_ctx(table_ctx) + .build() + .unwrap(); + expr_generator.generate(rng) + } else { + let location = rng.gen_bool(0.5); + let expr_generator = AlterExprAddColumnGeneratorBuilder::default() + .table_ctx(table_ctx) + .location(location) + .build() + .unwrap(); + expr_generator.generate(rng) + } + } +} + +impl Arbitrary<'_> for FuzzInput { + fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result { + let seed = u.int_in_range(u64::MIN..=u64::MAX)?; + let mut rng = ChaChaRng::seed_from_u64(seed); + let actions = rng.gen_range(1..256); + + Ok(FuzzInput { seed, actions }) + } +} + +async fn execute_alter_table(ctx: FuzzContext, input: FuzzInput) -> Result<()> { + info!("input: {input:?}"); + let mut rng = ChaChaRng::seed_from_u64(input.seed); + + // Create table + let expr = generate_create_table_expr(&mut rng).unwrap(); + let translator = CreateTableExprTranslator; + let sql = translator.translate(&expr)?; + let result = sqlx::query(&sql) + .execute(&ctx.greptime) + .await + .context(error::ExecuteQuerySnafu { sql: &sql })?; + info!("Create table: {sql}, result: {result:?}"); + + // Alter table actions + let mut table_ctx = Arc::new(TableContext::from(&expr)); + for _ in 0..input.actions { + let expr = generate_alter_table_expr(table_ctx.clone(), &mut rng).unwrap(); + let translator = AlterTableExprTranslator; + let sql = translator.translate(&expr)?; + let result = sqlx::query(&sql) + .execute(&ctx.greptime) + .await + .context(error::ExecuteQuerySnafu { sql: &sql })?; + info!("Alter table: {sql}, result: {result:?}"); + // Applies changes + table_ctx = Arc::new(Arc::unwrap_or_clone(table_ctx).alter(expr).unwrap()); + + // Validates columns + let mut column_entries = validator::column::fetch_columns( + &ctx.greptime, + "public".into(), + table_ctx.name.clone(), + ) + .await?; + column_entries.sort_by(|a, b| a.column_name.cmp(&b.column_name)); + let mut columns = table_ctx.columns.clone(); + columns.sort_by(|a, b| a.name.value.cmp(&b.name.value)); + validator::column::assert_eq(&column_entries, &columns)?; + } + + // Cleans up + let table_name = table_ctx.name.clone(); + let sql = format!("DROP TABLE {}", table_name); + let result = sqlx::query(&sql) + .execute(&ctx.greptime) + .await + .context(error::ExecuteQuerySnafu { sql })?; + info!("Drop table: {}, result: {result:?}", table_name); + ctx.close().await; + + Ok(()) +} + +fuzz_target!(|input: FuzzInput| { + common_telemetry::init_default_ut_logging(); + common_runtime::block_on_write(async { + let Connections { mysql } = init_greptime_connections().await; + let ctx = FuzzContext { + greptime: mysql.expect("mysql connection init must be succeed"), + }; + execute_alter_table(ctx, input) + .await + .unwrap_or_else(|err| panic!("fuzz test must be succeed: {err:?}")); + }) +}); diff --git a/tests-fuzz/targets/fuzz_create_table.rs b/tests-fuzz/targets/fuzz_create_table.rs index 7af489b1c2..6d351778dc 100644 --- a/tests-fuzz/targets/fuzz_create_table.rs +++ b/tests-fuzz/targets/fuzz_create_table.rs @@ -33,7 +33,6 @@ use tests_fuzz::translator::mysql::create_expr::CreateTableExprTranslator; use tests_fuzz::translator::DslTranslator; use tests_fuzz::utils::{init_greptime_connections, Connections}; use tests_fuzz::validator; -use tests_fuzz::validator::column::fetch_columns; struct FuzzContext { greptime: Pool, @@ -85,9 +84,10 @@ async fn execute_create_table(ctx: FuzzContext, input: FuzzInput) -> Result<()> .context(error::ExecuteQuerySnafu { sql: &sql })?; info!("Create table: {sql}, result: {result:?}"); - // Validate columns + // Validates columns let mut column_entries = - fetch_columns(&ctx.greptime, "public".into(), expr.table_name.clone()).await?; + validator::column::fetch_columns(&ctx.greptime, "public".into(), expr.table_name.clone()) + .await?; column_entries.sort_by(|a, b| a.column_name.cmp(&b.column_name)); let mut columns = expr.columns.clone(); columns.sort_by(|a, b| a.name.value.cmp(&b.name.value));