Compare commits

..

11 Commits

Author SHA1 Message Date
Weny Xu
d57b144b2f chore: change test_remove_outdated_meta_task sleep time to 40ms (#2620)
chore: change test_remove_outdated_meta_task sleep time to 300ms
2023-10-18 11:33:35 +00:00
WU Jingdi
46e106bcc3 feat: allow nest range expr in Range Query (#2557)
* feat: eable range expr nest

* fix: change range expr rewrite format

* chore: organize range query tests

* chore: change range expr name(e.g. MAX(v) RANGE 5s FILL 6)

* chore: add range query test

* chore: fix code advice

* chore: fix ca
2023-10-18 07:03:26 +00:00
localhost
a7507a2b12 chore: change telemetry report url to resolve connectivity issues (#2608)
chore: change otel report url to resolve connectivity issues
2023-10-18 06:58:54 +00:00
Wei
5b8e5066a0 refactor: make ReadableSize more readable. (#2614)
* refactor: ReadableSize is readable.

* docs: Update src/common/base/src/readable_size.rs

---------

Co-authored-by: Yingwen <realevenyag@gmail.com>
2023-10-18 06:32:50 +00:00
Weny Xu
dcd481e6a4 feat: stop the procedure manager if a new leader is elected (#2576)
* feat: stop the procedure manager if a new leader is elected

* chore: apply suggestions from CR

* chore: apply suggestions

* chore: apply suggestions from CR

* feat: add should_report to GreptimeDBTelemetry

Signed-off-by: WenyXu <wenymedia@gmail.com>

* refactor: refactor subscribing leader change loop

---------

Signed-off-by: WenyXu <wenymedia@gmail.com>
2023-10-18 06:12:28 +00:00
zyy17
3217b56cc1 ci: release new version '0.4.0' -> '0.4.1' (#2611) 2023-10-17 07:33:41 +00:00
shuiyisong
eccad647d0 chore: add export data to migrate tool (#2610)
* chore: add export data to migrate tool

* chore: export copy from sql too
2023-10-17 06:33:58 +00:00
Yun Chen
829db8c5c1 fix!: align frontend cmd name to rpc_* (#2609)
fix: align frontend cmd name to rpc_*
2023-10-17 06:18:18 +00:00
Ruihang Xia
9056c3a6aa feat: implement greptime cli export (#2535)
* feat: implement greptime cli export

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* fix clippy

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* read information schema

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

* parse database name from cli params

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>

---------

Signed-off-by: Ruihang Xia <waynestxia@gmail.com>
2023-10-17 01:56:52 +00:00
ZhangJian He
d9e7b898a3 feat: add walconfig dir back (#2606)
Signed-off-by: ZhangJian He <shoothzj@gmail.com>
2023-10-16 11:26:06 +00:00
zyy17
59d4081f7a ci: correct image name of dev build (#2603) 2023-10-16 03:54:44 +00:00
51 changed files with 2540 additions and 855 deletions

View File

@@ -248,7 +248,7 @@ jobs:
with:
src-image-registry: docker.io
src-image-namespace: ${{ vars.IMAGE_NAMESPACE }}
src-image-name: greptimedb
src-image-name: ${{ env.IMAGE_NAME }}
dst-image-registry-username: ${{ secrets.ALICLOUD_USERNAME }}
dst-image-registry-password: ${{ secrets.ALICLOUD_PASSWORD }}
dst-image-registry: ${{ vars.ACR_IMAGE_REGISTRY }}

122
Cargo.lock generated
View File

@@ -204,7 +204,7 @@ checksum = "8f1f8f5a6f3d50d89e3797d7593a50f96bb2aaa20ca0cc7be1fb673232c91d72"
[[package]]
name = "api"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"common-base",
"common-error",
@@ -666,7 +666,7 @@ dependencies = [
[[package]]
name = "auth"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"async-trait",
@@ -839,7 +839,7 @@ dependencies = [
[[package]]
name = "benchmarks"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"arrow",
"chrono",
@@ -1222,7 +1222,7 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "catalog"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"arc-swap",
@@ -1506,7 +1506,7 @@ checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961"
[[package]]
name = "client"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"arrow-flight",
@@ -1536,7 +1536,7 @@ dependencies = [
"rand",
"session",
"snafu",
"substrait 0.4.0",
"substrait 0.4.1",
"substrait 0.7.5",
"tokio",
"tokio-stream",
@@ -1573,7 +1573,7 @@ dependencies = [
[[package]]
name = "cmd"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"anymap",
"async-trait",
@@ -1621,7 +1621,7 @@ dependencies = [
"servers",
"session",
"snafu",
"substrait 0.4.0",
"substrait 0.4.1",
"table",
"temp-env",
"tikv-jemallocator",
@@ -1654,7 +1654,7 @@ checksum = "55b672471b4e9f9e95499ea597ff64941a309b2cdbffcc46f2cc5e2d971fd335"
[[package]]
name = "common-base"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"anymap",
"bitvec",
@@ -1669,7 +1669,7 @@ dependencies = [
[[package]]
name = "common-catalog"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"chrono",
"common-error",
@@ -1682,7 +1682,7 @@ dependencies = [
[[package]]
name = "common-config"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"common-base",
"humantime-serde",
@@ -1691,7 +1691,7 @@ dependencies = [
[[package]]
name = "common-datasource"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"arrow",
"arrow-schema",
@@ -1720,7 +1720,7 @@ dependencies = [
[[package]]
name = "common-error"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"snafu",
"strum 0.25.0",
@@ -1728,7 +1728,7 @@ dependencies = [
[[package]]
name = "common-function"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"arc-swap",
"chrono-tz 0.6.3",
@@ -1751,7 +1751,7 @@ dependencies = [
[[package]]
name = "common-greptimedb-telemetry"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"async-trait",
"common-error",
@@ -1770,7 +1770,7 @@ dependencies = [
[[package]]
name = "common-grpc"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"arrow-flight",
@@ -1800,7 +1800,7 @@ dependencies = [
[[package]]
name = "common-grpc-expr"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"async-trait",
@@ -1819,7 +1819,7 @@ dependencies = [
[[package]]
name = "common-macro"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"arc-swap",
"backtrace",
@@ -1836,7 +1836,7 @@ dependencies = [
[[package]]
name = "common-mem-prof"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"common-error",
"common-macro",
@@ -1849,7 +1849,7 @@ dependencies = [
[[package]]
name = "common-meta"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"arrow-flight",
@@ -1887,7 +1887,7 @@ dependencies = [
[[package]]
name = "common-procedure"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"async-stream",
"async-trait",
@@ -1911,7 +1911,7 @@ dependencies = [
[[package]]
name = "common-procedure-test"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"async-trait",
"common-procedure",
@@ -1919,7 +1919,7 @@ dependencies = [
[[package]]
name = "common-query"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"async-trait",
@@ -1942,7 +1942,7 @@ dependencies = [
[[package]]
name = "common-recordbatch"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"common-error",
"common-macro",
@@ -1959,7 +1959,7 @@ dependencies = [
[[package]]
name = "common-runtime"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"async-trait",
"common-error",
@@ -1976,7 +1976,7 @@ dependencies = [
[[package]]
name = "common-telemetry"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"backtrace",
"common-error",
@@ -2003,7 +2003,7 @@ dependencies = [
[[package]]
name = "common-test-util"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"once_cell",
"rand",
@@ -2012,7 +2012,7 @@ dependencies = [
[[package]]
name = "common-time"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"arrow",
"chrono",
@@ -2027,7 +2027,7 @@ dependencies = [
[[package]]
name = "common-version"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"build-data",
]
@@ -2665,7 +2665,7 @@ dependencies = [
[[package]]
name = "datanode"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"arrow-flight",
@@ -2724,7 +2724,7 @@ dependencies = [
"sql",
"storage",
"store-api",
"substrait 0.4.0",
"substrait 0.4.1",
"table",
"tokio",
"tokio-stream",
@@ -2738,7 +2738,7 @@ dependencies = [
[[package]]
name = "datatypes"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"arrow",
"arrow-array",
@@ -3201,7 +3201,7 @@ dependencies = [
[[package]]
name = "file-engine"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"async-trait",
@@ -3311,7 +3311,7 @@ dependencies = [
[[package]]
name = "frontend"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"arc-swap",
@@ -3375,7 +3375,7 @@ dependencies = [
"storage",
"store-api",
"strfmt",
"substrait 0.4.0",
"substrait 0.4.1",
"table",
"tokio",
"toml 0.7.6",
@@ -5006,7 +5006,7 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f"
[[package]]
name = "log-store"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"async-stream",
"async-trait",
@@ -5276,7 +5276,7 @@ dependencies = [
[[package]]
name = "meta-client"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"async-trait",
@@ -5306,7 +5306,7 @@ dependencies = [
[[package]]
name = "meta-srv"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"anymap",
"api",
@@ -5498,7 +5498,7 @@ dependencies = [
[[package]]
name = "mito2"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"anymap",
"api",
@@ -5960,7 +5960,7 @@ dependencies = [
[[package]]
name = "object-store"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"anyhow",
"async-trait",
@@ -6184,7 +6184,7 @@ dependencies = [
[[package]]
name = "operator"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"async-compat",
@@ -6229,7 +6229,7 @@ dependencies = [
"sqlparser 0.34.0",
"storage",
"store-api",
"substrait 0.4.0",
"substrait 0.4.1",
"table",
"tokio",
"tonic 0.9.2",
@@ -6449,7 +6449,7 @@ dependencies = [
[[package]]
name = "partition"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"async-trait",
@@ -6775,7 +6775,7 @@ dependencies = [
[[package]]
name = "plugins"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"auth",
"common-base",
@@ -7025,7 +7025,7 @@ dependencies = [
[[package]]
name = "promql"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"async-recursion",
"async-trait",
@@ -7287,7 +7287,7 @@ dependencies = [
[[package]]
name = "query"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"ahash 0.8.3",
"api",
@@ -7344,7 +7344,7 @@ dependencies = [
"stats-cli",
"store-api",
"streaming-stats",
"substrait 0.4.0",
"substrait 0.4.1",
"table",
"tokio",
"tokio-stream",
@@ -8543,7 +8543,7 @@ checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
[[package]]
name = "script"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"arc-swap",
@@ -8823,7 +8823,7 @@ dependencies = [
[[package]]
name = "servers"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"aide",
"api",
@@ -8917,7 +8917,7 @@ dependencies = [
[[package]]
name = "session"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"arc-swap",
@@ -9195,7 +9195,7 @@ dependencies = [
[[package]]
name = "sql"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"common-base",
@@ -9246,7 +9246,7 @@ dependencies = [
[[package]]
name = "sqlness-runner"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"async-trait",
"clap 4.4.1",
@@ -9266,13 +9266,13 @@ dependencies = [
[[package]]
name = "sqlparser"
version = "0.34.0"
source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=296a4f6c73b129d6f565a42a2e5e53c6bc2b9da4#296a4f6c73b129d6f565a42a2e5e53c6bc2b9da4"
source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=6cf9d23d5b8fbecd65efc1d9afb7e80ad7a424da#6cf9d23d5b8fbecd65efc1d9afb7e80ad7a424da"
dependencies = [
"lazy_static",
"log",
"regex",
"sqlparser 0.35.0",
"sqlparser_derive 0.1.1 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=296a4f6c73b129d6f565a42a2e5e53c6bc2b9da4)",
"sqlparser_derive 0.1.1 (git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=6cf9d23d5b8fbecd65efc1d9afb7e80ad7a424da)",
]
[[package]]
@@ -9299,7 +9299,7 @@ dependencies = [
[[package]]
name = "sqlparser_derive"
version = "0.1.1"
source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=296a4f6c73b129d6f565a42a2e5e53c6bc2b9da4#296a4f6c73b129d6f565a42a2e5e53c6bc2b9da4"
source = "git+https://github.com/GreptimeTeam/sqlparser-rs.git?rev=6cf9d23d5b8fbecd65efc1d9afb7e80ad7a424da#6cf9d23d5b8fbecd65efc1d9afb7e80ad7a424da"
dependencies = [
"proc-macro2",
"quote",
@@ -9452,7 +9452,7 @@ dependencies = [
[[package]]
name = "storage"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"arc-swap",
@@ -9506,7 +9506,7 @@ dependencies = [
[[package]]
name = "store-api"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"aquamarine",
@@ -9644,7 +9644,7 @@ dependencies = [
[[package]]
name = "substrait"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"async-recursion",
"async-trait",
@@ -9802,7 +9802,7 @@ dependencies = [
[[package]]
name = "table"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"anymap",
"async-trait",
@@ -9908,7 +9908,7 @@ dependencies = [
[[package]]
name = "tests-integration"
version = "0.4.0"
version = "0.4.1"
dependencies = [
"api",
"async-trait",
@@ -9961,7 +9961,7 @@ dependencies = [
"sql",
"sqlx",
"store-api",
"substrait 0.4.0",
"substrait 0.4.1",
"table",
"tempfile",
"tokio",

View File

@@ -55,7 +55,7 @@ members = [
resolver = "2"
[workspace.package]
version = "0.4.0"
version = "0.4.1"
edition = "2021"
license = "Apache-2.0"
@@ -103,7 +103,7 @@ serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
smallvec = "1"
snafu = { version = "0.7", features = ["backtraces"] }
sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "296a4f6c73b129d6f565a42a2e5e53c6bc2b9da4", features = [
sqlparser = { git = "https://github.com/GreptimeTeam/sqlparser-rs.git", rev = "6cf9d23d5b8fbecd65efc1d9afb7e80ad7a424da", features = [
"visitor",
] }
strum = { version = "0.25", features = ["derive"] }

View File

@@ -82,6 +82,8 @@ enable = true
# WAL options.
[wal]
# WAL data directory
# dir = "/tmp/greptimedb/wal"
# WAL file size in bytes.
file_size = "256MB"
# WAL purge threshold.

View File

@@ -167,11 +167,14 @@ impl Database {
}
}
pub async fn sql(&self, sql: &str) -> Result<Output> {
pub async fn sql<S>(&self, sql: S) -> Result<Output>
where
S: AsRef<str>,
{
let _timer = timer!(metrics::METRIC_GRPC_SQL);
self.do_get(
Request::Query(QueryRequest {
query: Some(Query::Sql(sql.to_string())),
query: Some(Query::Sql(sql.as_ref().to_string())),
}),
0,
)

View File

@@ -14,6 +14,7 @@
mod bench;
mod cmd;
mod export;
mod helper;
mod repl;
// TODO(weny): Removes it
@@ -27,6 +28,7 @@ use common_telemetry::logging::LoggingOptions;
pub use repl::Repl;
use upgrade::UpgradeCommand;
use self::export::ExportCommand;
use crate::error::Result;
use crate::options::{Options, TopLevelOptions};
@@ -81,6 +83,7 @@ enum SubCommand {
// Attach(AttachCommand),
Upgrade(UpgradeCommand),
Bench(BenchTableMetadataCommand),
Export(ExportCommand),
}
impl SubCommand {
@@ -89,6 +92,7 @@ impl SubCommand {
// SubCommand::Attach(cmd) => cmd.build().await,
SubCommand::Upgrade(cmd) => cmd.build().await,
SubCommand::Bench(cmd) => cmd.build().await,
SubCommand::Export(cmd) => cmd.build().await,
}
}
}

395
src/cmd/src/cli/export.rs Normal file
View File

@@ -0,0 +1,395 @@
// Copyright 2023 Greptime Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::path::Path;
use std::sync::Arc;
use async_trait::async_trait;
use clap::{Parser, ValueEnum};
use client::{Client, Database, DEFAULT_SCHEMA_NAME};
use common_query::Output;
use common_recordbatch::util::collect;
use common_telemetry::{debug, error, info, warn};
use datatypes::scalars::ScalarVector;
use datatypes::vectors::{StringVector, Vector};
use snafu::{OptionExt, ResultExt};
use tokio::fs::File;
use tokio::io::AsyncWriteExt;
use tokio::sync::Semaphore;
use crate::cli::{Instance, Tool};
use crate::error::{
CollectRecordBatchesSnafu, ConnectServerSnafu, EmptyResultSnafu, Error, FileIoSnafu,
InvalidDatabaseNameSnafu, NotDataFromOutputSnafu, RequestDatabaseSnafu, Result,
};
type TableReference = (String, String, String);
#[derive(Debug, Default, Clone, ValueEnum)]
enum ExportTarget {
/// Corresponding to `SHOW CREATE TABLE`
#[default]
CreateTable,
/// Corresponding to `EXPORT TABLE`
TableData,
}
#[derive(Debug, Default, Parser)]
pub struct ExportCommand {
/// Server address to connect
#[clap(long)]
addr: String,
/// Directory to put the exported data. E.g.: /tmp/greptimedb-export
#[clap(long)]
output_dir: String,
/// The name of the catalog to export. Default to "greptime-*"".
#[clap(long, default_value = "")]
database: String,
/// Parallelism of the export.
#[clap(long, short = 'j', default_value = "1")]
export_jobs: usize,
/// Max retry times for each job.
#[clap(long, default_value = "3")]
max_retry: usize,
/// Things to export
#[clap(long, short = 't', value_enum)]
target: ExportTarget,
}
impl ExportCommand {
pub async fn build(&self) -> Result<Instance> {
let client = Client::with_urls([self.addr.clone()]);
client
.health_check()
.await
.with_context(|_| ConnectServerSnafu {
addr: self.addr.clone(),
})?;
let (catalog, schema) = split_database(&self.database)?;
let database_client = Database::new(
catalog.clone(),
schema.clone().unwrap_or(DEFAULT_SCHEMA_NAME.to_string()),
client,
);
Ok(Instance::Tool(Box::new(Export {
client: database_client,
catalog,
schema,
output_dir: self.output_dir.clone(),
parallelism: self.export_jobs,
target: self.target.clone(),
})))
}
}
pub struct Export {
client: Database,
catalog: String,
schema: Option<String>,
output_dir: String,
parallelism: usize,
target: ExportTarget,
}
impl Export {
/// Iterate over all db names.
///
/// Newbie: `db_name` is catalog + schema.
async fn iter_db_names(&self) -> Result<Vec<(String, String)>> {
if let Some(schema) = &self.schema {
Ok(vec![(self.catalog.clone(), schema.clone())])
} else {
let mut client = self.client.clone();
client.set_catalog(self.catalog.clone());
let result =
client
.sql("show databases")
.await
.with_context(|_| RequestDatabaseSnafu {
sql: "show databases".to_string(),
})?;
let Output::Stream(stream) = result else {
NotDataFromOutputSnafu.fail()?
};
let record_batch = collect(stream)
.await
.context(CollectRecordBatchesSnafu)?
.pop()
.context(EmptyResultSnafu)?;
let schemas = record_batch
.column(0)
.as_any()
.downcast_ref::<StringVector>()
.unwrap();
let mut result = Vec::with_capacity(schemas.len());
for i in 0..schemas.len() {
let schema = schemas.get_data(i).unwrap().to_owned();
result.push((self.catalog.clone(), schema));
}
Ok(result)
}
}
/// Return a list of [`TableReference`] to be exported.
/// Includes all tables under the given `catalog` and `schema`
async fn get_table_list(&self, catalog: &str, schema: &str) -> Result<Vec<TableReference>> {
// TODO: SQL injection hurts
let sql = format!(
"select table_catalog, table_schema, table_name from \
information_schema.tables where table_type = \'BASE TABLE\'\
and table_catalog = \'{catalog}\' and table_schema = \'{schema}\'",
);
let mut client = self.client.clone();
client.set_catalog(catalog);
client.set_schema(schema);
let result = client
.sql(&sql)
.await
.with_context(|_| RequestDatabaseSnafu { sql })?;
let Output::Stream(stream) = result else {
NotDataFromOutputSnafu.fail()?
};
let Some(record_batch) = collect(stream)
.await
.context(CollectRecordBatchesSnafu)?
.pop()
else {
return Ok(vec![]);
};
debug!("Fetched table list: {}", record_batch.pretty_print());
if record_batch.num_rows() == 0 {
return Ok(vec![]);
}
let mut result = Vec::with_capacity(record_batch.num_rows());
let catalog_column = record_batch
.column(0)
.as_any()
.downcast_ref::<StringVector>()
.unwrap();
let schema_column = record_batch
.column(1)
.as_any()
.downcast_ref::<StringVector>()
.unwrap();
let table_column = record_batch
.column(2)
.as_any()
.downcast_ref::<StringVector>()
.unwrap();
for i in 0..record_batch.num_rows() {
let catalog = catalog_column.get_data(i).unwrap().to_owned();
let schema = schema_column.get_data(i).unwrap().to_owned();
let table = table_column.get_data(i).unwrap().to_owned();
result.push((catalog, schema, table));
}
Ok(result)
}
async fn show_create_table(&self, catalog: &str, schema: &str, table: &str) -> Result<String> {
let sql = format!("show create table {}.{}.{}", catalog, schema, table);
let mut client = self.client.clone();
client.set_catalog(catalog);
client.set_schema(schema);
let result = client
.sql(&sql)
.await
.with_context(|_| RequestDatabaseSnafu { sql })?;
let Output::Stream(stream) = result else {
NotDataFromOutputSnafu.fail()?
};
let record_batch = collect(stream)
.await
.context(CollectRecordBatchesSnafu)?
.pop()
.context(EmptyResultSnafu)?;
let create_table = record_batch
.column(1)
.as_any()
.downcast_ref::<StringVector>()
.unwrap()
.get_data(0)
.unwrap();
Ok(format!("{create_table};\n"))
}
async fn export_create_table(&self) -> Result<()> {
let semaphore = Arc::new(Semaphore::new(self.parallelism));
let db_names = self.iter_db_names().await?;
let db_count = db_names.len();
let mut tasks = Vec::with_capacity(db_names.len());
for (catalog, schema) in db_names {
let semaphore_moved = semaphore.clone();
tasks.push(async move {
let _permit = semaphore_moved.acquire().await.unwrap();
let table_list = self.get_table_list(&catalog, &schema).await?;
let table_count = table_list.len();
tokio::fs::create_dir_all(&self.output_dir)
.await
.context(FileIoSnafu)?;
let output_file =
Path::new(&self.output_dir).join(format!("{catalog}-{schema}.sql"));
let mut file = File::create(output_file).await.context(FileIoSnafu)?;
for (c, s, t) in table_list {
match self.show_create_table(&c, &s, &t).await {
Err(e) => {
error!(e; "Failed to export table {}.{}.{}", c, s, t)
}
Ok(create_table) => {
file.write_all(create_table.as_bytes())
.await
.context(FileIoSnafu)?;
}
}
}
info!("finished exporting {catalog}.{schema} with {table_count} tables",);
Ok::<(), Error>(())
});
}
let success = futures::future::join_all(tasks)
.await
.into_iter()
.filter(|r| match r {
Ok(_) => true,
Err(e) => {
error!(e; "export job failed");
false
}
})
.count();
info!("success {success}/{db_count} jobs");
Ok(())
}
async fn export_table_data(&self) -> Result<()> {
let semaphore = Arc::new(Semaphore::new(self.parallelism));
let db_names = self.iter_db_names().await?;
let db_count = db_names.len();
let mut tasks = Vec::with_capacity(db_names.len());
for (catalog, schema) in db_names {
let semaphore_moved = semaphore.clone();
tasks.push(async move {
let _permit = semaphore_moved.acquire().await.unwrap();
tokio::fs::create_dir_all(&self.output_dir)
.await
.context(FileIoSnafu)?;
let output_dir = Path::new(&self.output_dir).join(format!("{catalog}-{schema}/"));
let mut client = self.client.clone();
client.set_catalog(catalog.clone());
client.set_schema(schema.clone());
// copy database to
let sql = format!(
"copy database {} to '{}' with (format='parquet');",
schema,
output_dir.to_str().unwrap()
);
client
.sql(sql.clone())
.await
.context(RequestDatabaseSnafu { sql })?;
info!("finished exporting {catalog}.{schema} data");
// export copy from sql
let dir_filenames = match output_dir.read_dir() {
Ok(dir) => dir,
Err(_) => {
warn!("empty database {catalog}.{schema}");
return Ok(());
}
};
let copy_from_file =
Path::new(&self.output_dir).join(format!("{catalog}-{schema}_copy_from.sql"));
let mut file = File::create(copy_from_file).await.context(FileIoSnafu)?;
let copy_from_sql = dir_filenames
.into_iter()
.map(|file| {
let file = file.unwrap();
let filename = file.file_name().into_string().unwrap();
format!(
"copy {} from '{}' with (format='parquet');\n",
filename.replace(".parquet", ""),
file.path().to_str().unwrap()
)
})
.collect::<Vec<_>>()
.join("");
file.write_all(copy_from_sql.as_bytes())
.await
.context(FileIoSnafu)?;
info!("finished exporting {catalog}.{schema} copy_from.sql");
Ok::<(), Error>(())
});
}
let success = futures::future::join_all(tasks)
.await
.into_iter()
.filter(|r| match r {
Ok(_) => true,
Err(e) => {
error!(e; "export job failed");
false
}
})
.count();
info!("success {success}/{db_count} jobs");
Ok(())
}
}
#[async_trait]
impl Tool for Export {
async fn do_work(&self) -> Result<()> {
match self.target {
ExportTarget::CreateTable => self.export_create_table().await,
ExportTarget::TableData => self.export_table_data().await,
}
}
}
/// Split at `-`.
fn split_database(database: &str) -> Result<(String, Option<String>)> {
let (catalog, schema) = database
.split_once('-')
.with_context(|| InvalidDatabaseNameSnafu {
database: database.to_string(),
})?;
if schema == "*" {
Ok((catalog.to_string(), None))
} else {
Ok((catalog.to_string(), Some(schema.to_string())))
}
}

View File

@@ -96,6 +96,8 @@ struct StartCommand {
#[clap(long)]
data_home: Option<String>,
#[clap(long)]
wal_dir: Option<String>,
#[clap(long)]
http_addr: Option<String>,
#[clap(long)]
http_timeout: Option<u64>,
@@ -149,6 +151,10 @@ impl StartCommand {
opts.storage.data_home = data_home.clone();
}
if let Some(wal_dir) = &self.wal_dir {
opts.wal.dir = Some(wal_dir.clone());
}
if let Some(http_addr) = &self.http_addr {
opts.http.addr = http_addr.clone();
}
@@ -255,6 +261,7 @@ mod tests {
assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr);
assert_eq!(Some(42), options.node_id);
assert_eq!("/other/wal", options.wal.dir.unwrap());
assert_eq!(Duration::from_secs(600), options.wal.purge_interval);
assert_eq!(1024 * 1024 * 1024, options.wal.file_size.0);
@@ -439,6 +446,7 @@ mod tests {
|| {
let command = StartCommand {
config_file: Some(file.path().to_str().unwrap().to_string()),
wal_dir: Some("/other/wal/dir".to_string()),
env_prefix: env_prefix.to_string(),
..Default::default()
};
@@ -466,6 +474,9 @@ mod tests {
// Should be read from config file, config file > env > default values.
assert_eq!(opts.storage.compaction.max_purge_tasks, 32);
// Should be read from cli, cli > config file > env > default values.
assert_eq!(opts.wal.dir.unwrap(), "/other/wal/dir");
// Should be default value.
assert_eq!(
opts.storage.manifest.checkpoint_margin,

View File

@@ -37,6 +37,18 @@ pub enum Error {
source: common_meta::error::Error,
},
#[snafu(display("Failed to start procedure manager"))]
StartProcedureManager {
location: Location,
source: common_procedure::error::Error,
},
#[snafu(display("Failed to stop procedure manager"))]
StopProcedureManager {
location: Location,
source: common_procedure::error::Error,
},
#[snafu(display("Failed to start datanode"))]
StartDatanode {
location: Location,
@@ -174,12 +186,39 @@ pub enum Error {
location: Location,
},
#[snafu(display("Failed to connect server at {addr}"))]
ConnectServer {
addr: String,
source: client::error::Error,
location: Location,
},
#[snafu(display("Failed to serde json"))]
SerdeJson {
#[snafu(source)]
error: serde_json::error::Error,
location: Location,
},
#[snafu(display("Expect data from output, but got another thing"))]
NotDataFromOutput { location: Location },
#[snafu(display("Empty result from output"))]
EmptyResult { location: Location },
#[snafu(display("Failed to manipulate file"))]
FileIo {
location: Location,
#[snafu(source)]
error: std::io::Error,
},
#[snafu(display("Invalid database name: {}", database))]
InvalidDatabaseName {
location: Location,
database: String,
},
#[snafu(display("Failed to create directory {}", dir))]
CreateDir {
dir: String,
@@ -204,13 +243,18 @@ impl ErrorExt for Error {
Error::IterStream { source, .. } | Error::InitMetadata { source, .. } => {
source.status_code()
}
Error::ConnectServer { source, .. } => source.status_code(),
Error::MissingConfig { .. }
| Error::LoadLayeredConfig { .. }
| Error::IllegalConfig { .. }
| Error::InvalidReplCommand { .. }
| Error::ConnectEtcd { .. }
| Error::NotDataFromOutput { .. }
| Error::CreateDir { .. }
| Error::ConnectEtcd { .. } => StatusCode::InvalidArguments,
| Error::EmptyResult { .. }
| Error::InvalidDatabaseName { .. } => StatusCode::InvalidArguments,
Error::StartProcedureManager { source, .. }
| Error::StopProcedureManager { source, .. } => source.status_code(),
Error::ReplCreation { .. } | Error::Readline { .. } => StatusCode::Internal,
Error::RequestDatabase { source, .. } => source.status_code(),
Error::CollectRecordBatches { source, .. }
@@ -222,7 +266,7 @@ impl ErrorExt for Error {
Error::SubstraitEncodeLogicalPlan { source, .. } => source.status_code(),
Error::StartCatalogManager { source, .. } => source.status_code(),
Error::SerdeJson { .. } => StatusCode::Unexpected,
Error::SerdeJson { .. } | Error::FileIo { .. } => StatusCode::Unexpected,
}
}

View File

@@ -89,7 +89,7 @@ pub struct StartCommand {
#[clap(long)]
http_timeout: Option<u64>,
#[clap(long)]
grpc_addr: Option<String>,
rpc_addr: Option<String>,
#[clap(long)]
mysql_addr: Option<String>,
#[clap(long)]
@@ -150,7 +150,7 @@ impl StartCommand {
opts.http.disable_dashboard = disable_dashboard;
}
if let Some(addr) = &self.grpc_addr {
if let Some(addr) = &self.rpc_addr {
opts.grpc.addr = addr.clone()
}

View File

@@ -263,6 +263,9 @@ mod tests {
]
);
// Should be the values from config file, not environment variables.
assert_eq!(opts.wal.dir.unwrap(), "/tmp/greptimedb/wal");
// Should be default values.
assert_eq!(opts.node_id, None);
},

View File

@@ -43,7 +43,8 @@ use snafu::ResultExt;
use crate::error::{
CreateDirSnafu, IllegalConfigSnafu, InitMetadataSnafu, Result, ShutdownDatanodeSnafu,
ShutdownFrontendSnafu, StartDatanodeSnafu, StartFrontendSnafu,
ShutdownFrontendSnafu, StartDatanodeSnafu, StartFrontendSnafu, StartProcedureManagerSnafu,
StopProcedureManagerSnafu,
};
use crate::options::{MixOptions, Options, TopLevelOptions};
@@ -163,6 +164,7 @@ impl StandaloneOptions {
pub struct Instance {
datanode: Datanode,
frontend: FeInstance,
procedure_manager: ProcedureManagerRef,
}
impl Instance {
@@ -171,6 +173,11 @@ impl Instance {
self.datanode.start().await.context(StartDatanodeSnafu)?;
info!("Datanode instance started");
self.procedure_manager
.start()
.await
.context(StartProcedureManagerSnafu)?;
self.frontend.start().await.context(StartFrontendSnafu)?;
Ok(())
}
@@ -181,6 +188,11 @@ impl Instance {
.await
.context(ShutdownFrontendSnafu)?;
self.procedure_manager
.stop()
.await
.context(StopProcedureManagerSnafu)?;
self.datanode
.shutdown()
.await
@@ -354,7 +366,7 @@ impl StartCommand {
let mut frontend = build_frontend(
fe_plugins,
kv_store,
procedure_manager,
procedure_manager.clone(),
catalog_manager,
region_server,
)
@@ -365,7 +377,11 @@ impl StartCommand {
.await
.context(StartFrontendSnafu)?;
Ok(Instance { datanode, frontend })
Ok(Instance {
datanode,
frontend,
procedure_manager,
})
}
}
@@ -493,6 +509,8 @@ mod tests {
assert_eq!(None, fe_opts.mysql.reject_no_database);
assert!(fe_opts.influxdb.enable);
assert_eq!("/tmp/greptimedb/test/wal", dn_opts.wal.dir.unwrap());
match &dn_opts.storage.store {
datanode::config::ObjectStoreConfig::S3(s3_config) => {
assert_eq!(

View File

@@ -14,10 +14,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// This file is copied from https://github.com/tikv/raft-engine/blob/8dd2a39f359ff16f5295f35343f626e0c10132fa/src/util.rs without any modification.
// This file is copied from https://github.com/tikv/raft-engine/blob/8dd2a39f359ff16f5295f35343f626e0c10132fa/src/util.rs
use std::fmt;
use std::fmt::{Display, Write};
use std::fmt::{self, Debug, Display, Write};
use std::ops::{Div, Mul};
use std::str::FromStr;
@@ -34,7 +33,7 @@ pub const GIB: u64 = MIB * BINARY_DATA_MAGNITUDE;
pub const TIB: u64 = GIB * BINARY_DATA_MAGNITUDE;
pub const PIB: u64 = TIB * BINARY_DATA_MAGNITUDE;
#[derive(Clone, Debug, Copy, PartialEq, Eq, PartialOrd)]
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd)]
pub struct ReadableSize(pub u64);
impl ReadableSize {
@@ -155,6 +154,12 @@ impl FromStr for ReadableSize {
}
}
impl Debug for ReadableSize {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self)
}
}
impl Display for ReadableSize {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if self.0 >= PIB {

View File

@@ -20,6 +20,8 @@ use serde::{Deserialize, Serialize};
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
#[serde(default)]
pub struct WalConfig {
// wal directory
pub dir: Option<String>,
// wal file size in bytes
pub file_size: ReadableSize,
// wal purge threshold in bytes
@@ -36,7 +38,8 @@ pub struct WalConfig {
impl Default for WalConfig {
fn default() -> Self {
Self {
file_size: ReadableSize::mb(256), // log file size 256MB
dir: None,
file_size: ReadableSize::mb(256), // log file size 256MB
purge_threshold: ReadableSize::gb(4), // purge threshold 4GB
purge_interval: Duration::from_secs(600),
read_batch_size: 128,

View File

@@ -58,9 +58,15 @@ impl Function for RangeFunction {
"range_fn"
}
// range_fn will never been used, return_type could be arbitrary value, is not important
fn return_type(&self, _input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
Ok(ConcreteDataType::float64_datatype())
// The first argument to range_fn is the expression to be evaluated
fn return_type(&self, input_types: &[ConcreteDataType]) -> Result<ConcreteDataType> {
input_types
.first()
.cloned()
.ok_or(DataFusionError::Internal(
"No expr found in range_fn".into(),
))
.context(GeneralDataFusionSnafu)
}
/// `range_fn` will never been used. As long as a legal signature is returned, the specific content of the signature does not matter.

View File

@@ -15,6 +15,8 @@
use std::env;
use std::io::ErrorKind;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use std::time::Duration;
use common_runtime::error::{Error, Result};
@@ -24,7 +26,7 @@ use reqwest::{Client, Response};
use serde::{Deserialize, Serialize};
/// The URL to report telemetry data.
pub const TELEMETRY_URL: &str = "https://api.greptime.cloud/db/otel/statistics";
pub const TELEMETRY_URL: &str = "https://telemetry.greptimestats.com/db/otel/statistics";
/// The local installation uuid cache file
const UUID_FILE_NAME: &str = ".greptimedb-telemetry-uuid";
@@ -36,13 +38,26 @@ const GREPTIMEDB_TELEMETRY_CLIENT_CONNECT_TIMEOUT: Duration = Duration::from_sec
const GREPTIMEDB_TELEMETRY_CLIENT_REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
pub enum GreptimeDBTelemetryTask {
Enable(RepeatedTask<Error>),
Enable((RepeatedTask<Error>, Arc<AtomicBool>)),
Disable,
}
impl GreptimeDBTelemetryTask {
pub fn enable(interval: Duration, task_fn: BoxedTaskFunction<Error>) -> Self {
GreptimeDBTelemetryTask::Enable(RepeatedTask::new(interval, task_fn))
pub fn should_report(&self, value: bool) {
match self {
GreptimeDBTelemetryTask::Enable((_, should_report)) => {
should_report.store(value, Ordering::Relaxed);
}
GreptimeDBTelemetryTask::Disable => {}
}
}
pub fn enable(
interval: Duration,
task_fn: BoxedTaskFunction<Error>,
should_report: Arc<AtomicBool>,
) -> Self {
GreptimeDBTelemetryTask::Enable((RepeatedTask::new(interval, task_fn), should_report))
}
pub fn disable() -> Self {
@@ -51,7 +66,7 @@ impl GreptimeDBTelemetryTask {
pub fn start(&self) -> Result<()> {
match self {
GreptimeDBTelemetryTask::Enable(task) => {
GreptimeDBTelemetryTask::Enable((task, _)) => {
print_anonymous_usage_data_disclaimer();
task.start(common_runtime::bg_runtime())
}
@@ -61,7 +76,7 @@ impl GreptimeDBTelemetryTask {
pub async fn stop(&self) -> Result<()> {
match self {
GreptimeDBTelemetryTask::Enable(task) => task.stop().await,
GreptimeDBTelemetryTask::Enable((task, _)) => task.stop().await,
GreptimeDBTelemetryTask::Disable => Ok(()),
}
}
@@ -191,6 +206,7 @@ pub struct GreptimeDBTelemetry {
client: Option<Client>,
working_home: Option<String>,
telemetry_url: &'static str,
should_report: Arc<AtomicBool>,
}
#[async_trait::async_trait]
@@ -200,13 +216,19 @@ impl TaskFunction<Error> for GreptimeDBTelemetry {
}
async fn call(&mut self) -> Result<()> {
self.report_telemetry_info().await;
if self.should_report.load(Ordering::Relaxed) {
self.report_telemetry_info().await;
}
Ok(())
}
}
impl GreptimeDBTelemetry {
pub fn new(working_home: Option<String>, statistics: Box<dyn Collector + Send + Sync>) -> Self {
pub fn new(
working_home: Option<String>,
statistics: Box<dyn Collector + Send + Sync>,
should_report: Arc<AtomicBool>,
) -> Self {
let client = Client::builder()
.connect_timeout(GREPTIMEDB_TELEMETRY_CLIENT_CONNECT_TIMEOUT)
.timeout(GREPTIMEDB_TELEMETRY_CLIENT_REQUEST_TIMEOUT)
@@ -216,6 +238,7 @@ impl GreptimeDBTelemetry {
statistics,
client: client.ok(),
telemetry_url: TELEMETRY_URL,
should_report,
}
}
@@ -250,7 +273,8 @@ impl GreptimeDBTelemetry {
mod tests {
use std::convert::Infallible;
use std::env;
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::{AtomicBool, AtomicUsize};
use std::sync::Arc;
use std::time::Duration;
use common_test_util::ports;
@@ -370,7 +394,11 @@ mod tests {
let working_home = working_home_temp.path().to_str().unwrap().to_string();
let test_statistic = Box::new(TestStatistic);
let mut test_report = GreptimeDBTelemetry::new(Some(working_home.clone()), test_statistic);
let mut test_report = GreptimeDBTelemetry::new(
Some(working_home.clone()),
test_statistic,
Arc::new(AtomicBool::new(true)),
);
let url = Box::leak(format!("{}:{}", "http://localhost", port).into_boxed_str());
test_report.telemetry_url = url;
let response = test_report.report_telemetry_info().await.unwrap();
@@ -384,7 +412,11 @@ mod tests {
assert_eq!(1, body.nodes.unwrap());
let failed_statistic = Box::new(FailedStatistic);
let mut failed_report = GreptimeDBTelemetry::new(Some(working_home), failed_statistic);
let mut failed_report = GreptimeDBTelemetry::new(
Some(working_home),
failed_statistic,
Arc::new(AtomicBool::new(true)),
);
failed_report.telemetry_url = url;
let response = failed_report.report_telemetry_info().await;
assert!(response.is_none());

View File

@@ -258,7 +258,6 @@ impl<T: Serialize + DeserializeOwned> DeserializedValueWithBytes<T> {
self.bytes.to_vec()
}
#[cfg(feature = "testing")]
/// Notes: used for test purpose.
pub fn from_inner(inner: T) -> Self {
let bytes = serde_json::to_vec(&inner).unwrap();

View File

@@ -34,6 +34,9 @@ pub enum Error {
#[snafu(display("Loader {} is already registered", name))]
LoaderConflict { name: String, location: Location },
#[snafu(display("Procedure Manager is stopped"))]
ManagerNotStart { location: Location },
#[snafu(display("Failed to serialize to json"))]
ToJson {
#[snafu(source)]
@@ -148,7 +151,8 @@ impl ErrorExt for Error {
| Error::FromJson { .. }
| Error::RetryTimesExceeded { .. }
| Error::RetryLater { .. }
| Error::WaitWatcher { .. } => StatusCode::Internal,
| Error::WaitWatcher { .. }
| Error::ManagerNotStart { .. } => StatusCode::Internal,
Error::LoaderConflict { .. } | Error::DuplicateProcedure { .. } => {
StatusCode::InvalidArguments
}

View File

@@ -14,6 +14,8 @@
//! Common traits and structures for the procedure framework.
#![feature(assert_matches)]
pub mod error;
pub mod local;
pub mod options;

View File

@@ -16,20 +16,21 @@ mod lock;
mod runner;
use std::collections::{HashMap, VecDeque};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, RwLock};
use std::time::{Duration, Instant};
use async_trait::async_trait;
use backon::ExponentialBuilder;
use common_runtime::{RepeatedTask, TaskFunction};
use common_telemetry::logging;
use common_telemetry::{info, logging};
use snafu::{ensure, ResultExt};
use tokio::sync::watch::{self, Receiver, Sender};
use tokio::sync::Notify;
use tokio::sync::{Mutex as TokioMutex, Notify};
use crate::error::{
DuplicateProcedureSnafu, Error, LoaderConflictSnafu, Result, StartRemoveOutdatedMetaTaskSnafu,
StopRemoveOutdatedMetaTaskSnafu,
DuplicateProcedureSnafu, Error, LoaderConflictSnafu, ManagerNotStartSnafu, Result,
StartRemoveOutdatedMetaTaskSnafu, StopRemoveOutdatedMetaTaskSnafu,
};
use crate::local::lock::LockMap;
use crate::local::runner::Runner;
@@ -135,6 +136,8 @@ pub(crate) struct ManagerContext {
messages: Mutex<HashMap<ProcedureId, ProcedureMessage>>,
/// Ids and finished time of finished procedures.
finished_procedures: Mutex<VecDeque<(ProcedureId, Instant)>>,
/// Running flag.
running: Arc<AtomicBool>,
}
#[async_trait]
@@ -153,9 +156,29 @@ impl ManagerContext {
procedures: RwLock::new(HashMap::new()),
messages: Mutex::new(HashMap::new()),
finished_procedures: Mutex::new(VecDeque::new()),
running: Arc::new(AtomicBool::new(false)),
}
}
#[cfg(test)]
pub(crate) fn set_running(&self) {
self.running.store(true, Ordering::Relaxed);
}
/// Set the running flag.
pub(crate) fn start(&self) {
self.running.store(true, Ordering::Relaxed);
}
pub(crate) fn stop(&self) {
self.running.store(false, Ordering::Relaxed);
}
/// Return `ProcedureManager` is running.
pub(crate) fn running(&self) -> bool {
self.running.load(Ordering::Relaxed)
}
/// Returns true if the procedure with specific `procedure_id` exists.
fn contains_procedure(&self, procedure_id: ProcedureId) -> bool {
let procedures = self.procedures.read().unwrap();
@@ -368,29 +391,37 @@ pub struct LocalManager {
procedure_store: Arc<ProcedureStore>,
max_retry_times: usize,
retry_delay: Duration,
remove_outdated_meta_task: RepeatedTask<Error>,
/// GC task.
remove_outdated_meta_task: TokioMutex<Option<RepeatedTask<Error>>>,
config: ManagerConfig,
}
impl LocalManager {
/// Create a new [LocalManager] with specific `config`.
pub fn new(config: ManagerConfig, state_store: StateStoreRef) -> LocalManager {
let manager_ctx = Arc::new(ManagerContext::new());
let remove_outdated_meta_task = RepeatedTask::new(
config.remove_outdated_meta_task_interval,
Box::new(RemoveOutdatedMetaFunction {
manager_ctx: manager_ctx.clone(),
ttl: config.remove_outdated_meta_ttl,
}),
);
LocalManager {
manager_ctx,
procedure_store: Arc::new(ProcedureStore::new(&config.parent_path, state_store)),
max_retry_times: config.max_retry_times,
retry_delay: config.retry_delay,
remove_outdated_meta_task,
remove_outdated_meta_task: TokioMutex::new(None),
config,
}
}
/// Build remove outedated meta task
pub fn build_remove_outdated_meta_task(&self) -> RepeatedTask<Error> {
RepeatedTask::new(
self.config.remove_outdated_meta_task_interval,
Box::new(RemoveOutdatedMetaFunction {
manager_ctx: self.manager_ctx.clone(),
ttl: self.config.remove_outdated_meta_ttl,
}),
)
}
/// Submit a root procedure with given `procedure_id`.
fn submit_root(
&self,
@@ -398,6 +429,8 @@ impl LocalManager {
step: u32,
procedure: BoxedProcedure,
) -> Result<Watcher> {
ensure!(self.manager_ctx.running(), ManagerNotStartSnafu);
let meta = Arc::new(ProcedureMeta::new(procedure_id, None, procedure.lock_key()));
let runner = Runner {
meta: meta.clone(),
@@ -426,44 +459,8 @@ impl LocalManager {
Ok(watcher)
}
}
#[async_trait]
impl ProcedureManager for LocalManager {
fn register_loader(&self, name: &str, loader: BoxedProcedureLoader) -> Result<()> {
let mut loaders = self.manager_ctx.loaders.lock().unwrap();
ensure!(!loaders.contains_key(name), LoaderConflictSnafu { name });
let _ = loaders.insert(name.to_string(), loader);
Ok(())
}
fn start(&self) -> Result<()> {
self.remove_outdated_meta_task
.start(common_runtime::bg_runtime())
.context(StartRemoveOutdatedMetaTaskSnafu)?;
Ok(())
}
async fn stop(&self) -> Result<()> {
self.remove_outdated_meta_task
.stop()
.await
.context(StopRemoveOutdatedMetaTaskSnafu)?;
Ok(())
}
async fn submit(&self, procedure: ProcedureWithId) -> Result<Watcher> {
let procedure_id = procedure.id;
ensure!(
!self.manager_ctx.contains_procedure(procedure_id),
DuplicateProcedureSnafu { procedure_id }
);
self.submit_root(procedure.id, 0, procedure.procedure)
}
/// Recovers unfinished procedures and reruns them.
async fn recover(&self) -> Result<()> {
logging::info!("LocalManager start to recover");
let recover_start = Instant::now();
@@ -519,6 +516,64 @@ impl ProcedureManager for LocalManager {
Ok(())
}
}
#[async_trait]
impl ProcedureManager for LocalManager {
fn register_loader(&self, name: &str, loader: BoxedProcedureLoader) -> Result<()> {
let mut loaders = self.manager_ctx.loaders.lock().unwrap();
ensure!(!loaders.contains_key(name), LoaderConflictSnafu { name });
let _ = loaders.insert(name.to_string(), loader);
Ok(())
}
async fn start(&self) -> Result<()> {
let mut task = self.remove_outdated_meta_task.lock().await;
if task.is_some() {
return Ok(());
}
let task_inner = self.build_remove_outdated_meta_task();
task_inner
.start(common_runtime::bg_runtime())
.context(StartRemoveOutdatedMetaTaskSnafu)?;
*task = Some(task_inner);
self.manager_ctx.start();
info!("LocalManager is start.");
self.recover().await
}
async fn stop(&self) -> Result<()> {
let mut task = self.remove_outdated_meta_task.lock().await;
if let Some(task) = task.take() {
task.stop().await.context(StopRemoveOutdatedMetaTaskSnafu)?;
}
self.manager_ctx.stop();
info!("LocalManager is stopped.");
Ok(())
}
async fn submit(&self, procedure: ProcedureWithId) -> Result<Watcher> {
let procedure_id = procedure.id;
ensure!(
!self.manager_ctx.contains_procedure(procedure_id),
DuplicateProcedureSnafu { procedure_id }
);
self.submit_root(procedure.id, 0, procedure.procedure)
}
async fn procedure_state(&self, procedure_id: ProcedureId) -> Result<Option<ProcedureState>> {
Ok(self.manager_ctx.state(procedure_id))
@@ -569,12 +624,14 @@ pub(crate) mod test_util {
#[cfg(test)]
mod tests {
use std::assert_matches::assert_matches;
use common_error::mock::MockError;
use common_error::status_code::StatusCode;
use common_test_util::temp_dir::create_temp_dir;
use super::*;
use crate::error::Error;
use crate::error::{self, Error};
use crate::store::state_store::ObjectStateStore;
use crate::{Context, Procedure, Status};
@@ -691,6 +748,7 @@ mod tests {
};
let state_store = Arc::new(ObjectStateStore::new(test_util::new_object_store(&dir)));
let manager = LocalManager::new(config, state_store);
manager.manager_ctx.start();
manager
.register_loader("ProcedureToLoad", ProcedureToLoad::loader())
@@ -714,6 +772,7 @@ mod tests {
};
let state_store = Arc::new(ObjectStateStore::new(object_store.clone()));
let manager = LocalManager::new(config, state_store);
manager.manager_ctx.start();
manager
.register_loader("ProcedureToLoad", ProcedureToLoad::loader())
@@ -762,6 +821,7 @@ mod tests {
};
let state_store = Arc::new(ObjectStateStore::new(test_util::new_object_store(&dir)));
let manager = LocalManager::new(config, state_store);
manager.manager_ctx.start();
let procedure_id = ProcedureId::random();
assert!(manager
@@ -812,6 +872,7 @@ mod tests {
};
let state_store = Arc::new(ObjectStateStore::new(test_util::new_object_store(&dir)));
let manager = LocalManager::new(config, state_store);
manager.manager_ctx.start();
#[derive(Debug)]
struct MockProcedure {
@@ -864,6 +925,66 @@ mod tests {
}
#[tokio::test]
async fn test_procedure_manager_stopped() {
let dir = create_temp_dir("procedure_manager_stopped");
let config = ManagerConfig {
parent_path: "data/".to_string(),
max_retry_times: 3,
retry_delay: Duration::from_millis(500),
..Default::default()
};
let state_store = Arc::new(ObjectStateStore::new(test_util::new_object_store(&dir)));
let manager = LocalManager::new(config, state_store);
let mut procedure = ProcedureToLoad::new("submit");
procedure.lock_key = LockKey::single("test.submit");
let procedure_id = ProcedureId::random();
assert_matches!(
manager
.submit(ProcedureWithId {
id: procedure_id,
procedure: Box::new(procedure),
})
.await
.unwrap_err(),
error::Error::ManagerNotStart { .. }
);
}
#[tokio::test]
async fn test_procedure_manager_restart() {
let dir = create_temp_dir("procedure_manager_restart");
let config = ManagerConfig {
parent_path: "data/".to_string(),
max_retry_times: 3,
retry_delay: Duration::from_millis(500),
..Default::default()
};
let state_store = Arc::new(ObjectStateStore::new(test_util::new_object_store(&dir)));
let manager = LocalManager::new(config, state_store);
manager.start().await.unwrap();
manager.stop().await.unwrap();
manager.start().await.unwrap();
let mut procedure = ProcedureToLoad::new("submit");
procedure.lock_key = LockKey::single("test.submit");
let procedure_id = ProcedureId::random();
assert!(manager
.submit(ProcedureWithId {
id: procedure_id,
procedure: Box::new(procedure),
})
.await
.is_ok());
assert!(manager
.procedure_state(procedure_id)
.await
.unwrap()
.is_some());
}
#[tokio::test(flavor = "multi_thread")]
async fn test_remove_outdated_meta_task() {
let dir = create_temp_dir("remove_outdated_meta_task");
let object_store = test_util::new_object_store(&dir);
@@ -876,6 +997,7 @@ mod tests {
};
let state_store = Arc::new(ObjectStateStore::new(object_store.clone()));
let manager = LocalManager::new(config, state_store);
manager.manager_ctx.set_running();
let mut procedure = ProcedureToLoad::new("submit");
procedure.lock_key = LockKey::single("test.submit");
@@ -889,8 +1011,9 @@ mod tests {
.is_ok());
let mut watcher = manager.procedure_watcher(procedure_id).unwrap();
watcher.changed().await.unwrap();
manager.start().unwrap();
tokio::time::sleep(Duration::from_millis(10)).await;
manager.start().await.unwrap();
tokio::time::sleep(Duration::from_millis(300)).await;
assert!(manager
.procedure_state(procedure_id)
.await
@@ -902,6 +1025,8 @@ mod tests {
let mut procedure = ProcedureToLoad::new("submit");
procedure.lock_key = LockKey::single("test.submit");
let procedure_id = ProcedureId::random();
manager.manager_ctx.set_running();
assert!(manager
.submit(ProcedureWithId {
id: procedure_id,
@@ -911,11 +1036,33 @@ mod tests {
.is_ok());
let mut watcher = manager.procedure_watcher(procedure_id).unwrap();
watcher.changed().await.unwrap();
tokio::time::sleep(Duration::from_millis(10)).await;
tokio::time::sleep(Duration::from_millis(300)).await;
assert!(manager
.procedure_state(procedure_id)
.await
.unwrap()
.is_some());
// After restart
let mut procedure = ProcedureToLoad::new("submit");
procedure.lock_key = LockKey::single("test.submit");
let procedure_id = ProcedureId::random();
assert!(manager
.submit(ProcedureWithId {
id: procedure_id,
procedure: Box::new(procedure),
})
.await
.is_ok());
let mut watcher = manager.procedure_watcher(procedure_id).unwrap();
watcher.changed().await.unwrap();
manager.start().await.unwrap();
tokio::time::sleep(Duration::from_millis(300)).await;
assert!(manager
.procedure_state(procedure_id)
.await
.unwrap()
.is_none());
}
}

View File

@@ -19,7 +19,7 @@ use backon::{BackoffBuilder, ExponentialBuilder};
use common_telemetry::logging;
use tokio::time;
use crate::error::{ProcedurePanicSnafu, Result};
use crate::error::{self, ProcedurePanicSnafu, Result};
use crate::local::{ManagerContext, ProcedureMeta, ProcedureMetaRef};
use crate::store::ProcedureStore;
use crate::ProcedureState::Retrying;
@@ -102,7 +102,6 @@ impl Drop for ProcedureGuard {
}
}
// TODO(yingwen): Support cancellation.
pub(crate) struct Runner {
pub(crate) meta: ProcedureMetaRef,
pub(crate) procedure: BoxedProcedure,
@@ -114,6 +113,11 @@ pub(crate) struct Runner {
}
impl Runner {
/// Return `ProcedureManager` is running.
pub(crate) fn running(&self) -> bool {
self.manager_ctx.running()
}
/// Run the procedure.
pub(crate) async fn run(mut self) {
// Ensure we can update the procedure state.
@@ -152,6 +156,12 @@ impl Runner {
let procedure_ids = self.manager_ctx.procedures_in_tree(&self.meta);
// Clean resources.
self.manager_ctx.on_procedures_finish(&procedure_ids);
// If `ProcedureManager` is stopped, it stops the current task immediately without deleting the procedure.
if !self.running() {
return;
}
for id in procedure_ids {
if let Err(e) = self.store.delete_procedure(id).await {
logging::error!(
@@ -186,6 +196,13 @@ impl Runner {
let mut retry = self.exponential_builder.build();
let mut retry_times = 0;
loop {
// Don't store state if `ProcedureManager` is stopped.
if !self.running() {
self.meta.set_state(ProcedureState::Failed {
error: Arc::new(error::ManagerNotStartSnafu {}.build()),
});
return;
}
match self.execute_once(ctx).await {
ExecResult::Done | ExecResult::Failed => return,
ExecResult::Continue => (),
@@ -238,6 +255,14 @@ impl Runner {
status.need_persist(),
);
// Don't store state if `ProcedureManager` is stopped.
if !self.running() {
self.meta.set_state(ProcedureState::Failed {
error: Arc::new(error::ManagerNotStartSnafu {}.build()),
});
return ExecResult::Failed;
}
if status.need_persist() {
if let Err(err) = self.persist_procedure().await {
self.meta.set_state(ProcedureState::retrying(Arc::new(err)));
@@ -272,6 +297,14 @@ impl Runner {
e.is_retry_later(),
);
// Don't store state if `ProcedureManager` is stopped.
if !self.running() {
self.meta.set_state(ProcedureState::Failed {
error: Arc::new(error::ManagerNotStartSnafu {}.build()),
});
return ExecResult::Failed;
}
if e.is_retry_later() {
self.meta.set_state(ProcedureState::retrying(Arc::new(e)));
return ExecResult::RetryLater;
@@ -581,6 +614,7 @@ mod tests {
let object_store = test_util::new_object_store(&dir);
let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
let mut runner = new_runner(meta, Box::new(normal), procedure_store.clone());
runner.manager_ctx.start();
let res = runner.execute_once(&ctx).await;
assert!(res.is_continue(), "{res:?}");
@@ -641,6 +675,7 @@ mod tests {
let object_store = test_util::new_object_store(&dir);
let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
let mut runner = new_runner(meta, Box::new(suspend), procedure_store);
runner.manager_ctx.start();
let res = runner.execute_once(&ctx).await;
assert!(res.is_continue(), "{res:?}");
@@ -742,6 +777,7 @@ mod tests {
let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
let mut runner = new_runner(meta.clone(), Box::new(parent), procedure_store.clone());
let manager_ctx = Arc::new(ManagerContext::new());
manager_ctx.start();
// Manually add this procedure to the manager ctx.
assert!(manager_ctx.try_insert_procedure(meta));
// Replace the manager ctx.
@@ -769,6 +805,70 @@ mod tests {
}
}
#[tokio::test]
async fn test_running_is_stopped() {
let exec_fn = move |_| async move { Ok(Status::Executing { persist: true }) }.boxed();
let normal = ProcedureAdapter {
data: "normal".to_string(),
lock_key: LockKey::single("catalog.schema.table"),
exec_fn,
};
let dir = create_temp_dir("test_running_is_stopped");
let meta = normal.new_meta(ROOT_ID);
let ctx = context_without_provider(meta.id);
let object_store = test_util::new_object_store(&dir);
let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
let mut runner = new_runner(meta, Box::new(normal), procedure_store.clone());
runner.manager_ctx.start();
let res = runner.execute_once(&ctx).await;
assert!(res.is_continue(), "{res:?}");
check_files(
&object_store,
&procedure_store,
ctx.procedure_id,
&["0000000000.step"],
)
.await;
runner.manager_ctx.stop();
let res = runner.execute_once(&ctx).await;
assert!(res.is_failed());
// Shouldn't write any files
check_files(
&object_store,
&procedure_store,
ctx.procedure_id,
&["0000000000.step"],
)
.await;
}
#[tokio::test]
async fn test_running_is_stopped_on_error() {
let exec_fn =
|_| async { Err(Error::external(MockError::new(StatusCode::Unexpected))) }.boxed();
let normal = ProcedureAdapter {
data: "fail".to_string(),
lock_key: LockKey::single("catalog.schema.table"),
exec_fn,
};
let dir = create_temp_dir("test_running_is_stopped_on_error");
let meta = normal.new_meta(ROOT_ID);
let ctx = context_without_provider(meta.id);
let object_store = test_util::new_object_store(&dir);
let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
let mut runner = new_runner(meta, Box::new(normal), procedure_store.clone());
runner.manager_ctx.stop();
let res = runner.execute_once(&ctx).await;
assert!(res.is_failed(), "{res:?}");
// Shouldn't write any files
check_files(&object_store, &procedure_store, ctx.procedure_id, &[]).await;
}
#[tokio::test]
async fn test_execute_on_error() {
let exec_fn =
@@ -785,6 +885,7 @@ mod tests {
let object_store = test_util::new_object_store(&dir);
let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
let mut runner = new_runner(meta.clone(), Box::new(fail), procedure_store.clone());
runner.manager_ctx.start();
let res = runner.execute_once(&ctx).await;
assert!(res.is_failed(), "{res:?}");
@@ -826,6 +927,7 @@ mod tests {
let object_store = test_util::new_object_store(&dir);
let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
let mut runner = new_runner(meta.clone(), Box::new(retry_later), procedure_store.clone());
runner.manager_ctx.start();
let res = runner.execute_once(&ctx).await;
assert!(res.is_retry_later(), "{res:?}");
@@ -863,6 +965,8 @@ mod tests {
Box::new(exceed_max_retry_later),
procedure_store,
);
runner.manager_ctx.start();
runner.exponential_builder = ExponentialBuilder::default()
.with_min_delay(Duration::from_millis(1))
.with_max_times(3);
@@ -933,8 +1037,8 @@ mod tests {
let object_store = test_util::new_object_store(&dir);
let procedure_store = Arc::new(ProcedureStore::from_object_store(object_store.clone()));
let mut runner = new_runner(meta.clone(), Box::new(parent), procedure_store);
let manager_ctx = Arc::new(ManagerContext::new());
manager_ctx.start();
// Manually add this procedure to the manager ctx.
assert!(manager_ctx.try_insert_procedure(meta.clone()));
// Replace the manager ctx.

View File

@@ -279,8 +279,14 @@ pub trait ProcedureManager: Send + Sync + 'static {
/// Registers loader for specific procedure type `name`.
fn register_loader(&self, name: &str, loader: BoxedProcedureLoader) -> Result<()>;
fn start(&self) -> Result<()>;
/// Starts the background GC task.
///
/// Recovers unfinished procedures and reruns them.
///
/// Callers should ensure all loaders are registered.
async fn start(&self) -> Result<()>;
/// Stops the background GC task.
async fn stop(&self) -> Result<()>;
/// Submits a procedure to execute.
@@ -288,11 +294,6 @@ pub trait ProcedureManager: Send + Sync + 'static {
/// Returns a [Watcher] to watch the created procedure.
async fn submit(&self, procedure: ProcedureWithId) -> Result<Watcher>;
/// Recovers unfinished procedures and reruns them.
///
/// Callers should ensure all loaders are registered.
async fn recover(&self) -> Result<()>;
/// Query the procedure state.
///
/// Returns `Ok(None)` if the procedure doesn't exist.

View File

@@ -71,6 +71,7 @@ mod tests {
};
let state_store = Arc::new(ObjectStateStore::new(test_util::new_object_store(&dir)));
let manager = LocalManager::new(config, state_store);
manager.start().await.unwrap();
#[derive(Debug)]
struct MockProcedure {

View File

@@ -13,8 +13,10 @@
// limitations under the License.
use std::collections::HashMap;
use std::slice;
use std::sync::Arc;
use datafusion::arrow::util::pretty::pretty_format_batches;
use datatypes::schema::SchemaRef;
use datatypes::value::Value;
use datatypes::vectors::{Helper, VectorRef};
@@ -169,6 +171,13 @@ impl RecordBatch {
Ok(vectors)
}
/// Pretty display this record batch like a table
pub fn pretty_print(&self) -> String {
pretty_format_batches(slice::from_ref(&self.df_record_batch))
.map(|t| t.to_string())
.unwrap_or("failed to pretty display a record batch".to_string())
}
}
impl Serialize for RecordBatch {

View File

@@ -367,7 +367,10 @@ impl DatanodeBuilder {
/// Build [RaftEngineLogStore]
async fn build_log_store(opts: &DatanodeOptions) -> Result<Arc<RaftEngineLogStore>> {
let data_home = normalize_dir(&opts.storage.data_home);
let wal_dir = format!("{}{WAL_DIR}", data_home);
let wal_dir = match &opts.wal.dir {
Some(dir) => dir.clone(),
None => format!("{}{WAL_DIR}", data_home),
};
let wal_config = opts.wal.clone();
// create WAL directory

View File

@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::atomic::AtomicBool;
use std::sync::Arc;
use async_trait::async_trait;
@@ -60,6 +61,8 @@ pub async fn get_greptimedb_telemetry_task(
if !enable || cfg!(test) || cfg!(debug_assertions) {
return Arc::new(GreptimeDBTelemetryTask::disable());
}
// Always enable.
let should_report = Arc::new(AtomicBool::new(true));
match mode {
Mode::Standalone => Arc::new(GreptimeDBTelemetryTask::enable(
@@ -70,7 +73,9 @@ pub async fn get_greptimedb_telemetry_task(
uuid: default_get_uuid(&working_home),
retry: 0,
}),
should_report.clone(),
)),
should_report,
)),
Mode::Distributed => Arc::new(GreptimeDBTelemetryTask::disable()),
}

View File

@@ -111,8 +111,7 @@ impl MetaSrvInstance {
.await
.context(error::SendShutdownSignalSnafu)?;
}
self.meta_srv.shutdown();
self.meta_srv.shutdown().await?;
self.http_srv
.shutdown()
.await

View File

@@ -41,6 +41,12 @@ pub enum Error {
source: common_meta::error::Error,
},
#[snafu(display("Failed to start telemetry task"))]
StartTelemetryTask {
location: Location,
source: common_runtime::error::Error,
},
#[snafu(display("Failed to submit ddl task"))]
SubmitDdlTask {
location: Location,
@@ -393,8 +399,14 @@ pub enum Error {
#[snafu(display("Missing required parameter, param: {:?}", param))]
MissingRequiredParameter { param: String },
#[snafu(display("Failed to recover procedure"))]
RecoverProcedure {
#[snafu(display("Failed to start procedure manager"))]
StartProcedureManager {
location: Location,
source: common_procedure::Error,
},
#[snafu(display("Failed to stop procedure manager"))]
StopProcedureManager {
location: Location,
source: common_procedure::Error,
},
@@ -616,16 +628,19 @@ impl ErrorExt for Error {
Error::RequestDatanode { source, .. } => source.status_code(),
Error::InvalidCatalogValue { source, .. }
| Error::InvalidFullTableName { source, .. } => source.status_code(),
Error::RecoverProcedure { source, .. }
| Error::SubmitProcedure { source, .. }
| Error::WaitProcedure { source, .. } => source.status_code(),
Error::SubmitProcedure { source, .. } | Error::WaitProcedure { source, .. } => {
source.status_code()
}
Error::ShutdownServer { source, .. } | Error::StartHttp { source, .. } => {
source.status_code()
}
Error::StartProcedureManager { source, .. }
| Error::StopProcedureManager { source, .. } => source.status_code(),
Error::ListCatalogs { source, .. } | Error::ListSchemas { source, .. } => {
source.status_code()
}
Error::StartTelemetryTask { source, .. } => source.status_code(),
Error::RegionFailoverCandidatesNotFound { .. } => StatusCode::RuntimeResourcesExhausted,
Error::NextSequence { source, .. } => source.status_code(),

View File

@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::sync::atomic::AtomicBool;
use std::sync::Arc;
use async_trait::async_trait;
@@ -63,7 +64,8 @@ pub async fn get_greptimedb_telemetry_task(
if !enable || cfg!(test) || cfg!(debug_assertions) {
return Arc::new(GreptimeDBTelemetryTask::disable());
}
// Controlled by meta server state, only leader reports the info.
let should_report = Arc::new(AtomicBool::new(false));
Arc::new(GreptimeDBTelemetryTask::enable(
TELEMETRY_INTERVAL,
Box::new(GreptimeDBTelemetry::new(
@@ -73,6 +75,8 @@ pub async fn get_greptimedb_telemetry_task(
uuid: default_get_uuid(&working_home),
retry: 0,
}),
should_report.clone(),
)),
should_report,
))
}

View File

@@ -28,7 +28,7 @@ use common_meta::sequence::SequenceRef;
use common_procedure::options::ProcedureConfig;
use common_procedure::ProcedureManagerRef;
use common_telemetry::logging::LoggingOptions;
use common_telemetry::{debug, error, info, warn};
use common_telemetry::{error, info, warn};
use serde::{Deserialize, Serialize};
use servers::http::HttpOptions;
use snafu::ResultExt;
@@ -37,7 +37,10 @@ use tokio::sync::broadcast::error::RecvError;
use crate::cluster::MetaPeerClientRef;
use crate::election::{Election, LeaderChangeMessage};
use crate::error::{InitMetadataSnafu, RecoverProcedureSnafu, Result};
use crate::error::{
InitMetadataSnafu, Result, StartProcedureManagerSnafu, StartTelemetryTaskSnafu,
StopProcedureManagerSnafu,
};
use crate::handler::HeartbeatHandlerGroup;
use crate::lock::DistLockRef;
use crate::pubsub::{PublishRef, SubscribeManagerRef};
@@ -169,6 +172,37 @@ pub struct SelectorContext {
pub type SelectorRef = Arc<dyn Selector<Context = SelectorContext, Output = Vec<Peer>>>;
pub type ElectionRef = Arc<dyn Election<Leader = LeaderValue>>;
pub struct MetaStateHandler {
procedure_manager: ProcedureManagerRef,
subscribe_manager: Option<SubscribeManagerRef>,
greptimedb_telemetry_task: Arc<GreptimeDBTelemetryTask>,
}
impl MetaStateHandler {
pub async fn on_become_leader(&self) {
if let Err(e) = self.procedure_manager.start().await {
error!(e; "Failed to start procedure manager");
}
self.greptimedb_telemetry_task.should_report(true);
}
pub async fn on_become_follower(&self) {
// Stops the procedures.
if let Err(e) = self.procedure_manager.stop().await {
error!(e; "Failed to stop procedure manager");
}
// Suspends reporting.
self.greptimedb_telemetry_task.should_report(false);
if let Some(sub_manager) = self.subscribe_manager.clone() {
info!("Leader changed, un_subscribe all");
if let Err(e) = sub_manager.un_subscribe_all() {
error!("Failed to un_subscribe all, error: {}", e);
}
}
}
}
#[derive(Clone)]
pub struct MetaSrv {
started: Arc<AtomicBool>,
@@ -212,7 +246,15 @@ impl MetaSrv {
let leader_cached_kv_store = self.leader_cached_kv_store.clone();
let subscribe_manager = self.subscribe_manager();
let mut rx = election.subscribe_leader_change();
let task_handler = self.greptimedb_telemetry_task.clone();
let greptimedb_telemetry_task = self.greptimedb_telemetry_task.clone();
greptimedb_telemetry_task
.start()
.context(StartTelemetryTaskSnafu)?;
let state_handler = MetaStateHandler {
greptimedb_telemetry_task,
subscribe_manager,
procedure_manager,
};
let _handle = common_runtime::spawn_bg(async move {
loop {
match rx.recv().await {
@@ -225,28 +267,12 @@ impl MetaSrv {
);
match msg {
LeaderChangeMessage::Elected(_) => {
if let Err(e) = procedure_manager.recover().await {
error!("Failed to recover procedures, error: {e}");
}
let _ = task_handler.start().map_err(|e| {
debug!(
"Failed to start greptimedb telemetry task, error: {e}"
);
});
state_handler.on_become_leader().await;
}
LeaderChangeMessage::StepDown(leader) => {
if let Some(sub_manager) = subscribe_manager.clone() {
info!("Leader changed, un_subscribe all");
if let Err(e) = sub_manager.un_subscribe_all() {
error!("Failed to un_subscribe all, error: {}", e);
}
}
error!("Leader :{:?} step down", leader);
let _ = task_handler.stop().await.map_err(|e| {
debug!(
"Failed to stop greptimedb telemetry task, error: {e}"
);
});
state_handler.on_become_follower().await;
}
}
}
@@ -259,6 +285,8 @@ impl MetaSrv {
}
}
}
state_handler.on_become_follower().await;
});
let election = election.clone();
@@ -275,9 +303,9 @@ impl MetaSrv {
});
} else {
self.procedure_manager
.recover()
.start()
.await
.context(RecoverProcedureSnafu)?;
.context(StartProcedureManagerSnafu)?;
}
info!("MetaSrv started");
@@ -291,8 +319,12 @@ impl MetaSrv {
.context(InitMetadataSnafu)
}
pub fn shutdown(&self) {
pub async fn shutdown(&self) -> Result<()> {
self.started.store(false, Ordering::Relaxed);
self.procedure_manager
.stop()
.await
.context(StopProcedureManagerSnafu)
}
#[inline]

View File

@@ -274,6 +274,9 @@ pub enum Error {
#[snafu(display("Missing table mutation handler"))]
MissingTableMutationHandler { location: Location },
#[snafu(display("Range Query: {}", msg))]
RangeQuery { msg: String, location: Location },
}
impl ErrorExt for Error {
@@ -281,7 +284,9 @@ impl ErrorExt for Error {
use Error::*;
match self {
QueryParse { .. } | MultipleStatements { .. } => StatusCode::InvalidSyntax,
QueryParse { .. } | MultipleStatements { .. } | RangeQuery { .. } => {
StatusCode::InvalidSyntax
}
UnsupportedExpr { .. }
| Unimplemented { .. }
| CatalogNotFound { .. }

View File

@@ -79,7 +79,7 @@ impl DfLogicalPlanner {
let result = sql_to_rel
.statement_to_plan(df_stmt)
.context(PlanSqlSnafu)?;
let plan = RangePlanRewriter::new(table_provider, context_provider)
let plan = RangePlanRewriter::new(table_provider)
.rewrite(result)
.await?;
Ok(LogicalPlan::DfPlan(plan))

View File

@@ -21,7 +21,7 @@ use std::task::{Context, Poll};
use std::time::Duration;
use ahash::RandomState;
use arrow::compute;
use arrow::compute::{self, cast_with_options, CastOptions};
use arrow_schema::{DataType, Field, Schema, SchemaRef, TimeUnit};
use common_query::DfPhysicalPlan;
use common_recordbatch::DfSendableRecordBatchStream;
@@ -33,6 +33,7 @@ use datafusion::physical_plan::udaf::create_aggregate_expr as create_aggr_udf_ex
use datafusion::physical_plan::{
DisplayAs, DisplayFormatType, ExecutionPlan, RecordBatchStream, SendableRecordBatchStream,
};
use datafusion::physical_planner::create_physical_sort_expr;
use datafusion_common::utils::get_arrayref_at_indices;
use datafusion_common::{DFField, DFSchema, DFSchemaRef, DataFusionError, ScalarValue};
use datafusion_expr::utils::exprlist_to_fields;
@@ -54,22 +55,135 @@ use crate::error::{DataFusionSnafu, Result};
type Millisecond = <TimestampMillisecondType as ArrowPrimitiveType>::Native;
#[derive(PartialEq, Eq, Hash, Clone, Debug)]
#[derive(PartialEq, Eq, Debug, Hash, Clone)]
pub enum Fill {
Null,
Prev,
Linear,
Const(ScalarValue),
}
impl Display for Fill {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Fill::Null => write!(f, "NULL"),
Fill::Prev => write!(f, "PREV"),
Fill::Linear => write!(f, "LINEAR"),
Fill::Const(x) => write!(f, "{}", x),
}
}
}
impl Fill {
pub fn try_from_str(value: &str, datatype: &DataType) -> DfResult<Self> {
let s = value.to_uppercase();
match s.as_str() {
"NULL" | "" => Ok(Self::Null),
"PREV" => Ok(Self::Prev),
"LINEAR" => {
if datatype.is_numeric() {
Ok(Self::Linear)
} else {
Err(DataFusionError::Plan(format!(
"Use FILL LINEAR on Non-numeric DataType {}",
datatype
)))
}
}
_ => ScalarValue::try_from_string(s.clone(), datatype)
.map_err(|err| {
DataFusionError::Plan(format!(
"{} is not a valid fill option, fail to convert to a const value. {{ {} }}",
s, err
))
})
.map(Fill::Const),
}
}
/// The input `data` contains data on a complete time series.
/// If the filling strategy is `PREV` or `LINEAR`, caller must be ensured that the incoming `data` is ascending time order.
pub fn apply_fill_strategy(&self, data: &mut [ScalarValue]) -> DfResult<()> {
let len = data.len();
for i in 0..len {
if data[i].is_null() {
match self {
Fill::Null => continue,
Fill::Prev => {
if i != 0 {
data[i] = data[i - 1].clone()
}
}
Fill::Linear => {
if 0 < i && i < len - 1 {
match (&data[i - 1], &data[i + 1]) {
(ScalarValue::Float64(Some(a)), ScalarValue::Float64(Some(b))) => {
data[i] = ScalarValue::Float64(Some((a + b) / 2.0));
}
(ScalarValue::Float32(Some(a)), ScalarValue::Float32(Some(b))) => {
data[i] = ScalarValue::Float32(Some((a + b) / 2.0));
}
(a, b) => {
if !a.is_null() && !b.is_null() {
return Err(DataFusionError::Execution(
"RangePlan: Apply Fill LINEAR strategy on Non-floating type".to_string()));
} else {
continue;
}
}
}
}
}
Fill::Const(v) => data[i] = v.clone(),
}
}
}
Ok(())
}
}
#[derive(Eq, Clone, Debug)]
pub struct RangeFn {
/// with format like `max(a) 300s null`
pub name: String,
pub data_type: DataType,
pub expr: Expr,
pub range: Duration,
pub fill: String,
pub fill: Fill,
/// If the `FIll` strategy is `Linear` and the output is an integer,
/// it is possible to calculate a floating point number.
/// So for `FILL==LINEAR`, the entire data will be implicitly converted to Float type
/// If `need_cast==true`, `data_type` may not consist with type `expr` generated.
pub need_cast: bool,
}
impl PartialEq for RangeFn {
fn eq(&self, other: &Self) -> bool {
self.name == other.name
}
}
impl PartialOrd for RangeFn {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
Some(self.cmp(other))
}
}
impl Ord for RangeFn {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.name.cmp(&other.name)
}
}
impl std::hash::Hash for RangeFn {
fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
self.name.hash(state);
}
}
impl Display for RangeFn {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"RangeFn {{ expr:{} range:{}s fill:{} }}",
self.expr.display_name().unwrap_or("?".into()),
self.range.as_secs(),
self.fill,
)
write!(f, "{}", self.name)
}
}
@@ -105,16 +219,21 @@ impl RangeSelect {
) -> Result<Self> {
let mut fields = range_expr
.iter()
.map(|RangeFn { expr, .. }| {
Ok(DFField::new_unqualified(
&expr.display_name()?,
expr.get_type(input.schema())?,
// TODO(Taylor-lagrange): We have not implemented fill currently,
// it is possible that some columns may not be able to aggregate data,
// so we temporarily set that all data is nullable
true,
))
})
.map(
|RangeFn {
name,
data_type,
fill,
..
}| {
Ok(DFField::new_unqualified(
name,
data_type.clone(),
// Only when data fill with Const option, the data can't be null
!matches!(fill, Fill::Const(..)),
))
},
)
.collect::<DfResult<Vec<_>>>()
.context(DataFusionSnafu)?;
// add align_ts
@@ -135,10 +254,8 @@ impl RangeSelect {
DFSchema::new_with_metadata(by_fields, input.schema().metadata().clone())
.context(DataFusionSnafu)?,
);
// If the result of the project plan happens to be the schema of the range plan, no project plan is required
// that need project is identical to range plan schema.
// 1. all exprs in project must belong to range schema
// 2. range schema and project exprs must have same size
// If the results of project plan can be obtained directly from range plan without any additional calculations, no project plan is required.
// We can simply project the final output of the range plan to produce the final result.
let schema_project = projection_expr
.iter()
.map(|project_expr| {
@@ -268,52 +385,68 @@ impl RangeSelect {
.range_expr
.iter()
.map(|range_fn| {
let (expr, args) = match &range_fn.expr {
let expr = match &range_fn.expr {
Expr::AggregateFunction(aggr) => {
let args = self.create_physical_expr_list(
&aggr.args,
input_dfschema,
&input_schema,
session_state,
)?;
Ok((
create_aggr_expr(
&aggr.fun,
false,
&args,
&[],
let order_by = if let Some(exprs) = &aggr.order_by {
exprs
.iter()
.map(|x| {
create_physical_sort_expr(
x,
input_dfschema,
&input_schema,
session_state.execution_props(),
)
})
.collect::<DfResult<Vec<_>>>()?
} else {
vec![]
};
let expr = create_aggr_expr(
&aggr.fun,
false,
&self.create_physical_expr_list(
&aggr.args,
input_dfschema,
&input_schema,
range_fn.expr.display_name()?,
session_state,
)?,
args,
))
&order_by,
&input_schema,
range_fn.expr.display_name()?,
)?;
Ok(expr)
}
Expr::AggregateUDF(aggr_udf) => {
let args = self.create_physical_expr_list(
&aggr_udf.args,
input_dfschema,
&input_schema,
session_state,
)?;
Ok((
create_aggr_udf_expr(
&aggr_udf.fun,
&args,
let expr = create_aggr_udf_expr(
&aggr_udf.fun,
&self.create_physical_expr_list(
&aggr_udf.args,
input_dfschema,
&input_schema,
range_fn.expr.display_name()?,
session_state,
)?,
args,
))
&input_schema,
range_fn.expr.display_name()?,
)?;
Ok(expr)
}
_ => Err(DataFusionError::Plan(format!(
"Unexpected Expr:{} in RangeSelect",
range_fn.expr.display_name()?
))),
}?;
let args = expr.expressions();
Ok(RangeFnExec {
expr,
args,
range: range_fn.range.as_millis() as Millisecond,
fill: range_fn.fill.clone(),
need_cast: if range_fn.need_cast {
Some(range_fn.data_type.clone())
} else {
None
},
})
})
.collect::<DfResult<Vec<_>>>()?;
@@ -348,6 +481,8 @@ struct RangeFnExec {
pub expr: Arc<dyn AggregateExpr>,
pub args: Vec<Arc<dyn PhysicalExpr>>,
pub range: Millisecond,
pub fill: Fill,
pub need_cast: Option<DataType>,
}
#[derive(Debug)]
@@ -540,6 +675,15 @@ fn align_to_calendar(
}
}
fn cast_scalar_values(values: &mut [ScalarValue], data_type: &DataType) -> DfResult<()> {
let array = ScalarValue::iter_to_array(values.to_vec())?;
let cast_array = cast_with_options(&array, data_type, &CastOptions::default())?;
for (i, value) in values.iter_mut().enumerate() {
*value = ScalarValue::try_from_array(&cast_array, i)?;
}
Ok(())
}
impl RangeSelectStream {
fn evaluate_many(
&self,
@@ -648,20 +792,57 @@ impl RangeSelectStream {
let mut columns: Vec<Arc<dyn Array>> =
Vec::with_capacity(1 + self.range_exec.len() + self.by.len());
let mut ts_builder = TimestampMillisecondBuilder::with_capacity(self.output_num_rows);
let mut all_scalar = vec![vec![]; self.range_exec.len()];
let mut all_scalar = vec![Vec::with_capacity(self.output_num_rows); self.range_exec.len()];
let mut by_rows = Vec::with_capacity(self.output_num_rows);
let mut start_index = 0;
// RangePlan is calculated on a row basis. If a column uses the PREV or LINEAR filling strategy,
// we must arrange the data in the entire data row to determine the NULL filling value.
let need_sort_output = self
.range_exec
.iter()
.any(|range| range.fill == Fill::Linear || range.fill == Fill::Prev);
for SeriesState {
row,
align_ts_accumulator,
} in self.series_map.values()
{
for (ts, accumulators) in align_ts_accumulator {
for (i, accumulator) in accumulators.iter().enumerate() {
all_scalar[i].push(accumulator.evaluate()?);
// collect data on time series
if !need_sort_output {
for (ts, accumulators) in align_ts_accumulator {
for (i, accumulator) in accumulators.iter().enumerate() {
all_scalar[i].push(accumulator.evaluate()?);
}
ts_builder.append_value(*ts);
}
by_rows.push(row.row());
ts_builder.append_value(*ts);
} else {
let mut keys = align_ts_accumulator.keys().copied().collect::<Vec<_>>();
keys.sort();
for key in &keys {
for (i, accumulator) in
align_ts_accumulator.get(key).unwrap().iter().enumerate()
{
all_scalar[i].push(accumulator.evaluate()?);
}
}
ts_builder.append_slice(&keys);
}
// apply fill strategy on time series
for (
i,
RangeFnExec {
fill, need_cast, ..
},
) in self.range_exec.iter().enumerate()
{
let time_series_data =
&mut all_scalar[i][start_index..start_index + align_ts_accumulator.len()];
if let Some(data_type) = need_cast {
cast_scalar_values(time_series_data, data_type)?;
}
fill.apply_fill_strategy(time_series_data)?;
}
by_rows.resize(by_rows.len() + align_ts_accumulator.len(), row.row());
start_index += align_ts_accumulator.len();
}
for column_scalar in all_scalar {
columns.push(ScalarValue::iter_to_array(column_scalar)?);
@@ -720,15 +901,15 @@ impl Stream for RangeSelectStream {
}
ExecutionState::ProducingOutput => {
let result = self.generate_output();
match result {
return match result {
// made output
Ok(batch) => {
self.exec_state = ExecutionState::Done;
return Poll::Ready(Some(Ok(batch)));
Poll::Ready(Some(Ok(batch)))
}
// error making output
Err(error) => return Poll::Ready(Some(Err(error))),
}
Err(error) => Poll::Ready(Some(Err(error))),
};
}
ExecutionState::Done => return Poll::Ready(None),
}
@@ -738,6 +919,34 @@ impl Stream for RangeSelectStream {
#[cfg(test)]
mod test {
macro_rules! nullable_array {
($builder:ident,) => {
};
($array_type:ident ; $($tail:tt)*) => {
paste::item! {
{
let mut builder = arrow::array::[<$array_type Builder>]::new();
nullable_array!(builder, $($tail)*);
builder.finish()
}
}
};
($builder:ident, null) => {
$builder.append_null();
};
($builder:ident, null, $($tail:tt)*) => {
$builder.append_null();
nullable_array!($builder, $($tail)*);
};
($builder:ident, $value:literal) => {
$builder.append_value($value);
};
($builder:ident, $value:literal, $($tail:tt)*) => {
$builder.append_value($value);
nullable_array!($builder, $($tail)*);
};
}
use arrow_schema::SortOptions;
use datafusion::arrow::datatypes::{
ArrowPrimitiveType, DataType, Field, Schema, TimestampMillisecondType,
@@ -747,33 +956,45 @@ mod test {
use datafusion::prelude::SessionContext;
use datafusion_physical_expr::expressions::{self, Column};
use datafusion_physical_expr::PhysicalSortExpr;
use datatypes::arrow::array::{Int64Array, TimestampMillisecondArray};
use datatypes::arrow::array::TimestampMillisecondArray;
use datatypes::arrow_array::StringArray;
use super::*;
const TIME_INDEX_COLUMN: &str = "timestamp";
fn prepare_test_data() -> MemoryExec {
fn prepare_test_data(is_float: bool) -> MemoryExec {
let schema = Arc::new(Schema::new(vec![
Field::new(TIME_INDEX_COLUMN, TimestampMillisecondType::DATA_TYPE, true),
Field::new("value", DataType::Int64, true),
Field::new(
"value",
if is_float {
DataType::Float64
} else {
DataType::Int64
},
true,
),
Field::new("host", DataType::Utf8, true),
]));
let timestamp_column = Arc::new(TimestampMillisecondArray::from(vec![
// host 1 every 5s
0, 5_000, 10_000, 15_000, 20_000, 25_000, 30_000, 35_000, 40_000,
// host 2 every 5s
0, 5_000, 10_000, 15_000, 20_000, 25_000, 30_000, 35_000, 40_000,
let timestamp_column: Arc<dyn Array> = Arc::new(TimestampMillisecondArray::from(vec![
0, 5_000, 10_000, 15_000, 20_000, // host 1 every 5s
0, 5_000, 10_000, 15_000, 20_000, // host 2 every 5s
])) as _;
let values = vec![
0, 1, 2, 3, 4, 5, 6, 7, 8, // data for host 1
9, 10, 11, 12, 13, 14, 15, 16, 17, // data for host 2
];
let mut host = vec!["host1"; 9];
host.extend(vec!["host2"; 9]);
let value_column = Arc::new(Int64Array::from(values)) as _;
let host_column = Arc::new(StringArray::from(host)) as _;
let mut host = vec!["host1"; 5];
host.extend(vec!["host2"; 5]);
let value_column: Arc<dyn Array> = if is_float {
Arc::new(nullable_array!(Float64;
0.0, null, 1.0, null, 2.0, // data for host 1
3.0, null, 4.0, null, 5.0 // data for host 2
)) as _
} else {
Arc::new(nullable_array!(Int64;
0, null, 1, null, 2, // data for host 1
3, null, 4, null, 5 // data for host 2
)) as _
};
let host_column: Arc<dyn Array> = Arc::new(StringArray::from(host)) as _;
let data = RecordBatch::try_new(
schema.clone(),
vec![timestamp_column, value_column, host_column],
@@ -787,12 +1008,25 @@ mod test {
range1: Millisecond,
range2: Millisecond,
align: Millisecond,
fill: Fill,
is_float: bool,
expected: String,
) {
let memory_exec = Arc::new(prepare_test_data());
let data_type = if is_float {
DataType::Float64
} else {
DataType::Int64
};
let (need_cast, schema_data_type) = if !is_float && fill == Fill::Linear {
// data_type = DataType::Float64;
(Some(DataType::Float64), DataType::Float64)
} else {
(None, data_type.clone())
};
let memory_exec = Arc::new(prepare_test_data(is_float));
let schema = Arc::new(Schema::new(vec![
Field::new("MIN(value)", DataType::Int64, true),
Field::new("MAX(value)", DataType::Int64, true),
Field::new("MIN(value)", schema_data_type.clone(), true),
Field::new("MAX(value)", schema_data_type, true),
Field::new(TIME_INDEX_COLUMN, TimestampMillisecondType::DATA_TYPE, true),
Field::new("host", DataType::Utf8, true),
]));
@@ -803,19 +1037,23 @@ mod test {
expr: Arc::new(expressions::Min::new(
Arc::new(Column::new("value", 1)),
"MIN(value)",
DataType::Int64,
data_type.clone(),
)),
args: vec![Arc::new(Column::new("value", 1))],
range: range1,
fill: fill.clone(),
need_cast: need_cast.clone(),
},
RangeFnExec {
expr: Arc::new(expressions::Max::new(
Arc::new(Column::new("value", 1)),
"MAX(value)",
DataType::Int64,
data_type,
)),
args: vec![Arc::new(Column::new("value", 1))],
range: range2,
fill,
need_cast,
},
],
align,
@@ -852,85 +1090,225 @@ mod test {
.await
.unwrap();
let result_literal = datatypes::arrow::util::pretty::pretty_format_batches(&result)
let result_literal = arrow::util::pretty::pretty_format_batches(&result)
.unwrap()
.to_string();
assert_eq!(result_literal, expected);
}
#[tokio::test]
async fn range_10s_align_5s() {
let expected = String::from(
"+------------+------------+---------------------+-------+\
\n| MIN(value) | MAX(value) | timestamp | host |\
\n+------------+------------+---------------------+-------+\
\n| 0 | 0 | 1970-01-01T00:00:00 | host1 |\
\n| 0 | 1 | 1970-01-01T00:00:05 | host1 |\
\n| 1 | 2 | 1970-01-01T00:00:10 | host1 |\
\n| 2 | 3 | 1970-01-01T00:00:15 | host1 |\
\n| 3 | 4 | 1970-01-01T00:00:20 | host1 |\
\n| 4 | 5 | 1970-01-01T00:00:25 | host1 |\
\n| 5 | 6 | 1970-01-01T00:00:30 | host1 |\
\n| 6 | 7 | 1970-01-01T00:00:35 | host1 |\
\n| 7 | 8 | 1970-01-01T00:00:40 | host1 |\
\n| 8 | 8 | 1970-01-01T00:00:45 | host1 |\
\n| 9 | 9 | 1970-01-01T00:00:00 | host2 |\
\n| 9 | 10 | 1970-01-01T00:00:05 | host2 |\
\n| 10 | 11 | 1970-01-01T00:00:10 | host2 |\
\n| 11 | 12 | 1970-01-01T00:00:15 | host2 |\
\n| 12 | 13 | 1970-01-01T00:00:20 | host2 |\
\n| 13 | 14 | 1970-01-01T00:00:25 | host2 |\
\n| 14 | 15 | 1970-01-01T00:00:30 | host2 |\
\n| 15 | 16 | 1970-01-01T00:00:35 | host2 |\
\n| 16 | 17 | 1970-01-01T00:00:40 | host2 |\
\n| 17 | 17 | 1970-01-01T00:00:45 | host2 |\
\n+------------+------------+---------------------+-------+",
);
do_range_select_test(10_000, 10_000, 5_000, expected).await;
}
#[tokio::test]
async fn range_10s_align_1000s() {
let expected = String::from(
"+------------+------------+---------------------+-------+\
\n| MIN(value) | MAX(value) | timestamp | host |\
\n+------------+------------+---------------------+-------+\
\n| 0 | 0 | 1970-01-01T00:00:00 | host1 |\
\n| 9 | 9 | 1970-01-01T00:00:00 | host2 |\
\n| 0.0 | 0.0 | 1970-01-01T00:00:00 | host1 |\
\n| 3.0 | 3.0 | 1970-01-01T00:00:00 | host2 |\
\n+------------+------------+---------------------+-------+",
);
do_range_select_test(10_000, 10_000, 1_000_000, expected).await;
do_range_select_test(10_000, 10_000, 1_000_000, Fill::Null, true, expected).await;
}
#[tokio::test]
async fn range_10s_5s_align_5s() {
async fn range_fill_null() {
let expected = String::from(
"+------------+------------+---------------------+-------+\
\n| MIN(value) | MAX(value) | timestamp | host |\
\n+------------+------------+---------------------+-------+\
\n| 0.0 | 0.0 | 1970-01-01T00:00:00 | host1 |\
\n| 0.0 | | 1970-01-01T00:00:05 | host1 |\
\n| 1.0 | 1.0 | 1970-01-01T00:00:10 | host1 |\
\n| 1.0 | | 1970-01-01T00:00:15 | host1 |\
\n| 2.0 | 2.0 | 1970-01-01T00:00:20 | host1 |\
\n| 2.0 | | 1970-01-01T00:00:25 | host1 |\
\n| 3.0 | 3.0 | 1970-01-01T00:00:00 | host2 |\
\n| 3.0 | | 1970-01-01T00:00:05 | host2 |\
\n| 4.0 | 4.0 | 1970-01-01T00:00:10 | host2 |\
\n| 4.0 | | 1970-01-01T00:00:15 | host2 |\
\n| 5.0 | 5.0 | 1970-01-01T00:00:20 | host2 |\
\n| 5.0 | | 1970-01-01T00:00:25 | host2 |\
\n+------------+------------+---------------------+-------+",
);
do_range_select_test(10_000, 5_000, 5_000, Fill::Null, true, expected).await;
}
#[tokio::test]
async fn range_fill_prev() {
let expected = String::from(
"+------------+------------+---------------------+-------+\
\n| MIN(value) | MAX(value) | timestamp | host |\
\n+------------+------------+---------------------+-------+\
\n| 0.0 | 0.0 | 1970-01-01T00:00:00 | host1 |\
\n| 0.0 | 0.0 | 1970-01-01T00:00:05 | host1 |\
\n| 1.0 | 1.0 | 1970-01-01T00:00:10 | host1 |\
\n| 1.0 | 1.0 | 1970-01-01T00:00:15 | host1 |\
\n| 2.0 | 2.0 | 1970-01-01T00:00:20 | host1 |\
\n| 2.0 | 2.0 | 1970-01-01T00:00:25 | host1 |\
\n| 3.0 | 3.0 | 1970-01-01T00:00:00 | host2 |\
\n| 3.0 | 3.0 | 1970-01-01T00:00:05 | host2 |\
\n| 4.0 | 4.0 | 1970-01-01T00:00:10 | host2 |\
\n| 4.0 | 4.0 | 1970-01-01T00:00:15 | host2 |\
\n| 5.0 | 5.0 | 1970-01-01T00:00:20 | host2 |\
\n| 5.0 | 5.0 | 1970-01-01T00:00:25 | host2 |\
\n+------------+------------+---------------------+-------+",
);
do_range_select_test(10_000, 5_000, 5_000, Fill::Prev, true, expected).await;
}
#[tokio::test]
async fn range_fill_linear() {
let expected = String::from(
"+------------+------------+---------------------+-------+\
\n| MIN(value) | MAX(value) | timestamp | host |\
\n+------------+------------+---------------------+-------+\
\n| 0.0 | 0.0 | 1970-01-01T00:00:00 | host1 |\
\n| 0.0 | 0.5 | 1970-01-01T00:00:05 | host1 |\
\n| 1.0 | 1.0 | 1970-01-01T00:00:10 | host1 |\
\n| 1.0 | 1.5 | 1970-01-01T00:00:15 | host1 |\
\n| 2.0 | 2.0 | 1970-01-01T00:00:20 | host1 |\
\n| 2.0 | | 1970-01-01T00:00:25 | host1 |\
\n| 3.0 | 3.0 | 1970-01-01T00:00:00 | host2 |\
\n| 3.0 | 3.5 | 1970-01-01T00:00:05 | host2 |\
\n| 4.0 | 4.0 | 1970-01-01T00:00:10 | host2 |\
\n| 4.0 | 4.5 | 1970-01-01T00:00:15 | host2 |\
\n| 5.0 | 5.0 | 1970-01-01T00:00:20 | host2 |\
\n| 5.0 | | 1970-01-01T00:00:25 | host2 |\
\n+------------+------------+---------------------+-------+",
);
do_range_select_test(10_000, 5_000, 5_000, Fill::Linear, true, expected).await;
}
#[tokio::test]
async fn range_fill_integer_null() {
let expected = String::from(
"+------------+------------+---------------------+-------+\
\n| MIN(value) | MAX(value) | timestamp | host |\
\n+------------+------------+---------------------+-------+\
\n| 0 | 0 | 1970-01-01T00:00:00 | host1 |\
\n| 0 | 1 | 1970-01-01T00:00:05 | host1 |\
\n| 1 | 2 | 1970-01-01T00:00:10 | host1 |\
\n| 2 | 3 | 1970-01-01T00:00:15 | host1 |\
\n| 3 | 4 | 1970-01-01T00:00:20 | host1 |\
\n| 4 | 5 | 1970-01-01T00:00:25 | host1 |\
\n| 5 | 6 | 1970-01-01T00:00:30 | host1 |\
\n| 6 | 7 | 1970-01-01T00:00:35 | host1 |\
\n| 7 | 8 | 1970-01-01T00:00:40 | host1 |\
\n| 8 | | 1970-01-01T00:00:45 | host1 |\
\n| 9 | 9 | 1970-01-01T00:00:00 | host2 |\
\n| 9 | 10 | 1970-01-01T00:00:05 | host2 |\
\n| 10 | 11 | 1970-01-01T00:00:10 | host2 |\
\n| 11 | 12 | 1970-01-01T00:00:15 | host2 |\
\n| 12 | 13 | 1970-01-01T00:00:20 | host2 |\
\n| 13 | 14 | 1970-01-01T00:00:25 | host2 |\
\n| 14 | 15 | 1970-01-01T00:00:30 | host2 |\
\n| 15 | 16 | 1970-01-01T00:00:35 | host2 |\
\n| 16 | 17 | 1970-01-01T00:00:40 | host2 |\
\n| 17 | | 1970-01-01T00:00:45 | host2 |\
\n| 0 | | 1970-01-01T00:00:05 | host1 |\
\n| 1 | 1 | 1970-01-01T00:00:10 | host1 |\
\n| 1 | | 1970-01-01T00:00:15 | host1 |\
\n| 2 | 2 | 1970-01-01T00:00:20 | host1 |\
\n| 2 | | 1970-01-01T00:00:25 | host1 |\
\n| 3 | 3 | 1970-01-01T00:00:00 | host2 |\
\n| 3 | | 1970-01-01T00:00:05 | host2 |\
\n| 4 | 4 | 1970-01-01T00:00:10 | host2 |\
\n| 4 | | 1970-01-01T00:00:15 | host2 |\
\n| 5 | 5 | 1970-01-01T00:00:20 | host2 |\
\n| 5 | | 1970-01-01T00:00:25 | host2 |\
\n+------------+------------+---------------------+-------+",
);
do_range_select_test(10_000, 5_000, 5_000, expected).await;
do_range_select_test(10_000, 5_000, 5_000, Fill::Null, false, expected).await;
}
#[tokio::test]
async fn range_fill_integer_linear() {
let expected = String::from(
"+------------+------------+---------------------+-------+\
\n| MIN(value) | MAX(value) | timestamp | host |\
\n+------------+------------+---------------------+-------+\
\n| 0.0 | 0.0 | 1970-01-01T00:00:00 | host1 |\
\n| 0.0 | 0.5 | 1970-01-01T00:00:05 | host1 |\
\n| 1.0 | 1.0 | 1970-01-01T00:00:10 | host1 |\
\n| 1.0 | 1.5 | 1970-01-01T00:00:15 | host1 |\
\n| 2.0 | 2.0 | 1970-01-01T00:00:20 | host1 |\
\n| 2.0 | | 1970-01-01T00:00:25 | host1 |\
\n| 3.0 | 3.0 | 1970-01-01T00:00:00 | host2 |\
\n| 3.0 | 3.5 | 1970-01-01T00:00:05 | host2 |\
\n| 4.0 | 4.0 | 1970-01-01T00:00:10 | host2 |\
\n| 4.0 | 4.5 | 1970-01-01T00:00:15 | host2 |\
\n| 5.0 | 5.0 | 1970-01-01T00:00:20 | host2 |\
\n| 5.0 | | 1970-01-01T00:00:25 | host2 |\
\n+------------+------------+---------------------+-------+",
);
do_range_select_test(10_000, 5_000, 5_000, Fill::Linear, false, expected).await;
}
#[tokio::test]
async fn range_fill_const() {
let expected = String::from(
"+------------+------------+---------------------+-------+\
\n| MIN(value) | MAX(value) | timestamp | host |\
\n+------------+------------+---------------------+-------+\
\n| 0.0 | 0.0 | 1970-01-01T00:00:00 | host1 |\
\n| 0.0 | 6.6 | 1970-01-01T00:00:05 | host1 |\
\n| 1.0 | 1.0 | 1970-01-01T00:00:10 | host1 |\
\n| 1.0 | 6.6 | 1970-01-01T00:00:15 | host1 |\
\n| 2.0 | 2.0 | 1970-01-01T00:00:20 | host1 |\
\n| 2.0 | 6.6 | 1970-01-01T00:00:25 | host1 |\
\n| 3.0 | 3.0 | 1970-01-01T00:00:00 | host2 |\
\n| 3.0 | 6.6 | 1970-01-01T00:00:05 | host2 |\
\n| 4.0 | 4.0 | 1970-01-01T00:00:10 | host2 |\
\n| 4.0 | 6.6 | 1970-01-01T00:00:15 | host2 |\
\n| 5.0 | 5.0 | 1970-01-01T00:00:20 | host2 |\
\n| 5.0 | 6.6 | 1970-01-01T00:00:25 | host2 |\
\n+------------+------------+---------------------+-------+",
);
do_range_select_test(
10_000,
5_000,
5_000,
Fill::Const(ScalarValue::Float64(Some(6.6))),
true,
expected,
)
.await;
}
#[test]
fn fill_test() {
assert!(Fill::try_from_str("Linear", &DataType::UInt8).unwrap() == Fill::Linear);
assert_eq!(
Fill::try_from_str("Linear", &DataType::Boolean)
.unwrap_err()
.to_string(),
"Error during planning: Use FILL LINEAR on Non-numeric DataType Boolean"
);
assert_eq!(
Fill::try_from_str("WHAT", &DataType::UInt8)
.unwrap_err()
.to_string(),
"Error during planning: WHAT is not a valid fill option, fail to convert to a const value. { Arrow error: Cast error: Cannot cast string 'WHAT' to value of UInt8 type }"
);
assert_eq!(
Fill::try_from_str("8.0", &DataType::UInt8)
.unwrap_err()
.to_string(),
"Error during planning: 8.0 is not a valid fill option, fail to convert to a const value. { Arrow error: Cast error: Cannot cast string '8.0' to value of UInt8 type }"
);
assert!(
Fill::try_from_str("8", &DataType::UInt8).unwrap()
== Fill::Const(ScalarValue::UInt8(Some(8)))
);
let mut test1 = vec![
ScalarValue::UInt8(Some(8)),
ScalarValue::UInt8(None),
ScalarValue::UInt8(Some(9)),
];
Fill::Null.apply_fill_strategy(&mut test1).unwrap();
assert_eq!(test1[1], ScalarValue::UInt8(None));
Fill::Prev.apply_fill_strategy(&mut test1).unwrap();
assert_eq!(test1[1], ScalarValue::UInt8(Some(8)));
test1[1] = ScalarValue::UInt8(None);
Fill::Const(ScalarValue::UInt8(Some(10)))
.apply_fill_strategy(&mut test1)
.unwrap();
assert_eq!(test1[1], ScalarValue::UInt8(Some(10)));
test1[1] = ScalarValue::UInt8(None);
assert_eq!(
Fill::Linear
.apply_fill_strategy(&mut test1)
.unwrap_err()
.to_string(),
"Execution error: RangePlan: Apply Fill LINEAR strategy on Non-floating type"
);
let mut test2 = vec![
ScalarValue::Float32(Some(8.0)),
ScalarValue::Float32(None),
ScalarValue::Float32(Some(9.0)),
];
Fill::Linear.apply_fill_strategy(&mut test2).unwrap();
assert_eq!(test2[1], ScalarValue::Float32(Some(8.5)));
}
}

View File

@@ -12,10 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use std::str::FromStr;
use std::collections::BTreeSet;
use std::sync::Arc;
use std::time::Duration;
use arrow_schema::DataType;
use async_recursion::async_recursion;
use catalog::table_source::DfTableSourceProvider;
use datafusion::datasource::DefaultTableSource;
@@ -23,47 +24,62 @@ use datafusion::prelude::Column;
use datafusion::scalar::ScalarValue;
use datafusion_common::tree_node::{TreeNode, TreeNodeRewriter, VisitRecursion};
use datafusion_common::{DFSchema, DataFusionError, Result as DFResult};
use datafusion_expr::expr::{AggregateFunction, AggregateUDF, ScalarUDF};
use datafusion_expr::expr::ScalarUDF;
use datafusion_expr::{
AggregateFunction as AggregateFn, Expr, Extension, LogicalPlan, LogicalPlanBuilder, Projection,
Aggregate, Expr, ExprSchemable, Extension, LogicalPlan, LogicalPlanBuilder, Projection,
};
use datafusion_sql::planner::ContextProvider;
use datatypes::prelude::ConcreteDataType;
use promql_parser::util::parse_duration;
use snafu::{OptionExt, ResultExt};
use table::table::adapter::DfTableProviderAdapter;
use super::plan::Fill;
use crate::error::{
CatalogSnafu, DataFusionSnafu, Result, TimeIndexNotFoundSnafu, UnknownTableSnafu,
CatalogSnafu, DataFusionSnafu, RangeQuerySnafu, Result, TimeIndexNotFoundSnafu,
UnknownTableSnafu,
};
use crate::range_select::plan::{RangeFn, RangeSelect};
use crate::DfContextProviderAdapter;
/// `RangeExprRewriter` will recursively search certain `Expr`, find all `range_fn` scalar udf contained in `Expr`,
/// and collect the information required by the RangeSelect query,
/// and finally modify the `range_fn` scalar udf to an ordinary column field.
pub struct RangeExprRewriter<'a> {
input_plan: &'a Arc<LogicalPlan>,
align: Duration,
by: Vec<Expr>,
range_fn: Vec<RangeFn>,
context_provider: &'a DfContextProviderAdapter,
/// Use `BTreeSet` to avoid in case like `avg(a) RANGE '5m' + avg(a) RANGE '5m'`, duplicate range expr `avg(a) RANGE '5m'` be calculate twice
range_fn: BTreeSet<RangeFn>,
sub_aggr: &'a Aggregate,
}
#[inline]
fn dispose_parse_error(expr: Option<&Expr>) -> DataFusionError {
DataFusionError::Plan(
expr.map(|x| {
format!(
"Illegal argument `{}` in range select query",
x.display_name().unwrap_or_default()
)
})
.unwrap_or("Missing argument in range select query".into()),
)
}
impl<'a> RangeExprRewriter<'a> {
pub fn gen_range_expr(&self, func_name: &str, args: Vec<Expr>) -> DFResult<Expr> {
match AggregateFn::from_str(func_name) {
Ok(agg_fn) => Ok(Expr::AggregateFunction(AggregateFunction::new(
agg_fn, args, false, None, None,
))),
Err(_) => match self.context_provider.get_aggregate_meta(func_name) {
Some(agg_udf) => Ok(Expr::AggregateUDF(AggregateUDF::new(
agg_udf, args, None, None,
))),
None => Err(DataFusionError::Plan(format!(
"{} is not a Aggregate function or a Aggregate UDF",
func_name
))),
},
pub fn get_range_expr(&self, args: &[Expr], i: usize) -> DFResult<Expr> {
match args.get(i) {
Some(Expr::Column(column)) => {
let index = self.sub_aggr.schema.index_of_column(column)?;
let len = self.sub_aggr.group_expr.len();
self.sub_aggr
.aggr_expr
.get(index - len)
.cloned()
.ok_or(DataFusionError::Plan(
"Range expr not found in underlying Aggregate Plan".into(),
))
}
other => Err(dispose_parse_error(other)),
}
}
}
@@ -71,9 +87,7 @@ impl<'a> RangeExprRewriter<'a> {
fn parse_str_expr(args: &[Expr], i: usize) -> DFResult<&str> {
match args.get(i) {
Some(Expr::Literal(ScalarValue::Utf8(Some(str)))) => Ok(str.as_str()),
_ => Err(DataFusionError::Plan(
"Illegal argument in range select query".into(),
)),
other => Err(dispose_parse_error(other)),
}
}
@@ -88,10 +102,8 @@ fn parse_expr_list(args: &[Expr], start: usize, len: usize) -> DFResult<Vec<Expr
| Expr::ScalarFunction(_)
| Expr::ScalarUDF(_),
) => args[i].clone(),
_ => {
return Err(DataFusionError::Plan(
"Illegal expr argument in range select query".into(),
))
other => {
return Err(dispose_parse_error(*other));
}
});
}
@@ -104,23 +116,22 @@ impl<'a> TreeNodeRewriter for RangeExprRewriter<'a> {
fn mutate(&mut self, node: Expr) -> DFResult<Expr> {
if let Expr::ScalarUDF(func) = &node {
if func.fun.name == "range_fn" {
// `range_fn(func_name, argc, [argv], range, fill, byc, [byv], align)`
// `argsv` and `byv` are variadic arguments, argc/byc indicate the length of arguments
let func_name = parse_str_expr(&func.args, 0)?;
let argc = str::parse::<usize>(parse_str_expr(&func.args, 1)?)
// `range_fn(func, range, fill, byc, [byv], align)`
// `[byv]` are variadic arguments, byc indicate the length of arguments
let range_expr = self.get_range_expr(&func.args, 0)?;
let range_str = parse_str_expr(&func.args, 1)?;
let byc = str::parse::<usize>(parse_str_expr(&func.args, 3)?)
.map_err(|e| DataFusionError::Plan(e.to_string()))?;
let byc = str::parse::<usize>(parse_str_expr(&func.args, argc + 4)?)
.map_err(|e| DataFusionError::Plan(e.to_string()))?;
let mut range_fn = RangeFn {
expr: Expr::Wildcard,
range: parse_duration(parse_str_expr(&func.args, argc + 2)?)
.map_err(DataFusionError::Plan)?,
fill: parse_str_expr(&func.args, argc + 3)?.to_string(),
};
let args = parse_expr_list(&func.args, 2, argc)?;
let by = parse_expr_list(&func.args, argc + 5, byc)?;
let align = parse_duration(parse_str_expr(&func.args, argc + byc + 5)?)
let by = parse_expr_list(&func.args, 4, byc)?;
let align = parse_duration(parse_str_expr(&func.args, byc + 4)?)
.map_err(DataFusionError::Plan)?;
let mut data_type = range_expr.get_type(self.input_plan.schema())?;
let mut need_cast = false;
let fill = Fill::try_from_str(parse_str_expr(&func.args, 2)?, &data_type)?;
if matches!(fill, Fill::Linear) && data_type.is_integer() {
data_type = DataType::Float64;
need_cast = true;
}
if !self.by.is_empty() && self.by != by {
return Err(DataFusionError::Plan(
"Inconsistent by given in Range Function Rewrite".into(),
@@ -135,9 +146,21 @@ impl<'a> TreeNodeRewriter for RangeExprRewriter<'a> {
} else {
self.align = align;
}
range_fn.expr = self.gen_range_expr(func_name, args)?;
let alias = Expr::Column(Column::from_name(range_fn.expr.display_name()?));
self.range_fn.push(range_fn);
let range_fn = RangeFn {
name: format!(
"{} RANGE {} FILL {}",
range_expr.display_name()?,
range_str,
fill
),
data_type,
expr: range_expr,
range: parse_duration(range_str).map_err(DataFusionError::Plan)?,
fill,
need_cast,
};
let alias = Expr::Column(Column::from_name(range_fn.name.clone()));
self.range_fn.insert(range_fn);
return Ok(alias);
}
}
@@ -146,25 +169,18 @@ impl<'a> TreeNodeRewriter for RangeExprRewriter<'a> {
}
/// In order to implement RangeSelect query like `avg(field_0) RANGE '5m' FILL NULL`,
/// All RangeSelect query items are converted into udf scalar function in sql parse stage, with format like `range_fn('avg', .....)`.
/// All RangeSelect query items are converted into udf scalar function in sql parse stage, with format like `range_fn(avg(field_0), .....)`.
/// `range_fn` contains all the parameters we need to execute RangeSelect.
/// In order to correctly execute the query process of range select, we need to modify the query plan generated by datafusion.
/// We need to recursively find the entire LogicalPlan, and find all `range_fn` scalar udf contained in the project plan,
/// collecting info we need to generate RangeSelect Query LogicalPlan and rewrite th original LogicalPlan.
pub struct RangePlanRewriter {
table_provider: DfTableSourceProvider,
context_provider: DfContextProviderAdapter,
}
impl RangePlanRewriter {
pub fn new(
table_provider: DfTableSourceProvider,
context_provider: DfContextProviderAdapter,
) -> Self {
Self {
table_provider,
context_provider,
}
pub fn new(table_provider: DfTableSourceProvider) -> Self {
Self { table_provider }
}
pub async fn rewrite(&mut self, plan: LogicalPlan) -> Result<LogicalPlan> {
@@ -185,17 +201,28 @@ impl RangePlanRewriter {
LogicalPlan::Projection(Projection { expr, input, .. })
if have_range_in_exprs(expr) =>
{
let input = if let Some(new_input) = new_inputs[0].take() {
Arc::new(new_input)
let (aggr_plan, input) = if let LogicalPlan::Aggregate(aggr) = input.as_ref() {
// Expr like `rate(max(a) RANGE '6m') RANGE '6m'` have legal syntax but illegal semantic.
if have_range_in_exprs(&aggr.aggr_expr) {
return RangeQuerySnafu {
msg: "Nest Range Query is not allowed",
}
.fail();
}
(aggr, aggr.input.clone())
} else {
input.clone()
return RangeQuerySnafu {
msg: "Window functions is not allowed in Range Query",
}
.fail();
};
let (time_index, default_by) = self.get_index_by(input.schema().clone()).await?;
let (time_index, default_by) = self.get_index_by(input.schema()).await?;
let mut range_rewriter = RangeExprRewriter {
input_plan: &input,
align: Duration::default(),
by: vec![],
range_fn: vec![],
context_provider: &self.context_provider,
range_fn: BTreeSet::new(),
sub_aggr: aggr_plan,
};
let new_expr = expr
.iter()
@@ -207,7 +234,7 @@ impl RangePlanRewriter {
}
let range_select = RangeSelect::try_new(
input.clone(),
range_rewriter.range_fn,
range_rewriter.range_fn.into_iter().collect(),
range_rewriter.align,
time_index,
range_rewriter.by,
@@ -252,7 +279,7 @@ impl RangePlanRewriter {
/// return `(time_index, [row_columns])` to the rewriter.
/// If the user does not explicitly use the `by` keyword to indicate time series,
/// `[row_columns]` will be use as default time series
async fn get_index_by(&mut self, schema: Arc<DFSchema>) -> Result<(Expr, Vec<Expr>)> {
async fn get_index_by(&mut self, schema: &Arc<DFSchema>) -> Result<(Expr, Vec<Expr>)> {
let mut time_index_expr = Expr::Wildcard;
let mut default_by = vec![];
for field in schema.fields() {
@@ -303,28 +330,27 @@ impl RangePlanRewriter {
}
}
fn have_range_in_exprs(exprs: &Vec<Expr>) -> bool {
let mut have = false;
for expr in exprs {
fn have_range_in_exprs(exprs: &[Expr]) -> bool {
exprs.iter().any(|expr| {
let mut find_range = false;
let _ = expr.apply(&mut |expr| {
if let Expr::ScalarUDF(ScalarUDF { fun, .. }) = expr {
if fun.name == "range_fn" {
have = true;
find_range = true;
return Ok(VisitRecursion::Stop);
}
}
Ok(VisitRecursion::Continue)
});
if have {
break;
}
}
have
find_range
})
}
#[cfg(test)]
mod test {
use std::error::Error;
use catalog::memory::MemoryCatalogManager;
use catalog::RegisterTableRequest;
use common_catalog::consts::{DEFAULT_CATALOG_NAME, DEFAULT_SCHEMA_NAME};
@@ -391,14 +417,14 @@ mod test {
QueryEngineFactory::new(catalog_list, None, None, false).query_engine()
}
async fn query_plan_compare(sql: &str, expected: String) {
async fn do_query(sql: &str) -> Result<crate::plan::LogicalPlan> {
let stmt = QueryLanguageParser::parse_sql(sql).unwrap();
let engine = create_test_engine().await;
let GreptimeLogicalPlan::DfPlan(plan) = engine
.planner()
.plan(stmt, QueryContext::arc())
.await
.unwrap();
engine.planner().plan(stmt, QueryContext::arc()).await
}
async fn query_plan_compare(sql: &str, expected: String) {
let GreptimeLogicalPlan::DfPlan(plan) = do_query(sql).await.unwrap();
assert_eq!(plan.display_indent_schema().to_string(), expected);
}
@@ -406,7 +432,7 @@ mod test {
async fn range_no_project() {
let query = r#"SELECT timestamp, tag_0, tag_1, avg(field_0 + field_1) RANGE '5m' FROM test ALIGN '1h' by (tag_0,tag_1);"#;
let expected = String::from(
"RangeSelect: range_exprs=[RangeFn { expr:AVG(test.field_0 + test.field_1) range:300s fill: }], align=3600s time_index=timestamp [timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8, AVG(test.field_0 + test.field_1):Float64;N]\
"RangeSelect: range_exprs=[AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL], align=3600s time_index=timestamp [timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8, AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL:Float64;N]\
\n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]"
);
query_plan_compare(query, expected).await;
@@ -414,11 +440,10 @@ mod test {
#[tokio::test]
async fn range_expr_calculation() {
let query =
r#"SELECT avg(field_0 + field_1)/4 RANGE '5m' FROM test ALIGN '1h' by (tag_0,tag_1);"#;
let query = r#"SELECT (avg(field_0 + field_1)/4) RANGE '5m' FROM test ALIGN '1h' by (tag_0,tag_1);"#;
let expected = String::from(
"Projection: AVG(test.field_0 + test.field_1) / Int64(4) [AVG(test.field_0 + test.field_1) / Int64(4):Float64;N]\
\n RangeSelect: range_exprs=[RangeFn { expr:AVG(test.field_0 + test.field_1) range:300s fill: }], align=3600s time_index=timestamp [AVG(test.field_0 + test.field_1):Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\
"Projection: AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL / Int64(4) [AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL / Int64(4):Float64;N]\
\n RangeSelect: range_exprs=[AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL], align=3600s time_index=timestamp [AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\
\n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]"
);
query_plan_compare(query, expected).await;
@@ -427,10 +452,10 @@ mod test {
#[tokio::test]
async fn range_multi_args() {
let query =
r#"SELECT covar(field_0 + field_1, field_1)/4 RANGE '5m' FROM test ALIGN '1h';"#;
r#"SELECT (covar(field_0 + field_1, field_1)/4) RANGE '5m' FROM test ALIGN '1h';"#;
let expected = String::from(
"Projection: COVARIANCE(test.field_0 + test.field_1,test.field_1) / Int64(4) [COVARIANCE(test.field_0 + test.field_1,test.field_1) / Int64(4):Float64;N]\
\n RangeSelect: range_exprs=[RangeFn { expr:COVARIANCE(test.field_0 + test.field_1,test.field_1) range:300s fill: }], align=3600s time_index=timestamp [COVARIANCE(test.field_0 + test.field_1,test.field_1):Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8]\
"Projection: COVARIANCE(test.field_0 + test.field_1,test.field_1) RANGE 5m FILL NULL / Int64(4) [COVARIANCE(test.field_0 + test.field_1,test.field_1) RANGE 5m FILL NULL / Int64(4):Float64;N]\
\n RangeSelect: range_exprs=[COVARIANCE(test.field_0 + test.field_1,test.field_1) RANGE 5m FILL NULL], align=3600s time_index=timestamp [COVARIANCE(test.field_0 + test.field_1,test.field_1) RANGE 5m FILL NULL:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8]\
\n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]"
);
query_plan_compare(query, expected).await;
@@ -438,10 +463,10 @@ mod test {
#[tokio::test]
async fn range_calculation() {
let query = r#"SELECT (avg(field_0)+sum(field_1))/4 RANGE '5m' FROM test ALIGN '1h' by (tag_0,tag_1) FILL NULL;"#;
let query = r#"SELECT ((avg(field_0)+sum(field_1))/4) RANGE '5m' FROM test ALIGN '1h' by (tag_0,tag_1) FILL NULL;"#;
let expected = String::from(
"Projection: (AVG(test.field_0) + SUM(test.field_1)) / Int64(4) [AVG(test.field_0) + SUM(test.field_1) / Int64(4):Float64;N]\
\n RangeSelect: range_exprs=[RangeFn { expr:AVG(test.field_0) range:300s fill:NULL }, RangeFn { expr:SUM(test.field_1) range:300s fill:NULL }], align=3600s time_index=timestamp [AVG(test.field_0):Float64;N, SUM(test.field_1):Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\
"Projection: (AVG(test.field_0) RANGE 5m FILL NULL + SUM(test.field_1) RANGE 5m FILL NULL) / Int64(4) [AVG(test.field_0) RANGE 5m FILL NULL + SUM(test.field_1) RANGE 5m FILL NULL / Int64(4):Float64;N]\
\n RangeSelect: range_exprs=[AVG(test.field_0) RANGE 5m FILL NULL, SUM(test.field_1) RANGE 5m FILL NULL], align=3600s time_index=timestamp [AVG(test.field_0) RANGE 5m FILL NULL:Float64;N, SUM(test.field_1) RANGE 5m FILL NULL:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\
\n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]"
);
query_plan_compare(query, expected).await;
@@ -449,12 +474,12 @@ mod test {
#[tokio::test]
async fn range_as_sub_query() {
let query = r#"SELECT foo + 1 from (SELECT (avg(field_0)+sum(field_1))/4 RANGE '5m' as foo FROM test ALIGN '1h' by (tag_0,tag_1) FILL NULL) where foo > 1;"#;
let query = r#"SELECT foo + 1 from (SELECT ((avg(field_0)+sum(field_1))/4) RANGE '5m' as foo FROM test ALIGN '1h' by (tag_0,tag_1) FILL NULL) where foo > 1;"#;
let expected = String::from(
"Projection: foo + Int64(1) [foo + Int64(1):Float64;N]\
\n Filter: foo > Int64(1) [foo:Float64;N]\
\n Projection: (AVG(test.field_0) + SUM(test.field_1)) / Int64(4) AS foo [foo:Float64;N]\
\n RangeSelect: range_exprs=[RangeFn { expr:AVG(test.field_0) range:300s fill:NULL }, RangeFn { expr:SUM(test.field_1) range:300s fill:NULL }], align=3600s time_index=timestamp [AVG(test.field_0):Float64;N, SUM(test.field_1):Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\
\n Projection: (AVG(test.field_0) RANGE 5m FILL NULL + SUM(test.field_1) RANGE 5m FILL NULL) / Int64(4) AS foo [foo:Float64;N]\
\n RangeSelect: range_exprs=[AVG(test.field_0) RANGE 5m FILL NULL, SUM(test.field_1) RANGE 5m FILL NULL], align=3600s time_index=timestamp [AVG(test.field_0) RANGE 5m FILL NULL:Float64;N, SUM(test.field_1) RANGE 5m FILL NULL:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\
\n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]"
);
query_plan_compare(query, expected).await;
@@ -462,14 +487,109 @@ mod test {
#[tokio::test]
async fn range_from_nest_query() {
let query = r#"SELECT (avg(a)+sum(b))/4 RANGE '5m' FROM (SELECT field_0 as a, field_1 as b, tag_0 as c, tag_1 as d, timestamp from test where field_0 > 1.0) ALIGN '1h' by (c, d) FILL NULL;"#;
let query = r#"SELECT ((avg(a)+sum(b))/4) RANGE '5m' FROM (SELECT field_0 as a, field_1 as b, tag_0 as c, tag_1 as d, timestamp from test where field_0 > 1.0) ALIGN '1h' by (c, d) FILL NULL;"#;
let expected = String::from(
"Projection: (AVG(a) + SUM(b)) / Int64(4) [AVG(a) + SUM(b) / Int64(4):Float64;N]\
\n RangeSelect: range_exprs=[RangeFn { expr:AVG(a) range:300s fill:NULL }, RangeFn { expr:SUM(b) range:300s fill:NULL }], align=3600s time_index=timestamp [AVG(a):Float64;N, SUM(b):Float64;N, timestamp:Timestamp(Millisecond, None), c:Utf8, d:Utf8]\
"Projection: (AVG(a) RANGE 5m FILL NULL + SUM(b) RANGE 5m FILL NULL) / Int64(4) [AVG(a) RANGE 5m FILL NULL + SUM(b) RANGE 5m FILL NULL / Int64(4):Float64;N]\
\n RangeSelect: range_exprs=[AVG(a) RANGE 5m FILL NULL, SUM(b) RANGE 5m FILL NULL], align=3600s time_index=timestamp [AVG(a) RANGE 5m FILL NULL:Float64;N, SUM(b) RANGE 5m FILL NULL:Float64;N, timestamp:Timestamp(Millisecond, None), c:Utf8, d:Utf8]\
\n Projection: test.field_0 AS a, test.field_1 AS b, test.tag_0 AS c, test.tag_1 AS d, test.timestamp [a:Float64;N, b:Float64;N, c:Utf8, d:Utf8, timestamp:Timestamp(Millisecond, None)]\
\n Filter: test.field_0 > Float64(1) [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]\
\n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]"
);
query_plan_compare(query, expected).await;
}
#[tokio::test]
async fn range_in_expr() {
let query = r#"SELECT sin(avg(field_0 + field_1) RANGE '5m' + 1) FROM test ALIGN '1h' by (tag_0,tag_1);"#;
let expected = String::from(
"Projection: sin(AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL + Int64(1)) [sin(AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL + Int64(1)):Float64;N]\
\n RangeSelect: range_exprs=[AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL], align=3600s time_index=timestamp [AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\
\n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]"
);
query_plan_compare(query, expected).await;
}
#[tokio::test]
async fn duplicate_range_expr() {
let query = r#"SELECT avg(field_0) RANGE '5m' FILL 6.0 + avg(field_0) RANGE '5m' FILL 6.0 FROM test ALIGN '1h' by (tag_0,tag_1);"#;
let expected = String::from(
"Projection: AVG(test.field_0) RANGE 5m FILL 6 + AVG(test.field_0) RANGE 5m FILL 6 [AVG(test.field_0) RANGE 5m FILL 6 + AVG(test.field_0) RANGE 5m FILL 6:Float64]\
\n RangeSelect: range_exprs=[AVG(test.field_0) RANGE 5m FILL 6], align=3600s time_index=timestamp [AVG(test.field_0) RANGE 5m FILL 6:Float64, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\
\n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]"
);
query_plan_compare(query, expected).await;
}
#[tokio::test]
async fn deep_nest_range_expr() {
let query = r#"SELECT round(sin(avg(field_0 + field_1) RANGE '5m' + 1)) FROM test ALIGN '1h' by (tag_0,tag_1);"#;
let expected = String::from(
"Projection: round(sin(AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL + Int64(1))) [round(sin(AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL + Int64(1))):Float64;N]\
\n RangeSelect: range_exprs=[AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL], align=3600s time_index=timestamp [AVG(test.field_0 + test.field_1) RANGE 5m FILL NULL:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\
\n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]"
);
query_plan_compare(query, expected).await;
}
#[tokio::test]
async fn complex_range_expr() {
let query = r#"SELECT gcd(CAST(max(field_0 + 1) Range '5m' FILL NULL AS Int64), CAST(tag_0 AS Int64)) + round(max(field_2+1) Range '6m' FILL NULL + 1) + max(field_2+3) Range '10m' FILL NULL * CAST(tag_1 AS Float64) + 1 FROM test ALIGN '1h' by (tag_0, tag_1);"#;
let expected = String::from(
"Projection: gcd(CAST(MAX(test.field_0 + Int64(1)) RANGE 5m FILL NULL AS Int64), CAST(test.tag_0 AS Int64)) + round(MAX(test.field_2 + Int64(1)) RANGE 6m FILL NULL + Int64(1)) + MAX(test.field_2 + Int64(3)) RANGE 10m FILL NULL * CAST(test.tag_1 AS Float64) + Int64(1) [gcd(MAX(test.field_0 + Int64(1)) RANGE 5m FILL NULL,test.tag_0) + round(MAX(test.field_2 + Int64(1)) RANGE 6m FILL NULL + Int64(1)) + MAX(test.field_2 + Int64(3)) RANGE 10m FILL NULL * test.tag_1 + Int64(1):Float64;N]\
\n RangeSelect: range_exprs=[MAX(test.field_0 + Int64(1)) RANGE 5m FILL NULL, MAX(test.field_2 + Int64(1)) RANGE 6m FILL NULL, MAX(test.field_2 + Int64(3)) RANGE 10m FILL NULL], align=3600s time_index=timestamp [MAX(test.field_0 + Int64(1)) RANGE 5m FILL NULL:Float64;N, MAX(test.field_2 + Int64(1)) RANGE 6m FILL NULL:Float64;N, MAX(test.field_2 + Int64(3)) RANGE 10m FILL NULL:Float64;N, timestamp:Timestamp(Millisecond, None), tag_0:Utf8, tag_1:Utf8]\
\n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]"
);
query_plan_compare(query, expected).await;
}
#[tokio::test]
async fn range_linear_on_integer() {
let query = r#"SELECT min(CAST(field_0 AS Int64) + CAST(field_1 AS Int64)) RANGE '5m' FILL LINEAR FROM test ALIGN '1h' by (tag_0,tag_1);"#;
let expected = String::from(
"RangeSelect: range_exprs=[MIN(test.field_0 + test.field_1) RANGE 5m FILL LINEAR], align=3600s time_index=timestamp [MIN(test.field_0 + test.field_1) RANGE 5m FILL LINEAR:Float64;N]\
\n TableScan: test [tag_0:Utf8, tag_1:Utf8, tag_2:Utf8, tag_3:Utf8, tag_4:Utf8, timestamp:Timestamp(Millisecond, None), field_0:Float64;N, field_1:Float64;N, field_2:Float64;N, field_3:Float64;N, field_4:Float64;N]"
);
query_plan_compare(query, expected).await;
}
#[tokio::test]
async fn range_nest_range_err() {
let query = r#"SELECT sum(avg(field_0 + field_1) RANGE '5m' + 1) RANGE '5m' + 1 FROM test ALIGN '1h' by (tag_0,tag_1);"#;
assert_eq!(
do_query(query).await.unwrap_err().to_string(),
"Range Query: Nest Range Query is not allowed"
)
}
#[tokio::test]
/// Start directly from the rewritten SQL and check whether the error reported by the range expression rewriting is as expected.
/// the right argument is `range_fn(avg(field_0), '5m', 'NULL', '0', '1h')`
async fn range_argument_err_1() {
let query = r#"SELECT range_fn('5m', avg(field_0), 'NULL', '1', tag_0, '1h') FROM test group by tag_0;"#;
let error = do_query(query)
.await
.unwrap_err()
.source()
.unwrap()
.to_string();
assert_eq!(
error,
"Error during planning: Illegal argument `Utf8(\"5m\")` in range select query"
)
}
#[tokio::test]
async fn range_argument_err_2() {
let query = r#"SELECT range_fn(avg(field_0), 5, 'NULL', '1', tag_0, '1h') FROM test group by tag_0;"#;
let error = do_query(query)
.await
.unwrap_err()
.source()
.unwrap()
.to_string();
assert_eq!(
error,
"Error during planning: Illegal argument `Int64(5)` in range select query"
)
}
}

View File

@@ -14,7 +14,7 @@
use std::ops::Deref;
use common_error::ext::{BoxedError, ErrorExt};
use common_error::ext::ErrorExt;
use common_query::Output;
use common_recordbatch::{RecordBatch, SendableRecordBatchStream};
use datatypes::prelude::{ConcreteDataType, Value};
@@ -28,7 +28,7 @@ use session::context::QueryContextRef;
use snafu::prelude::*;
use tokio::io::AsyncWrite;
use crate::error::{self, Error, OtherSnafu, Result};
use crate::error::{self, Error, Result};
use crate::metrics::*;
/// Try to write multiple output to the writer if possible.
@@ -148,7 +148,11 @@ impl<'a, W: AsyncWrite + Unpin> MysqlResultWriter<'a, W> {
.await?
}
Err(e) => {
return Err(e).map_err(BoxedError::new).context(OtherSnafu);
let err = e.to_string();
row_writer
.finish_error(ErrorKind::ER_INTERNAL_ERROR, &err.as_bytes())
.await?;
return Ok(());
}
}
}

View File

@@ -35,6 +35,8 @@ pub struct TableReference<'a> {
pub table: &'a str,
}
pub type OwnedTableReference = TableReference<'static>;
// TODO(LFC): Find a better place for `TableReference`,
// so that we can reuse the default catalog and schema consts.
// Could be done together with issue #559.

View File

@@ -92,7 +92,7 @@ impl GreptimeDbStandaloneBuilder {
.init()
.await
.unwrap();
procedure_manager.start().await.unwrap();
let instance = Instance::try_new_standalone(
kv_store,
procedure_manager,

View File

@@ -0,0 +1,54 @@
CREATE TABLE host (
ts timestamp(3) time index,
host STRING PRIMARY KEY,
val BIGINT,
);
Affected Rows: 0
INSERT INTO TABLE host VALUES
(0, 'host1', 0),
(5000, 'host1', null),
(10000, 'host1', 1),
(15000, 'host1', null),
(20000, 'host1', 2),
(0, 'host2', 3),
(5000, 'host2', null),
(10000, 'host2', 4),
(15000, 'host2', null),
(20000, 'host2', 5);
Affected Rows: 10
-- Test by calculate
SELECT ts, length(host), max(val) RANGE '5s' FROM host ALIGN '20s' BY (length(host)) ORDER BY ts;
+---------------------+-----------------------------+----------------------------------+
| ts | character_length(host.host) | MAX(host.val) RANGE 5s FILL NULL |
+---------------------+-----------------------------+----------------------------------+
| 1970-01-01T00:00:00 | 5 | 3 |
| 1970-01-01T00:00:20 | 5 | 5 |
+---------------------+-----------------------------+----------------------------------+
SELECT ts, max(val) RANGE '5s' FROM host ALIGN '20s' BY (2) ORDER BY ts;
+---------------------+----------------------------------+
| ts | MAX(host.val) RANGE 5s FILL NULL |
+---------------------+----------------------------------+
| 1970-01-01T00:00:00 | 3 |
| 1970-01-01T00:00:20 | 5 |
+---------------------+----------------------------------+
SELECT ts, CAST(length(host) as INT64) + 2, max(val) RANGE '5s' FROM host ALIGN '20s' BY (CAST(length(host) as INT64) + 2) ORDER BY ts;
+---------------------+----------------------------------------+----------------------------------+
| ts | character_length(host.host) + Int64(2) | MAX(host.val) RANGE 5s FILL NULL |
+---------------------+----------------------------------------+----------------------------------+
| 1970-01-01T00:00:00 | 7 | 3 |
| 1970-01-01T00:00:20 | 7 | 5 |
+---------------------+----------------------------------------+----------------------------------+
DROP TABLE host;
Affected Rows: 0

View File

@@ -0,0 +1,27 @@
CREATE TABLE host (
ts timestamp(3) time index,
host STRING PRIMARY KEY,
val BIGINT,
);
INSERT INTO TABLE host VALUES
(0, 'host1', 0),
(5000, 'host1', null),
(10000, 'host1', 1),
(15000, 'host1', null),
(20000, 'host1', 2),
(0, 'host2', 3),
(5000, 'host2', null),
(10000, 'host2', 4),
(15000, 'host2', null),
(20000, 'host2', 5);
-- Test by calculate
SELECT ts, length(host), max(val) RANGE '5s' FROM host ALIGN '20s' BY (length(host)) ORDER BY ts;
SELECT ts, max(val) RANGE '5s' FROM host ALIGN '20s' BY (2) ORDER BY ts;
SELECT ts, CAST(length(host) as INT64) + 2, max(val) RANGE '5s' FROM host ALIGN '20s' BY (CAST(length(host) as INT64) + 2) ORDER BY ts;
DROP TABLE host;

View File

@@ -0,0 +1,194 @@
CREATE TABLE host (
ts timestamp(3) time index,
host STRING PRIMARY KEY,
val BIGINT,
);
Affected Rows: 0
INSERT INTO TABLE host VALUES
(0, 'host1', 0),
(5000, 'host1', null),
(10000, 'host1', 1),
(15000, 'host1', null),
(20000, 'host1', 2),
(0, 'host2', 3),
(5000, 'host2', null),
(10000, 'host2', 4),
(15000, 'host2', null),
(20000, 'host2', 5);
Affected Rows: 10
-- Test range expr calculate
SELECT ts, host, covar(val, val) RANGE '20s' FROM host ALIGN '10s' ORDER BY host, ts;
+---------------------+-------+---------------------------------------------------+
| ts | host | COVARIANCE(host.val,host.val) RANGE 20s FILL NULL |
+---------------------+-------+---------------------------------------------------+
| 1970-01-01T00:00:00 | host1 | |
| 1970-01-01T00:00:10 | host1 | 0.5 |
| 1970-01-01T00:00:20 | host1 | 0.5 |
| 1970-01-01T00:00:30 | host1 | |
| 1970-01-01T00:00:00 | host2 | |
| 1970-01-01T00:00:10 | host2 | 0.5 |
| 1970-01-01T00:00:20 | host2 | 0.5 |
| 1970-01-01T00:00:30 | host2 | |
+---------------------+-------+---------------------------------------------------+
SELECT ts, host, 2 * min(val) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts;
+---------------------+-------+---------------------------------------------+
| ts | host | Int64(2) * MIN(host.val) RANGE 5s FILL NULL |
+---------------------+-------+---------------------------------------------+
| 1970-01-01T00:00:00 | host1 | 0 |
| 1970-01-01T00:00:05 | host1 | |
| 1970-01-01T00:00:10 | host1 | 2 |
| 1970-01-01T00:00:15 | host1 | |
| 1970-01-01T00:00:20 | host1 | 4 |
| 1970-01-01T00:00:00 | host2 | 6 |
| 1970-01-01T00:00:05 | host2 | |
| 1970-01-01T00:00:10 | host2 | 8 |
| 1970-01-01T00:00:15 | host2 | |
| 1970-01-01T00:00:20 | host2 | 10 |
+---------------------+-------+---------------------------------------------+
SELECT ts, host, min(val * 2) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts;
+---------------------+-------+---------------------------------------------+
| ts | host | MIN(host.val * Int64(2)) RANGE 5s FILL NULL |
+---------------------+-------+---------------------------------------------+
| 1970-01-01T00:00:00 | host1 | 0 |
| 1970-01-01T00:00:05 | host1 | |
| 1970-01-01T00:00:10 | host1 | 2 |
| 1970-01-01T00:00:15 | host1 | |
| 1970-01-01T00:00:20 | host1 | 4 |
| 1970-01-01T00:00:00 | host2 | 6 |
| 1970-01-01T00:00:05 | host2 | |
| 1970-01-01T00:00:10 | host2 | 8 |
| 1970-01-01T00:00:15 | host2 | |
| 1970-01-01T00:00:20 | host2 | 10 |
+---------------------+-------+---------------------------------------------+
SELECT ts, host, min(CAST(val as Float64)) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts;
+---------------------+-------+----------------------------------+
| ts | host | MIN(host.val) RANGE 5s FILL NULL |
+---------------------+-------+----------------------------------+
| 1970-01-01T00:00:00 | host1 | 0.0 |
| 1970-01-01T00:00:05 | host1 | |
| 1970-01-01T00:00:10 | host1 | 1.0 |
| 1970-01-01T00:00:15 | host1 | |
| 1970-01-01T00:00:20 | host1 | 2.0 |
| 1970-01-01T00:00:00 | host2 | 3.0 |
| 1970-01-01T00:00:05 | host2 | |
| 1970-01-01T00:00:10 | host2 | 4.0 |
| 1970-01-01T00:00:15 | host2 | |
| 1970-01-01T00:00:20 | host2 | 5.0 |
+---------------------+-------+----------------------------------+
SELECT ts, host, min(floor(CAST(val as Float64))) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts;
+---------------------+-------+-----------------------------------------+
| ts | host | MIN(floor(host.val)) RANGE 5s FILL NULL |
+---------------------+-------+-----------------------------------------+
| 1970-01-01T00:00:00 | host1 | 0.0 |
| 1970-01-01T00:00:05 | host1 | |
| 1970-01-01T00:00:10 | host1 | 1.0 |
| 1970-01-01T00:00:15 | host1 | |
| 1970-01-01T00:00:20 | host1 | 2.0 |
| 1970-01-01T00:00:00 | host2 | 3.0 |
| 1970-01-01T00:00:05 | host2 | |
| 1970-01-01T00:00:10 | host2 | 4.0 |
| 1970-01-01T00:00:15 | host2 | |
| 1970-01-01T00:00:20 | host2 | 5.0 |
+---------------------+-------+-----------------------------------------+
SELECT ts, host, floor(min(val) RANGE '5s') FROM host ALIGN '5s' ORDER BY host, ts;
+---------------------+-------+-----------------------------------------+
| ts | host | floor(MIN(host.val) RANGE 5s FILL NULL) |
+---------------------+-------+-----------------------------------------+
| 1970-01-01T00:00:00 | host1 | 0.0 |
| 1970-01-01T00:00:05 | host1 | |
| 1970-01-01T00:00:10 | host1 | 1.0 |
| 1970-01-01T00:00:15 | host1 | |
| 1970-01-01T00:00:20 | host1 | 2.0 |
| 1970-01-01T00:00:00 | host2 | 3.0 |
| 1970-01-01T00:00:05 | host2 | |
| 1970-01-01T00:00:10 | host2 | 4.0 |
| 1970-01-01T00:00:15 | host2 | |
| 1970-01-01T00:00:20 | host2 | 5.0 |
+---------------------+-------+-----------------------------------------+
-- Test complex range expr calculate
SELECT ts, host, (min(val) + max(val)) RANGE '20s' + 1.0 FROM host ALIGN '10s' ORDER BY host, ts;
+---------------------+-------+------------------------------------------------------------------------------------+
| ts | host | MIN(host.val) RANGE 20s FILL NULL + MAX(host.val) RANGE 20s FILL NULL + Float64(1) |
+---------------------+-------+------------------------------------------------------------------------------------+
| 1970-01-01T00:00:00 | host1 | 1.0 |
| 1970-01-01T00:00:10 | host1 | 2.0 |
| 1970-01-01T00:00:20 | host1 | 4.0 |
| 1970-01-01T00:00:30 | host1 | 5.0 |
| 1970-01-01T00:00:00 | host2 | 7.0 |
| 1970-01-01T00:00:10 | host2 | 8.0 |
| 1970-01-01T00:00:20 | host2 | 10.0 |
| 1970-01-01T00:00:30 | host2 | 11.0 |
+---------------------+-------+------------------------------------------------------------------------------------+
SELECT ts, host, covar(ceil(CAST(val as Float64)), floor(CAST(val as Float64))) RANGE '20s' FROM host ALIGN '10s' ORDER BY host, ts;
+---------------------+-------+----------------------------------------------------------------+
| ts | host | COVARIANCE(ceil(host.val),floor(host.val)) RANGE 20s FILL NULL |
+---------------------+-------+----------------------------------------------------------------+
| 1970-01-01T00:00:00 | host1 | |
| 1970-01-01T00:00:10 | host1 | 0.5 |
| 1970-01-01T00:00:20 | host1 | 0.5 |
| 1970-01-01T00:00:30 | host1 | |
| 1970-01-01T00:00:00 | host2 | |
| 1970-01-01T00:00:10 | host2 | 0.5 |
| 1970-01-01T00:00:20 | host2 | 0.5 |
| 1970-01-01T00:00:30 | host2 | |
+---------------------+-------+----------------------------------------------------------------+
SELECT ts, host, floor(cos(ceil(sin(min(val) RANGE '5s')))) FROM host ALIGN '5s' ORDER BY host, ts;
+---------------------+-------+---------------------------------------------------------+
| ts | host | floor(cos(ceil(sin(MIN(host.val) RANGE 5s FILL NULL)))) |
+---------------------+-------+---------------------------------------------------------+
| 1970-01-01T00:00:00 | host1 | 1.0 |
| 1970-01-01T00:00:05 | host1 | |
| 1970-01-01T00:00:10 | host1 | 0.0 |
| 1970-01-01T00:00:15 | host1 | |
| 1970-01-01T00:00:20 | host1 | 0.0 |
| 1970-01-01T00:00:00 | host2 | 0.0 |
| 1970-01-01T00:00:05 | host2 | |
| 1970-01-01T00:00:10 | host2 | 1.0 |
| 1970-01-01T00:00:15 | host2 | |
| 1970-01-01T00:00:20 | host2 | 1.0 |
+---------------------+-------+---------------------------------------------------------+
SELECT ts, host, gcd(CAST(max(floor(CAST(val as Float64))) RANGE '10s' FILL PREV as INT64) * 4, max(val * 4) RANGE '10s' FILL PREV) * length(host) + 1 FROM host ALIGN '5s' ORDER BY host, ts;
+---------------------+-------+------------------------------------------------------------------------------------------------------------------------------------------------+
| ts | host | gcd(MAX(floor(host.val)) RANGE 10s FILL PREV * Int64(4),MAX(host.val * Int64(4)) RANGE 10s FILL PREV) * character_length(host.host) + Int64(1) |
+---------------------+-------+------------------------------------------------------------------------------------------------------------------------------------------------+
| 1970-01-01T00:00:00 | host1 | 1 |
| 1970-01-01T00:00:05 | host1 | 1 |
| 1970-01-01T00:00:10 | host1 | 21 |
| 1970-01-01T00:00:15 | host1 | 21 |
| 1970-01-01T00:00:20 | host1 | 41 |
| 1970-01-01T00:00:25 | host1 | 41 |
| 1970-01-01T00:00:00 | host2 | 61 |
| 1970-01-01T00:00:05 | host2 | 61 |
| 1970-01-01T00:00:10 | host2 | 81 |
| 1970-01-01T00:00:15 | host2 | 81 |
| 1970-01-01T00:00:20 | host2 | 101 |
| 1970-01-01T00:00:25 | host2 | 101 |
+---------------------+-------+------------------------------------------------------------------------------------------------------------------------------------------------+
DROP TABLE host;
Affected Rows: 0

View File

@@ -0,0 +1,43 @@
CREATE TABLE host (
ts timestamp(3) time index,
host STRING PRIMARY KEY,
val BIGINT,
);
INSERT INTO TABLE host VALUES
(0, 'host1', 0),
(5000, 'host1', null),
(10000, 'host1', 1),
(15000, 'host1', null),
(20000, 'host1', 2),
(0, 'host2', 3),
(5000, 'host2', null),
(10000, 'host2', 4),
(15000, 'host2', null),
(20000, 'host2', 5);
-- Test range expr calculate
SELECT ts, host, covar(val, val) RANGE '20s' FROM host ALIGN '10s' ORDER BY host, ts;
SELECT ts, host, 2 * min(val) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts;
SELECT ts, host, min(val * 2) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts;
SELECT ts, host, min(CAST(val as Float64)) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts;
SELECT ts, host, min(floor(CAST(val as Float64))) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts;
SELECT ts, host, floor(min(val) RANGE '5s') FROM host ALIGN '5s' ORDER BY host, ts;
-- Test complex range expr calculate
SELECT ts, host, (min(val) + max(val)) RANGE '20s' + 1.0 FROM host ALIGN '10s' ORDER BY host, ts;
SELECT ts, host, covar(ceil(CAST(val as Float64)), floor(CAST(val as Float64))) RANGE '20s' FROM host ALIGN '10s' ORDER BY host, ts;
SELECT ts, host, floor(cos(ceil(sin(min(val) RANGE '5s')))) FROM host ALIGN '5s' ORDER BY host, ts;
SELECT ts, host, gcd(CAST(max(floor(CAST(val as Float64))) RANGE '10s' FILL PREV as INT64) * 4, max(val * 4) RANGE '10s' FILL PREV) * length(host) + 1 FROM host ALIGN '5s' ORDER BY host, ts;
DROP TABLE host;

View File

@@ -0,0 +1,82 @@
CREATE TABLE host (
ts timestamp(3) time index,
host STRING PRIMARY KEY,
val BIGINT,
);
Affected Rows: 0
INSERT INTO TABLE host VALUES
(0, 'host1', 0),
(5000, 'host1', null),
(10000, 'host1', 1),
(15000, 'host1', null),
(20000, 'host1', 2),
(0, 'host2', 3),
(5000, 'host2', null),
(10000, 'host2', 4),
(15000, 'host2', null),
(20000, 'host2', 5);
Affected Rows: 10
-- Test Invalid cases
-- 1. error timestamp
SELECT min(val) RANGE 'not_time' FROM host ALIGN '5s';
Error: 2000(InvalidSyntax), sql parser error: not a valid duration string: not_time
SELECT min(val) RANGE '5s' FROM host ALIGN 'not_time';
Error: 2000(InvalidSyntax), sql parser error: not a valid duration string: not_time
-- 2.1 no range param
SELECT min(val) FROM host ALIGN '5s';
Error: 2000(InvalidSyntax), sql parser error: Illegal Range select, no RANGE keyword found in any SelectItem
SELECT min(val) RANGE '10s', max(val) FROM host ALIGN '5s';
Error: 1003(Internal), No field named "MAX(host.val)". Valid fields are "MIN(host.val) RANGE 10s FILL NULL", host.ts, host.host.
SELECT min(val) * 2 RANGE '10s' FROM host ALIGN '5s';
Error: 2000(InvalidSyntax), sql parser error: Can't use the RANGE keyword in Expr 2 without function
SELECT 1 RANGE '10s' FILL NULL FROM host ALIGN '1h' FILL NULL;
Error: 2000(InvalidSyntax), sql parser error: Can't use the RANGE keyword in Expr 1 without function
-- 2.2 no align param
SELECT min(val) RANGE '5s' FROM host;
Error: 1003(Internal), Error during planning: Missing argument in range select query
-- 2.3 type mismatch
SELECT covar(ceil(val), floor(val)) RANGE '20s' FROM host ALIGN '10s';
Error: 1003(Internal), Internal error: Unsupported data type Int64 for function ceil. This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug report in our issue tracker
-- 2.4 nest query
SELECT min(max(val) RANGE '20s') RANGE '20s' FROM host ALIGN '10s';
Error: 2000(InvalidSyntax), Range Query: Nest Range Query is not allowed
-- 2.5 wrong Aggregate
SELECT rank() OVER (PARTITION BY host ORDER BY ts DESC) RANGE '10s' FROM host ALIGN '5s';
Error: 2000(InvalidSyntax), Range Query: Window functions is not allowed in Range Query
-- 2.6 invalid fill
SELECT min(val) RANGE '5s', min(val) RANGE '5s' FILL NULL FROM host ALIGN '5s';
Error: 1003(Internal), Schema contains duplicate unqualified field name "MIN(host.val) RANGE 5s FILL NULL"
SELECT min(val) RANGE '5s' FROM host ALIGN '5s' FILL 3.0;
Error: 1003(Internal), Error during planning: 3.0 is not a valid fill option, fail to convert to a const value. { Arrow error: Cast error: Cannot cast string '3.0' to value of Int64 type }
DROP TABLE host;
Affected Rows: 0

View File

@@ -0,0 +1,59 @@
CREATE TABLE host (
ts timestamp(3) time index,
host STRING PRIMARY KEY,
val BIGINT,
);
INSERT INTO TABLE host VALUES
(0, 'host1', 0),
(5000, 'host1', null),
(10000, 'host1', 1),
(15000, 'host1', null),
(20000, 'host1', 2),
(0, 'host2', 3),
(5000, 'host2', null),
(10000, 'host2', 4),
(15000, 'host2', null),
(20000, 'host2', 5);
-- Test Invalid cases
-- 1. error timestamp
SELECT min(val) RANGE 'not_time' FROM host ALIGN '5s';
SELECT min(val) RANGE '5s' FROM host ALIGN 'not_time';
-- 2.1 no range param
SELECT min(val) FROM host ALIGN '5s';
SELECT min(val) RANGE '10s', max(val) FROM host ALIGN '5s';
SELECT min(val) * 2 RANGE '10s' FROM host ALIGN '5s';
SELECT 1 RANGE '10s' FILL NULL FROM host ALIGN '1h' FILL NULL;
-- 2.2 no align param
SELECT min(val) RANGE '5s' FROM host;
-- 2.3 type mismatch
SELECT covar(ceil(val), floor(val)) RANGE '20s' FROM host ALIGN '10s';
-- 2.4 nest query
SELECT min(max(val) RANGE '20s') RANGE '20s' FROM host ALIGN '10s';
-- 2.5 wrong Aggregate
SELECT rank() OVER (PARTITION BY host ORDER BY ts DESC) RANGE '10s' FROM host ALIGN '5s';
-- 2.6 invalid fill
SELECT min(val) RANGE '5s', min(val) RANGE '5s' FILL NULL FROM host ALIGN '5s';
SELECT min(val) RANGE '5s' FROM host ALIGN '5s' FILL 3.0;
DROP TABLE host;

View File

@@ -0,0 +1,112 @@
CREATE TABLE host (
ts timestamp(3) time index,
host STRING PRIMARY KEY,
val BIGINT,
);
Affected Rows: 0
INSERT INTO TABLE host VALUES
(0, 'host1', 0),
(5000, 'host1', null),
(10000, 'host1', 1),
(15000, 'host1', null),
(20000, 'host1', 2),
(0, 'host2', 3),
(5000, 'host2', null),
(10000, 'host2', 4),
(15000, 'host2', null),
(20000, 'host2', 5);
Affected Rows: 10
-- Test Fill
SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts;
+---------------------+-------+----------------------------------+
| ts | host | MIN(host.val) RANGE 5s FILL NULL |
+---------------------+-------+----------------------------------+
| 1970-01-01T00:00:00 | host1 | 0 |
| 1970-01-01T00:00:05 | host1 | |
| 1970-01-01T00:00:10 | host1 | 1 |
| 1970-01-01T00:00:15 | host1 | |
| 1970-01-01T00:00:20 | host1 | 2 |
| 1970-01-01T00:00:00 | host2 | 3 |
| 1970-01-01T00:00:05 | host2 | |
| 1970-01-01T00:00:10 | host2 | 4 |
| 1970-01-01T00:00:15 | host2 | |
| 1970-01-01T00:00:20 | host2 | 5 |
+---------------------+-------+----------------------------------+
SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s' FILL NULL ORDER BY host, ts;
+---------------------+-------+----------------------------------+
| ts | host | MIN(host.val) RANGE 5s FILL NULL |
+---------------------+-------+----------------------------------+
| 1970-01-01T00:00:00 | host1 | 0 |
| 1970-01-01T00:00:05 | host1 | |
| 1970-01-01T00:00:10 | host1 | 1 |
| 1970-01-01T00:00:15 | host1 | |
| 1970-01-01T00:00:20 | host1 | 2 |
| 1970-01-01T00:00:00 | host2 | 3 |
| 1970-01-01T00:00:05 | host2 | |
| 1970-01-01T00:00:10 | host2 | 4 |
| 1970-01-01T00:00:15 | host2 | |
| 1970-01-01T00:00:20 | host2 | 5 |
+---------------------+-------+----------------------------------+
SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL 6 FROM host ALIGN '5s' ORDER BY host, ts;
+---------------------+-------+----------------------------------+-------------------------------+
| ts | host | MIN(host.val) RANGE 5s FILL NULL | MIN(host.val) RANGE 5s FILL 6 |
+---------------------+-------+----------------------------------+-------------------------------+
| 1970-01-01T00:00:00 | host1 | 0 | 0 |
| 1970-01-01T00:00:05 | host1 | | 6 |
| 1970-01-01T00:00:10 | host1 | 1 | 1 |
| 1970-01-01T00:00:15 | host1 | | 6 |
| 1970-01-01T00:00:20 | host1 | 2 | 2 |
| 1970-01-01T00:00:00 | host2 | 3 | 3 |
| 1970-01-01T00:00:05 | host2 | | 6 |
| 1970-01-01T00:00:10 | host2 | 4 | 4 |
| 1970-01-01T00:00:15 | host2 | | 6 |
| 1970-01-01T00:00:20 | host2 | 5 | 5 |
+---------------------+-------+----------------------------------+-------------------------------+
SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL PREV FROM host ALIGN '5s'ORDER BY host, ts;
+---------------------+-------+----------------------------------+----------------------------------+
| ts | host | MIN(host.val) RANGE 5s FILL NULL | MIN(host.val) RANGE 5s FILL PREV |
+---------------------+-------+----------------------------------+----------------------------------+
| 1970-01-01T00:00:00 | host1 | 0 | 0 |
| 1970-01-01T00:00:05 | host1 | | 0 |
| 1970-01-01T00:00:10 | host1 | 1 | 1 |
| 1970-01-01T00:00:15 | host1 | | 1 |
| 1970-01-01T00:00:20 | host1 | 2 | 2 |
| 1970-01-01T00:00:00 | host2 | 3 | 3 |
| 1970-01-01T00:00:05 | host2 | | 3 |
| 1970-01-01T00:00:10 | host2 | 4 | 4 |
| 1970-01-01T00:00:15 | host2 | | 4 |
| 1970-01-01T00:00:20 | host2 | 5 | 5 |
+---------------------+-------+----------------------------------+----------------------------------+
SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL LINEAR FROM host ALIGN '5s' ORDER BY host, ts;
+---------------------+-------+----------------------------------+------------------------------------+
| ts | host | MIN(host.val) RANGE 5s FILL NULL | MIN(host.val) RANGE 5s FILL LINEAR |
+---------------------+-------+----------------------------------+------------------------------------+
| 1970-01-01T00:00:00 | host1 | 0 | 0.0 |
| 1970-01-01T00:00:05 | host1 | | 0.5 |
| 1970-01-01T00:00:10 | host1 | 1 | 1.0 |
| 1970-01-01T00:00:15 | host1 | | 1.5 |
| 1970-01-01T00:00:20 | host1 | 2 | 2.0 |
| 1970-01-01T00:00:00 | host2 | 3 | 3.0 |
| 1970-01-01T00:00:05 | host2 | | 3.5 |
| 1970-01-01T00:00:10 | host2 | 4 | 4.0 |
| 1970-01-01T00:00:15 | host2 | | 4.5 |
| 1970-01-01T00:00:20 | host2 | 5 | 5.0 |
+---------------------+-------+----------------------------------+------------------------------------+
DROP TABLE host;
Affected Rows: 0

View File

@@ -0,0 +1,31 @@
CREATE TABLE host (
ts timestamp(3) time index,
host STRING PRIMARY KEY,
val BIGINT,
);
INSERT INTO TABLE host VALUES
(0, 'host1', 0),
(5000, 'host1', null),
(10000, 'host1', 1),
(15000, 'host1', null),
(20000, 'host1', 2),
(0, 'host2', 3),
(5000, 'host2', null),
(10000, 'host2', 4),
(15000, 'host2', null),
(20000, 'host2', 5);
-- Test Fill
SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts;
SELECT ts, host, min(val) RANGE '5s' FROM host ALIGN '5s' FILL NULL ORDER BY host, ts;
SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL 6 FROM host ALIGN '5s' ORDER BY host, ts;
SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL PREV FROM host ALIGN '5s'ORDER BY host, ts;
SELECT ts, host, min(val) RANGE '5s', min(val) RANGE '5s' FILL LINEAR FROM host ALIGN '5s' ORDER BY host, ts;
DROP TABLE host;

View File

@@ -0,0 +1,51 @@
CREATE TABLE host (
ts timestamp(3) time index,
host STRING PRIMARY KEY,
val BIGINT,
);
Affected Rows: 0
INSERT INTO TABLE host VALUES
(0, 'host1', 0),
(5000, 'host1', null),
(10000, 'host1', 1),
(15000, 'host1', null),
(20000, 'host1', 2),
(0, 'host2', 3),
(5000, 'host2', null),
(10000, 'host2', 4),
(15000, 'host2', null),
(20000, 'host2', 5);
Affected Rows: 10
-- Test range query in nest sql
SELECT ts, host, foo FROM (SELECT ts, host, min(val) RANGE '5s' AS foo FROM host ALIGN '5s') WHERE host = 'host1' ORDER BY host, ts;
+---------------------+-------+-----+
| ts | host | foo |
+---------------------+-------+-----+
| 1970-01-01T00:00:00 | host1 | 0 |
| 1970-01-01T00:00:05 | host1 | |
| 1970-01-01T00:00:10 | host1 | 1 |
| 1970-01-01T00:00:15 | host1 | |
| 1970-01-01T00:00:20 | host1 | 2 |
+---------------------+-------+-----+
SELECT ts, b, min(c) RANGE '5s' FROM (SELECT ts, host AS b, val AS c FROM host WHERE host = 'host1') ALIGN '5s' BY (b) ORDER BY b, ts;
+---------------------+-------+---------------------------+
| ts | b | MIN(c) RANGE 5s FILL NULL |
+---------------------+-------+---------------------------+
| 1970-01-01T00:00:00 | host1 | 0 |
| 1970-01-01T00:00:05 | host1 | |
| 1970-01-01T00:00:10 | host1 | 1 |
| 1970-01-01T00:00:15 | host1 | |
| 1970-01-01T00:00:20 | host1 | 2 |
+---------------------+-------+---------------------------+
DROP TABLE host;
Affected Rows: 0

View File

@@ -0,0 +1,25 @@
CREATE TABLE host (
ts timestamp(3) time index,
host STRING PRIMARY KEY,
val BIGINT,
);
INSERT INTO TABLE host VALUES
(0, 'host1', 0),
(5000, 'host1', null),
(10000, 'host1', 1),
(15000, 'host1', null),
(20000, 'host1', 2),
(0, 'host2', 3),
(5000, 'host2', null),
(10000, 'host2', 4),
(15000, 'host2', null),
(20000, 'host2', 5);
-- Test range query in nest sql
SELECT ts, host, foo FROM (SELECT ts, host, min(val) RANGE '5s' AS foo FROM host ALIGN '5s') WHERE host = 'host1' ORDER BY host, ts;
SELECT ts, b, min(c) RANGE '5s' FROM (SELECT ts, host AS b, val AS c FROM host WHERE host = 'host1') ALIGN '5s' BY (b) ORDER BY b, ts;
DROP TABLE host;

View File

@@ -0,0 +1,44 @@
CREATE TABLE host_sec (
ts timestamp(0) time index,
host STRING PRIMARY KEY,
val DOUBLE,
);
Affected Rows: 0
INSERT INTO TABLE host_sec VALUES
(0, 'host1', 0),
(5, 'host1', null),
(10, 'host1', 1),
(15, 'host1', null),
(20, 'host1', 2),
(0, 'host2', 3),
(5, 'host2', null),
(10, 'host2', 4),
(15, 'host2', null),
(20, 'host2', 5);
Affected Rows: 10
-- Test on Timestamps of different precisions
SELECT ts, host, min(val) RANGE '5s' FROM host_sec ALIGN '5s' ORDER BY host, ts;
+---------------------+-------+--------------------------------------+
| ts | host | MIN(host_sec.val) RANGE 5s FILL NULL |
+---------------------+-------+--------------------------------------+
| 1970-01-01T00:00:00 | host1 | 0.0 |
| 1970-01-01T00:00:05 | host1 | |
| 1970-01-01T00:00:10 | host1 | 1.0 |
| 1970-01-01T00:00:15 | host1 | |
| 1970-01-01T00:00:20 | host1 | 2.0 |
| 1970-01-01T00:00:00 | host2 | 3.0 |
| 1970-01-01T00:00:05 | host2 | |
| 1970-01-01T00:00:10 | host2 | 4.0 |
| 1970-01-01T00:00:15 | host2 | |
| 1970-01-01T00:00:20 | host2 | 5.0 |
+---------------------+-------+--------------------------------------+
DROP TABLE host_sec;
Affected Rows: 0

View File

@@ -0,0 +1,23 @@
CREATE TABLE host_sec (
ts timestamp(0) time index,
host STRING PRIMARY KEY,
val DOUBLE,
);
INSERT INTO TABLE host_sec VALUES
(0, 'host1', 0),
(5, 'host1', null),
(10, 'host1', 1),
(15, 'host1', null),
(20, 'host1', 2),
(0, 'host2', 3),
(5, 'host2', null),
(10, 'host2', 4),
(15, 'host2', null),
(20, 'host2', 5);
-- Test on Timestamps of different precisions
SELECT ts, host, min(val) RANGE '5s' FROM host_sec ALIGN '5s' ORDER BY host, ts;
DROP TABLE host_sec;

View File

@@ -1,324 +0,0 @@
CREATE TABLE host (
ts timestamp(3) time index,
host STRING PRIMARY KEY,
val DOUBLE,
);
Affected Rows: 0
INSERT INTO TABLE host VALUES
(0, 'host1', 0.0),
(5000, 'host1', 1.0),
(10000, 'host1', 2.0),
(15000, 'host1', 3.0),
(20000, 'host1', 4.0),
(25000, 'host1', 5.0),
(30000, 'host1', 6.0),
(35000, 'host1', 7.0),
(40000, 'host1', 8.0),
(0, 'host2', 9.0),
(5000, 'host2', 10.0),
(10000, 'host2', 11.0),
(15000, 'host2', 12.0),
(20000, 'host2', 13.0),
(25000, 'host2', 14.0),
(30000, 'host2', 15.0),
(35000, 'host2', 16.0),
(40000, 'host2', 17.0);
Affected Rows: 18
SELECT ts, host, min(val) RANGE '10s', max(val) RANGE '10s' FROM host ALIGN '5s' ORDER BY host, ts;
+---------------------+-------+---------------+---------------+
| ts | host | MIN(host.val) | MAX(host.val) |
+---------------------+-------+---------------+---------------+
| 1970-01-01T00:00:00 | host1 | 0.0 | 0.0 |
| 1970-01-01T00:00:05 | host1 | 0.0 | 1.0 |
| 1970-01-01T00:00:10 | host1 | 1.0 | 2.0 |
| 1970-01-01T00:00:15 | host1 | 2.0 | 3.0 |
| 1970-01-01T00:00:20 | host1 | 3.0 | 4.0 |
| 1970-01-01T00:00:25 | host1 | 4.0 | 5.0 |
| 1970-01-01T00:00:30 | host1 | 5.0 | 6.0 |
| 1970-01-01T00:00:35 | host1 | 6.0 | 7.0 |
| 1970-01-01T00:00:40 | host1 | 7.0 | 8.0 |
| 1970-01-01T00:00:45 | host1 | 8.0 | 8.0 |
| 1970-01-01T00:00:00 | host2 | 9.0 | 9.0 |
| 1970-01-01T00:00:05 | host2 | 9.0 | 10.0 |
| 1970-01-01T00:00:10 | host2 | 10.0 | 11.0 |
| 1970-01-01T00:00:15 | host2 | 11.0 | 12.0 |
| 1970-01-01T00:00:20 | host2 | 12.0 | 13.0 |
| 1970-01-01T00:00:25 | host2 | 13.0 | 14.0 |
| 1970-01-01T00:00:30 | host2 | 14.0 | 15.0 |
| 1970-01-01T00:00:35 | host2 | 15.0 | 16.0 |
| 1970-01-01T00:00:40 | host2 | 16.0 | 17.0 |
| 1970-01-01T00:00:45 | host2 | 17.0 | 17.0 |
+---------------------+-------+---------------+---------------+
SELECT ts, host, min(val / 2.0)/2 RANGE '10s', max(val / 2.0)/2 RANGE '10s' FROM host ALIGN '5s' ORDER BY host, ts;
+---------------------+-------+---------------------------------------+---------------------------------------+
| ts | host | MIN(host.val / Float64(2)) / Int64(2) | MAX(host.val / Float64(2)) / Int64(2) |
+---------------------+-------+---------------------------------------+---------------------------------------+
| 1970-01-01T00:00:00 | host1 | 0.0 | 0.0 |
| 1970-01-01T00:00:05 | host1 | 0.0 | 0.25 |
| 1970-01-01T00:00:10 | host1 | 0.25 | 0.5 |
| 1970-01-01T00:00:15 | host1 | 0.5 | 0.75 |
| 1970-01-01T00:00:20 | host1 | 0.75 | 1.0 |
| 1970-01-01T00:00:25 | host1 | 1.0 | 1.25 |
| 1970-01-01T00:00:30 | host1 | 1.25 | 1.5 |
| 1970-01-01T00:00:35 | host1 | 1.5 | 1.75 |
| 1970-01-01T00:00:40 | host1 | 1.75 | 2.0 |
| 1970-01-01T00:00:45 | host1 | 2.0 | 2.0 |
| 1970-01-01T00:00:00 | host2 | 2.25 | 2.25 |
| 1970-01-01T00:00:05 | host2 | 2.25 | 2.5 |
| 1970-01-01T00:00:10 | host2 | 2.5 | 2.75 |
| 1970-01-01T00:00:15 | host2 | 2.75 | 3.0 |
| 1970-01-01T00:00:20 | host2 | 3.0 | 3.25 |
| 1970-01-01T00:00:25 | host2 | 3.25 | 3.5 |
| 1970-01-01T00:00:30 | host2 | 3.5 | 3.75 |
| 1970-01-01T00:00:35 | host2 | 3.75 | 4.0 |
| 1970-01-01T00:00:40 | host2 | 4.0 | 4.25 |
| 1970-01-01T00:00:45 | host2 | 4.25 | 4.25 |
+---------------------+-------+---------------------------------------+---------------------------------------+
SELECT ts, covar(val, val) RANGE '10s', host FROM host ALIGN '5s' ORDER BY host, ts;
+---------------------+-------------------------------+-------+
| ts | COVARIANCE(host.val,host.val) | host |
+---------------------+-------------------------------+-------+
| 1970-01-01T00:00:00 | | host1 |
| 1970-01-01T00:00:05 | 0.5 | host1 |
| 1970-01-01T00:00:10 | 0.5 | host1 |
| 1970-01-01T00:00:15 | 0.5 | host1 |
| 1970-01-01T00:00:20 | 0.5 | host1 |
| 1970-01-01T00:00:25 | 0.5 | host1 |
| 1970-01-01T00:00:30 | 0.5 | host1 |
| 1970-01-01T00:00:35 | 0.5 | host1 |
| 1970-01-01T00:00:40 | 0.5 | host1 |
| 1970-01-01T00:00:45 | | host1 |
| 1970-01-01T00:00:00 | | host2 |
| 1970-01-01T00:00:05 | 0.5 | host2 |
| 1970-01-01T00:00:10 | 0.5 | host2 |
| 1970-01-01T00:00:15 | 0.5 | host2 |
| 1970-01-01T00:00:20 | 0.5 | host2 |
| 1970-01-01T00:00:25 | 0.5 | host2 |
| 1970-01-01T00:00:30 | 0.5 | host2 |
| 1970-01-01T00:00:35 | 0.5 | host2 |
| 1970-01-01T00:00:40 | 0.5 | host2 |
| 1970-01-01T00:00:45 | | host2 |
+---------------------+-------------------------------+-------+
SELECT covar(ceil(val), floor(val)) RANGE '10s', ts, host FROM host ALIGN '5s' ORDER BY host, ts;
+--------------------------------------------+---------------------+-------+
| COVARIANCE(ceil(host.val),floor(host.val)) | ts | host |
+--------------------------------------------+---------------------+-------+
| | 1970-01-01T00:00:00 | host1 |
| 0.5 | 1970-01-01T00:00:05 | host1 |
| 0.5 | 1970-01-01T00:00:10 | host1 |
| 0.5 | 1970-01-01T00:00:15 | host1 |
| 0.5 | 1970-01-01T00:00:20 | host1 |
| 0.5 | 1970-01-01T00:00:25 | host1 |
| 0.5 | 1970-01-01T00:00:30 | host1 |
| 0.5 | 1970-01-01T00:00:35 | host1 |
| 0.5 | 1970-01-01T00:00:40 | host1 |
| | 1970-01-01T00:00:45 | host1 |
| | 1970-01-01T00:00:00 | host2 |
| 0.5 | 1970-01-01T00:00:05 | host2 |
| 0.5 | 1970-01-01T00:00:10 | host2 |
| 0.5 | 1970-01-01T00:00:15 | host2 |
| 0.5 | 1970-01-01T00:00:20 | host2 |
| 0.5 | 1970-01-01T00:00:25 | host2 |
| 0.5 | 1970-01-01T00:00:30 | host2 |
| 0.5 | 1970-01-01T00:00:35 | host2 |
| 0.5 | 1970-01-01T00:00:40 | host2 |
| | 1970-01-01T00:00:45 | host2 |
+--------------------------------------------+---------------------+-------+
SELECT ts, host, covar((sin(val) + cos(val))/2.0 + 1.0, 2.0) RANGE '10s' FROM host ALIGN '5s' ORDER BY host, ts;
+---------------------+-------+--------------------------------------------------------------------------------+
| ts | host | COVARIANCE(sin(host.val) + cos(host.val) / Float64(2) + Float64(1),Float64(2)) |
+---------------------+-------+--------------------------------------------------------------------------------+
| 1970-01-01T00:00:00 | host1 | |
| 1970-01-01T00:00:05 | host1 | 0.0 |
| 1970-01-01T00:00:10 | host1 | 0.0 |
| 1970-01-01T00:00:15 | host1 | 0.0 |
| 1970-01-01T00:00:20 | host1 | 0.0 |
| 1970-01-01T00:00:25 | host1 | 0.0 |
| 1970-01-01T00:00:30 | host1 | 0.0 |
| 1970-01-01T00:00:35 | host1 | 0.0 |
| 1970-01-01T00:00:40 | host1 | 0.0 |
| 1970-01-01T00:00:45 | host1 | |
| 1970-01-01T00:00:00 | host2 | |
| 1970-01-01T00:00:05 | host2 | 0.0 |
| 1970-01-01T00:00:10 | host2 | 0.0 |
| 1970-01-01T00:00:15 | host2 | 0.0 |
| 1970-01-01T00:00:20 | host2 | 0.0 |
| 1970-01-01T00:00:25 | host2 | 0.0 |
| 1970-01-01T00:00:30 | host2 | 0.0 |
| 1970-01-01T00:00:35 | host2 | 0.0 |
| 1970-01-01T00:00:40 | host2 | 0.0 |
| 1970-01-01T00:00:45 | host2 | |
+---------------------+-------+--------------------------------------------------------------------------------+
SELECT ts, min(val) RANGE '10s', host, max(val) RANGE '10s' FROM host ALIGN '1000s' ORDER BY host, ts;
+---------------------+---------------+-------+---------------+
| ts | MIN(host.val) | host | MAX(host.val) |
+---------------------+---------------+-------+---------------+
| 1970-01-01T00:00:00 | 0.0 | host1 | 0.0 |
| 1970-01-01T00:00:00 | 9.0 | host2 | 9.0 |
+---------------------+---------------+-------+---------------+
SELECT ts, host, min(val) RANGE '10s', max(val) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts;
+---------------------+-------+---------------+---------------+
| ts | host | MIN(host.val) | MAX(host.val) |
+---------------------+-------+---------------+---------------+
| 1970-01-01T00:00:00 | host1 | 0.0 | 0.0 |
| 1970-01-01T00:00:05 | host1 | 0.0 | 1.0 |
| 1970-01-01T00:00:10 | host1 | 1.0 | 2.0 |
| 1970-01-01T00:00:15 | host1 | 2.0 | 3.0 |
| 1970-01-01T00:00:20 | host1 | 3.0 | 4.0 |
| 1970-01-01T00:00:25 | host1 | 4.0 | 5.0 |
| 1970-01-01T00:00:30 | host1 | 5.0 | 6.0 |
| 1970-01-01T00:00:35 | host1 | 6.0 | 7.0 |
| 1970-01-01T00:00:40 | host1 | 7.0 | 8.0 |
| 1970-01-01T00:00:45 | host1 | 8.0 | |
| 1970-01-01T00:00:00 | host2 | 9.0 | 9.0 |
| 1970-01-01T00:00:05 | host2 | 9.0 | 10.0 |
| 1970-01-01T00:00:10 | host2 | 10.0 | 11.0 |
| 1970-01-01T00:00:15 | host2 | 11.0 | 12.0 |
| 1970-01-01T00:00:20 | host2 | 12.0 | 13.0 |
| 1970-01-01T00:00:25 | host2 | 13.0 | 14.0 |
| 1970-01-01T00:00:30 | host2 | 14.0 | 15.0 |
| 1970-01-01T00:00:35 | host2 | 15.0 | 16.0 |
| 1970-01-01T00:00:40 | host2 | 16.0 | 17.0 |
| 1970-01-01T00:00:45 | host2 | 17.0 | |
+---------------------+-------+---------------+---------------+
SELECT ts, host, (min(val)+max(val))/4 RANGE '10s' FROM host ALIGN '5s' ORDER BY host, ts;
+---------------------+-------+------------------------------------------+
| ts | host | MIN(host.val) + MAX(host.val) / Int64(4) |
+---------------------+-------+------------------------------------------+
| 1970-01-01T00:00:00 | host1 | 0.0 |
| 1970-01-01T00:00:05 | host1 | 0.25 |
| 1970-01-01T00:00:10 | host1 | 0.75 |
| 1970-01-01T00:00:15 | host1 | 1.25 |
| 1970-01-01T00:00:20 | host1 | 1.75 |
| 1970-01-01T00:00:25 | host1 | 2.25 |
| 1970-01-01T00:00:30 | host1 | 2.75 |
| 1970-01-01T00:00:35 | host1 | 3.25 |
| 1970-01-01T00:00:40 | host1 | 3.75 |
| 1970-01-01T00:00:45 | host1 | 4.0 |
| 1970-01-01T00:00:00 | host2 | 4.5 |
| 1970-01-01T00:00:05 | host2 | 4.75 |
| 1970-01-01T00:00:10 | host2 | 5.25 |
| 1970-01-01T00:00:15 | host2 | 5.75 |
| 1970-01-01T00:00:20 | host2 | 6.25 |
| 1970-01-01T00:00:25 | host2 | 6.75 |
| 1970-01-01T00:00:30 | host2 | 7.25 |
| 1970-01-01T00:00:35 | host2 | 7.75 |
| 1970-01-01T00:00:40 | host2 | 8.25 |
| 1970-01-01T00:00:45 | host2 | 8.5 |
+---------------------+-------+------------------------------------------+
SELECT ts, host, foo FROM (SELECT ts, host, (min(val)+max(val))/4 RANGE '10s' AS foo FROM host ALIGN '5s' ORDER BY host, ts) WHERE foo > 5 ORDER BY host, ts;
+---------------------+-------+------+
| ts | host | foo |
+---------------------+-------+------+
| 1970-01-01T00:00:10 | host2 | 5.25 |
| 1970-01-01T00:00:15 | host2 | 5.75 |
| 1970-01-01T00:00:20 | host2 | 6.25 |
| 1970-01-01T00:00:25 | host2 | 6.75 |
| 1970-01-01T00:00:30 | host2 | 7.25 |
| 1970-01-01T00:00:35 | host2 | 7.75 |
| 1970-01-01T00:00:40 | host2 | 8.25 |
| 1970-01-01T00:00:45 | host2 | 8.5 |
+---------------------+-------+------+
SELECT ts, b, (min(c)+max(c))/4 RANGE '10s' FROM (SELECT ts, host AS b, val AS c FROM host WHERE val > 8.0) ALIGN '5s' BY (b) ORDER BY b, ts;
+---------------------+-------+----------------------------+
| ts | b | MIN(c) + MAX(c) / Int64(4) |
+---------------------+-------+----------------------------+
| 1970-01-01T00:00:00 | host2 | 4.5 |
| 1970-01-01T00:00:05 | host2 | 4.75 |
| 1970-01-01T00:00:10 | host2 | 5.25 |
| 1970-01-01T00:00:15 | host2 | 5.75 |
| 1970-01-01T00:00:20 | host2 | 6.25 |
| 1970-01-01T00:00:25 | host2 | 6.75 |
| 1970-01-01T00:00:30 | host2 | 7.25 |
| 1970-01-01T00:00:35 | host2 | 7.75 |
| 1970-01-01T00:00:40 | host2 | 8.25 |
| 1970-01-01T00:00:45 | host2 | 8.5 |
+---------------------+-------+----------------------------+
-- Test Invalid cases
-- 1. error timestamp
SELECT min(val) RANGE 'not_time' FROM host ALIGN '5s';
Error: 2000(InvalidSyntax), sql parser error: not a valid duration string: not_time
SELECT min(val) RANGE '5s' FROM host ALIGN 'not_time';
Error: 2000(InvalidSyntax), sql parser error: not a valid duration string: not_time
-- 2.1 no range param
SELECT min(val) FROM host ALIGN '5s';
Error: 2000(InvalidSyntax), sql parser error: RANGE argument not found in min(val)
SELECT min(val) RANGE '10s', max(val) FROM host ALIGN '5s';
Error: 2000(InvalidSyntax), sql parser error: RANGE argument not found in max(val)
-- 2.2 no align param
SELECT min(val) RANGE '5s' FROM host;
Error: 1003(Internal), Error during planning: Illegal argument in range select query
DROP TABLE host;
Affected Rows: 0
CREATE TABLE host_sec (
ts timestamp(0) time index,
host STRING PRIMARY KEY,
val DOUBLE,
);
Affected Rows: 0
INSERT INTO TABLE host_sec VALUES
(0, 'host1', 0.0),
(5, 'host1', 1.0),
(10, 'host1', 2.0),
(15, 'host1', 3.0),
(20, 'host1', 4.0),
(25, 'host1', 5.0),
(30, 'host1', 6.0),
(35, 'host1', 7.0),
(40, 'host1', 8.0),
(0, 'host2', 9.0),
(5, 'host2', 10.0),
(10, 'host2', 11.0),
(15, 'host2', 12.0),
(20, 'host2', 13.0),
(25, 'host2', 14.0),
(30, 'host2', 15.0),
(35, 'host2', 16.0),
(40, 'host2', 17.0);
Affected Rows: 18
-- TODO(ruihang): This query returns incorrect result.
-- SELECT ts, host, min(val) RANGE '10s', max(val) RANGE '10s' FROM host_sec ALIGN '5s' ORDER BY host, ts;
DROP TABLE host_sec;
Affected Rows: 0

View File

@@ -1,96 +0,0 @@
CREATE TABLE host (
ts timestamp(3) time index,
host STRING PRIMARY KEY,
val DOUBLE,
);
INSERT INTO TABLE host VALUES
(0, 'host1', 0.0),
(5000, 'host1', 1.0),
(10000, 'host1', 2.0),
(15000, 'host1', 3.0),
(20000, 'host1', 4.0),
(25000, 'host1', 5.0),
(30000, 'host1', 6.0),
(35000, 'host1', 7.0),
(40000, 'host1', 8.0),
(0, 'host2', 9.0),
(5000, 'host2', 10.0),
(10000, 'host2', 11.0),
(15000, 'host2', 12.0),
(20000, 'host2', 13.0),
(25000, 'host2', 14.0),
(30000, 'host2', 15.0),
(35000, 'host2', 16.0),
(40000, 'host2', 17.0);
SELECT ts, host, min(val) RANGE '10s', max(val) RANGE '10s' FROM host ALIGN '5s' ORDER BY host, ts;
SELECT ts, host, min(val / 2.0)/2 RANGE '10s', max(val / 2.0)/2 RANGE '10s' FROM host ALIGN '5s' ORDER BY host, ts;
SELECT ts, covar(val, val) RANGE '10s', host FROM host ALIGN '5s' ORDER BY host, ts;
SELECT covar(ceil(val), floor(val)) RANGE '10s', ts, host FROM host ALIGN '5s' ORDER BY host, ts;
SELECT ts, host, covar((sin(val) + cos(val))/2.0 + 1.0, 2.0) RANGE '10s' FROM host ALIGN '5s' ORDER BY host, ts;
SELECT ts, min(val) RANGE '10s', host, max(val) RANGE '10s' FROM host ALIGN '1000s' ORDER BY host, ts;
SELECT ts, host, min(val) RANGE '10s', max(val) RANGE '5s' FROM host ALIGN '5s' ORDER BY host, ts;
SELECT ts, host, (min(val)+max(val))/4 RANGE '10s' FROM host ALIGN '5s' ORDER BY host, ts;
SELECT ts, host, foo FROM (SELECT ts, host, (min(val)+max(val))/4 RANGE '10s' AS foo FROM host ALIGN '5s' ORDER BY host, ts) WHERE foo > 5 ORDER BY host, ts;
SELECT ts, b, (min(c)+max(c))/4 RANGE '10s' FROM (SELECT ts, host AS b, val AS c FROM host WHERE val > 8.0) ALIGN '5s' BY (b) ORDER BY b, ts;
-- Test Invalid cases
-- 1. error timestamp
SELECT min(val) RANGE 'not_time' FROM host ALIGN '5s';
SELECT min(val) RANGE '5s' FROM host ALIGN 'not_time';
-- 2.1 no range param
SELECT min(val) FROM host ALIGN '5s';
SELECT min(val) RANGE '10s', max(val) FROM host ALIGN '5s';
-- 2.2 no align param
SELECT min(val) RANGE '5s' FROM host;
DROP TABLE host;
CREATE TABLE host_sec (
ts timestamp(0) time index,
host STRING PRIMARY KEY,
val DOUBLE,
);
INSERT INTO TABLE host_sec VALUES
(0, 'host1', 0.0),
(5, 'host1', 1.0),
(10, 'host1', 2.0),
(15, 'host1', 3.0),
(20, 'host1', 4.0),
(25, 'host1', 5.0),
(30, 'host1', 6.0),
(35, 'host1', 7.0),
(40, 'host1', 8.0),
(0, 'host2', 9.0),
(5, 'host2', 10.0),
(10, 'host2', 11.0),
(15, 'host2', 12.0),
(20, 'host2', 13.0),
(25, 'host2', 14.0),
(30, 'host2', 15.0),
(35, 'host2', 16.0),
(40, 'host2', 17.0);
-- TODO(ruihang): This query returns incorrect result.
-- SELECT ts, host, min(val) RANGE '10s', max(val) RANGE '10s' FROM host_sec ALIGN '5s' ORDER BY host, ts;
DROP TABLE host_sec;