test: add fuzz test for create table (#3441)

* feat: add create table fuzz test

* chore: add ci cfg for fuzz tests

* refactor: remove redundant nightly config

* chore: run fuzz test in debug mode

* chore: use ubuntu-latest

* fix: close connection

* chore: add cache in fuzz test ci

* chore: apply suggestion from CR

* chore: apply suggestion from CR

* chore: refactor the fuzz test action

tests-fuzz/Cargo.toml

@@ -7,15 +7,22 @@ license.workspace = true
 [lints]
 workspace = true
 
+[package.metadata]
+cargo-fuzz = true
+
 [dependencies]
+arbitrary = { version = "1.3.0", features = ["derive"] }
 async-trait = { workspace = true }
 common-error = { workspace = true }
 common-macro = { workspace = true }
 common-query = { workspace = true }
+common-runtime = { workspace = true }
 common-telemetry = { workspace = true }
 datatypes = { workspace = true }
 derive_builder = { workspace = true }
+dotenv = "0.15"
 lazy_static = { workspace = true }
+libfuzzer-sys = "0.4"
 partition = { workspace = true }
 rand = { workspace = true }
 rand_chacha = "0.3.1"
@@ -24,6 +31,12 @@ serde_json = { workspace = true }
 snafu = { workspace = true }
 sql = { workspace = true }
 sqlparser.workspace = true
+sqlx = { version = "0.6", features = [
+    "runtime-tokio-rustls",
+    "mysql",
+    "postgres",
+    "chrono",
+] }
 
 [dev-dependencies]
 dotenv = "0.15"
@@ -34,3 +47,10 @@ sqlx = { version = "0.6", features = [
     "chrono",
 ] }
 tokio = { workspace = true }
+
+[[bin]]
+name = "fuzz_create_table"
+path = "targets/fuzz_create_table.rs"
+test = false
+bench = false
+doc = false

tests-fuzz/README.md

@@ -0,0 +1,41 @@
+# Fuzz Test for GreptimeDB
+
+## Setup
+1. Install the [fuzz](https://rust-fuzz.github.io/book/cargo-fuzz/setup.html) cli first.
+```bash
+cargo install cargo-fuzz
+```
+
+2. Start GreptimeDB
+3. Copy the `.env.example`, which is at project root, to `.env` and change the values on need.
+
+## Run
+1. List all fuzz targets
+```bash
+cargo fuzz list --fuzz-dir tests-fuzz
+```
+
+2. Run a fuzz target.
+```bash
+cargo fuzz run fuzz_create_table --fuzz-dir tests-fuzz
+```
+
+## Crash Reproduction
+If you want to reproduce a crash, you first need to obtain the Base64 encoded code, which usually appears at the end of a crash report, and store it in a file.
+
+Alternatively, if you already have the crash file, you can skip this step.
+
+```bash
+echo "Base64" > .crash
+```
+Print the `std::fmt::Debug` output for an input.
+
+```bash
+cargo fuzz fmt fuzz_target .crash --fuzz-dir tests-fuzz  
+```
+Rerun the fuzz test with the input.
+
+```bash
+cargo fuzz run fuzz_target .crash --fuzz-dir tests-fuzz
+```
+For more details, visit [cargo fuzz](https://rust-fuzz.github.io/book/cargo-fuzz/tutorial.html) or run the command `cargo fuzz --help`.

tests-fuzz/src/error.rs

@@ -38,4 +38,12 @@ pub enum Error {
 
     #[snafu(display("No droppable columns"))]
     DroppableColumns { location: Location },
+
+    #[snafu(display("Failed to execute query: {}", sql))]
+    ExecuteQuery {
+        sql: String,
+        #[snafu(source)]
+        error: sqlx::error::Error,
+        location: Location,
+    },
 }

tests-fuzz/src/generator.rs

@@ -57,7 +57,15 @@ macro_rules! impl_random {
     ($type: ident, $value:ident, $values: ident) => {
         impl<R: Rng> Random<$type, R> for $value {
             fn choose(&self, rng: &mut R, amount: usize) -> Vec<$type> {
-                $values.choose_multiple(rng, amount).cloned().collect()
+                // Collects the elements in deterministic order first.
+                let mut result = std::collections::BTreeSet::new();
+                while result.len() != amount {
+                    result.insert($values.choose(rng).unwrap().clone());
+                }
+                let mut result = result.into_iter().collect::<Vec<_>>();
+                // Shuffles the result slice.
+                result.shuffle(rng);
+                result
             }
         }
     };

tests-fuzz/src/generator/alter_expr.rs

@@ -155,7 +155,7 @@ mod tests {
             .generate(&mut rng)
             .unwrap();
         let serialized = serde_json::to_string(&expr).unwrap();
-        let expected = r#"{"table_name":"DigNissIMOS","alter_options":{"AddColumn":{"column":{"name":"sit","column_type":{"Boolean":null},"options":["PrimaryKey"]},"location":null}}}"#;
+        let expected = r#"{"table_name":"animI","alter_options":{"AddColumn":{"column":{"name":"velit","column_type":{"Int32":{}},"options":[{"DefaultValue":{"Int32":853246610}}]},"location":null}}}"#;
         assert_eq!(expected, serialized);
 
         let expr = AlterExprRenameGeneratorBuilder::default()
@@ -165,7 +165,8 @@ mod tests {
             .generate(&mut rng)
             .unwrap();
         let serialized = serde_json::to_string(&expr).unwrap();
-        let expected = r#"{"table_name":"DigNissIMOS","alter_options":{"RenameTable":{"new_table_name":"excepturi"}}}"#;
+        let expected =
+            r#"{"table_name":"animI","alter_options":{"RenameTable":{"new_table_name":"iure"}}}"#;
         assert_eq!(expected, serialized);
 
         let expr = AlterExprDropColumnGeneratorBuilder::default()
@@ -175,8 +176,7 @@ mod tests {
             .generate(&mut rng)
             .unwrap();
         let serialized = serde_json::to_string(&expr).unwrap();
-        let expected =
-            r#"{"table_name":"DigNissIMOS","alter_options":{"DropColumn":{"name":"INVentORE"}}}"#;
+        let expected = r#"{"table_name":"animI","alter_options":{"DropColumn":{"name":"toTAm"}}}"#;
         assert_eq!(expected, serialized);
     }
 }

tests-fuzz/src/generator/create_expr.rs

@@ -230,7 +230,7 @@ mod tests {
             .unwrap();
 
         let serialized = serde_json::to_string(&expr).unwrap();
-        let expected = r#"{"table_name":"iN","columns":[{"name":"CUlpa","column_type":{"Int16":{}},"options":["PrimaryKey","NotNull"]},{"name":"dEBiTiS","column_type":{"Timestamp":{"Second":null}},"options":["TimeIndex"]},{"name":"HArum","column_type":{"Int16":{}},"options":["NotNull"]},{"name":"NObIS","column_type":{"Int32":{}},"options":["PrimaryKey"]},{"name":"IMPEDiT","column_type":{"Int16":{}},"options":[{"DefaultValue":{"Int16":-25151}}]},{"name":"bLanDITIis","column_type":{"Boolean":null},"options":[{"DefaultValue":{"Boolean":true}}]},{"name":"Dolores","column_type":{"Float32":{}},"options":["PrimaryKey"]},{"name":"eSt","column_type":{"Float32":{}},"options":[{"DefaultValue":{"Float32":0.9152612}}]},{"name":"INVentORE","column_type":{"Int64":{}},"options":["PrimaryKey"]},{"name":"aDIpiSci","column_type":{"Float64":{}},"options":["Null"]}],"if_not_exists":true,"partition":{"partition_columns":["CUlpa"],"partition_bounds":[{"Value":{"Int16":15966}},{"Value":{"Int16":31925}},"MaxValue"]},"engine":"mito2","options":{},"primary_keys":[6,0,8,3]}"#;
+        let expected = r#"{"table_name":"tEmporIbUS","columns":[{"name":"IMpEdIT","column_type":{"String":null},"options":["PrimaryKey","NotNull"]},{"name":"natuS","column_type":{"Timestamp":{"Nanosecond":null}},"options":["TimeIndex"]},{"name":"ADIPisCI","column_type":{"Int16":{}},"options":[{"DefaultValue":{"Int16":4864}}]},{"name":"EXpEdita","column_type":{"Int64":{}},"options":["PrimaryKey"]},{"name":"cUlpA","column_type":{"Float64":{}},"options":["NotNull"]},{"name":"MOLeStIAs","column_type":{"Boolean":null},"options":["Null"]},{"name":"cUmquE","column_type":{"Float32":{}},"options":[{"DefaultValue":{"Float32":0.21569687}}]},{"name":"toTAm","column_type":{"Float64":{}},"options":["NotNull"]},{"name":"deBitIs","column_type":{"Float32":{}},"options":["Null"]},{"name":"QUi","column_type":{"Int64":{}},"options":["Null"]}],"if_not_exists":true,"partition":{"partition_columns":["IMpEdIT"],"partition_bounds":[{"Value":{"String":"򟘲"}},{"Value":{"String":"򴥫"}},"MaxValue"]},"engine":"mito2","options":{},"primary_keys":[0,3]}"#;
         assert_eq!(expected, serialized);
     }
 }

tests-fuzz/src/lib.rs

@@ -21,6 +21,7 @@ pub mod fake;
 pub mod generator;
 pub mod ir;
 pub mod translator;
+pub mod utils;
 
 #[cfg(test)]
 pub mod test_utils;

tests-fuzz/src/utils.rs

@@ -0,0 +1,42 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::env;
+
+use common_telemetry::info;
+use sqlx::mysql::MySqlPoolOptions;
+use sqlx::{MySql, Pool};
+
+pub struct Connections {
+    pub mysql: Option<Pool<MySql>>,
+}
+
+const GT_MYSQL_ADDR: &str = "GT_MYSQL_ADDR";
+
+pub async fn init_greptime_connections() -> Connections {
+    let _ = dotenv::dotenv();
+    let mysql = if let Ok(addr) = env::var(GT_MYSQL_ADDR) {
+        Some(
+            MySqlPoolOptions::new()
+                .connect(&format!("mysql://{addr}/public"))
+                .await
+                .unwrap(),
+        )
+    } else {
+        info!("GT_MYSQL_ADDR is empty, ignores test");
+        None
+    };
+
+    Connections { mysql }
+}

tests-fuzz/targets/fuzz_create_table.rs

@@ -0,0 +1,108 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#![no_main]
+
+use common_telemetry::info;
+use libfuzzer_sys::arbitrary::{Arbitrary, Unstructured};
+use libfuzzer_sys::fuzz_target;
+use rand::{Rng, SeedableRng};
+use rand_chacha::ChaChaRng;
+use snafu::ResultExt;
+use sqlx::{MySql, Pool};
+use tests_fuzz::error::{self, Result};
+use tests_fuzz::fake::{
+    merge_two_word_map_fn, random_capitalize_map, uppercase_and_keyword_backtick_map,
+    MappedGenerator, WordGenerator,
+};
+use tests_fuzz::generator::create_expr::CreateTableExprGeneratorBuilder;
+use tests_fuzz::generator::Generator;
+use tests_fuzz::ir::CreateTableExpr;
+use tests_fuzz::translator::mysql::create_expr::CreateTableExprTranslator;
+use tests_fuzz::translator::DslTranslator;
+use tests_fuzz::utils::{init_greptime_connections, Connections};
+
+struct FuzzContext {
+    greptime: Pool<MySql>,
+}
+
+impl FuzzContext {
+    async fn close(self) {
+        self.greptime.close().await;
+    }
+}
+
+#[derive(Clone, Debug)]
+struct FuzzInput {
+    seed: u64,
+    columns: usize,
+}
+
+impl Arbitrary<'_> for FuzzInput {
+    fn arbitrary(u: &mut Unstructured<'_>) -> arbitrary::Result<Self> {
+        let seed = u.int_in_range(u64::MIN..=u64::MAX)?;
+        let columns = u.int_in_range(2..=10)?;
+        Ok(FuzzInput { columns, seed })
+    }
+}
+
+fn generate_expr(input: FuzzInput) -> Result<CreateTableExpr> {
+    let mut rng = ChaChaRng::seed_from_u64(input.seed);
+    let create_table_generator = CreateTableExprGeneratorBuilder::default()
+        .name_generator(Box::new(MappedGenerator::new(
+            WordGenerator,
+            merge_two_word_map_fn(random_capitalize_map, uppercase_and_keyword_backtick_map),
+        )))
+        .columns(rng.gen_range(1..input.columns))
+        .engine("mito")
+        .build()
+        .unwrap();
+    create_table_generator.generate(&mut rng)
+}
+
+async fn execute_create_table(ctx: FuzzContext, input: FuzzInput) -> Result<()> {
+    info!("input: {input:?}");
+    let expr = generate_expr(input)?;
+    let translator = CreateTableExprTranslator;
+    let sql = translator.translate(&expr)?;
+    let result = sqlx::query(&sql)
+        .execute(&ctx.greptime)
+        .await
+        .context(error::ExecuteQuerySnafu { sql: &sql })?;
+    info!("Create table: {sql}, result: {result:?}");
+
+    // Cleans up
+    let sql = format!("DROP TABLE {}", expr.table_name);
+    let result = sqlx::query(&sql)
+        .execute(&ctx.greptime)
+        .await
+        .context(error::ExecuteQuerySnafu { sql })?;
+    info!("Drop table: {}, result: {result:?}", expr.table_name);
+    ctx.close().await;
+
+    Ok(())
+}
+
+fuzz_target!(|input: FuzzInput| {
+    common_telemetry::init_default_ut_logging();
+    common_runtime::block_on_write(async {
+        let Connections { mysql } = init_greptime_connections().await;
+        let ctx = FuzzContext {
+            greptime: mysql.expect("mysql connection init must be succeed"),
+        };
+        execute_create_table(ctx, input)
+            .await
+            .unwrap_or_else(|err| panic!("fuzz test must be succeed: {err:?}"));
+    })
+});