rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-23 00:10:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: adapt for cuckoo

Browse Source
This commit is contained in:
WenyXu
2024-04-16 14:16:37 +00:00
parent d12379106e
commit 7e1eed4b18
5 changed files with 185 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
tests-fuzz/src/data/lorem_words
View File

@@ -97,7 +97,6 @@ eum
iure
reprehenderit
qui
in
ea
voluptate
velit
@@ -138,10 +137,8 @@ unde
omnis
iste
natus
error
similique
sunt
in
culpa
qui
officia
@@ -210,7 +207,6 @@ quo
voluptas
nulla
pariatur
at
vero
eos
et

7
tests-fuzz/src/generator/alter_expr.rs
View File

@@ -27,7 +27,8 @@ use crate::generator::{ColumnOptionGenerator, ConcreteDataTypeGenerator, Generat
use crate::ir::alter_expr::{AlterTableExpr, AlterTableOperation};
use crate::ir::create_expr::ColumnOption;
use crate::ir::{
droppable_columns, generate_columns, generate_random_value, ColumnTypeGenerator, Ident,
droppable_columns, generate_columns, generate_random_value, generate_random_value_abs,
ColumnTypeGenerator, Ident,
};
fn add_column_options_generator<R: Rng>(
@@ -41,7 +42,7 @@ fn add_column_options_generator<R: Rng>(
match idx {
0 => vec![ColumnOption::Null],
1 => {
vec![ColumnOption::DefaultValue(generate_random_value(
vec![ColumnOption::DefaultValue(generate_random_value_abs(
rng,
column_type,
None,
@@ -50,7 +51,7 @@ fn add_column_options_generator<R: Rng>(
2 => {
vec![
ColumnOption::PrimaryKey,
ColumnOption::DefaultValue(generate_random_value(rng, column_type, None)),
ColumnOption::DefaultValue(generate_random_value_abs(rng, column_type, None)),
]
}
_ => unreachable!(),

57
tests-fuzz/src/ir.rs
View File

@@ -22,7 +22,8 @@ pub(crate) mod select_expr;
use core::fmt;
pub use alter_expr::AlterTableExpr;
use common_time::{Date, DateTime, Timestamp};
use common_time::timestamp::TimeUnit;
use common_time::{Date, DateTime, Interval, Timestamp};
pub use create_expr::{CreateDatabaseExpr, CreateTableExpr};
use datatypes::data_type::ConcreteDataType;
use datatypes::types::TimestampType;
@@ -102,29 +103,65 @@ pub fn generate_random_value<R: Rng>(
}
}
pub fn generate_random_value_abs<R: Rng>(
rng: &mut R,
datatype: &ConcreteDataType,
random_str: Option<&dyn Random<Ident, R>>,
) -> Value {
match datatype {
&ConcreteDataType::Boolean(_) => Value::from(rng.gen::<bool>()),
ConcreteDataType::Int16(_) => Value::from(i16::abs(rng.gen::<i16>())),
ConcreteDataType::Int32(_) => Value::from(i32::abs(rng.gen::<i32>())),
ConcreteDataType::Int64(_) => Value::from(i64::abs(rng.gen::<i64>())),
ConcreteDataType::Float32(_) => Value::from(f32::abs(rng.gen::<f32>())),
ConcreteDataType::Float64(_) => Value::from(f64::abs(rng.gen::<f64>())),
ConcreteDataType::String(_) => match random_str {
Some(random) => Value::from(random.gen(rng).value),
None => Value::from(rng.gen::<char>().to_string()),
},
ConcreteDataType::Date(_) => generate_random_date(rng),
ConcreteDataType::DateTime(_) => generate_random_datetime(rng),
&ConcreteDataType::Timestamp(ts_type) => generate_random_timestamp(rng, ts_type),
_ => unimplemented!("unsupported type: {datatype}"),
}
}
fn generate_random_timestamp<R: Rng>(rng: &mut R, ts_type: TimestampType) -> Value {
let v = match ts_type {
TimestampType::Second(_) => {
let min = i64::from(Timestamp::MIN_SECOND);
let max = i64::from(Timestamp::MAX_SECOND);
let now = Timestamp::current_time(TimeUnit::Second);
let min = now.sub_interval(Interval::from_year_month(12)).unwrap();
let max = now.add_interval(Interval::from_year_month(12)).unwrap();
let min = i64::from(min);
let max = i64::from(max);
let value = rng.gen_range(min..=max);
Timestamp::new_second(value)
}
TimestampType::Millisecond(_) => {
let min = i64::from(Timestamp::MIN_MILLISECOND);
let max = i64::from(Timestamp::MAX_MILLISECOND);
let now = Timestamp::current_time(TimeUnit::Millisecond);
let min = now.sub_interval(Interval::from_year_month(12)).unwrap();
let max = now.add_interval(Interval::from_year_month(12)).unwrap();
let min = i64::from(min);
let max = i64::from(max);
let value = rng.gen_range(min..=max);
Timestamp::new_millisecond(value)
}
TimestampType::Microsecond(_) => {
let min = i64::from(Timestamp::MIN_MICROSECOND);
let max = i64::from(Timestamp::MAX_MICROSECOND);
let now = Timestamp::current_time(TimeUnit::Microsecond);
let min = now.sub_interval(Interval::from_year_month(12)).unwrap();
let max = now.add_interval(Interval::from_year_month(12)).unwrap();
let min = i64::from(min);
let max = i64::from(max);
let value = rng.gen_range(min..=max);
Timestamp::new_microsecond(value)
}
TimestampType::Nanosecond(_) => {
let min = i64::from(Timestamp::MIN_NANOSECOND);
let max = i64::from(Timestamp::MAX_NANOSECOND);
let now = Timestamp::current_time(TimeUnit::Nanosecond);
let min = now.sub_interval(Interval::from_year_month(12)).unwrap();
let max = now.add_interval(Interval::from_year_month(12)).unwrap();
let min = i64::from(min);
let max = i64::from(max);
let value = rng.gen_range(min..=max);
Timestamp::new_nanosecond(value)
}
@@ -278,7 +315,7 @@ pub fn column_options_generator<R: Rng>(
match option_idx {
0 => vec![ColumnOption::Null],
1 => vec![ColumnOption::NotNull],
2 => vec![ColumnOption::DefaultValue(generate_random_value(
2 => vec![ColumnOption::DefaultValue(generate_random_value_abs(
rng,
column_type,
None,

135
tests-fuzz/src/validator/column.rs
View File

@@ -16,7 +16,9 @@ use common_telemetry::debug;
use datatypes::data_type::DataType;
use snafu::{ensure, ResultExt};
use sqlx::database::HasArguments;
use sqlx::{ColumnIndex, Database, Decode, Encode, Executor, IntoArguments, Type};
use sqlx::{
ColumnIndex, Database, Decode, Encode, Executor, IntoArguments, MySql, Pool, Row, Type,
};
use crate::error::{self, Result};
use crate::ir::create_expr::ColumnOption;
@@ -24,12 +26,15 @@ use crate::ir::{Column, Ident};
#[derive(Debug, sqlx::FromRow)]
pub struct ColumnEntry {
pub table_schema: String,
pub table_name: String,
#[sqlx(rename = "Field")]
pub column_name: String,
#[sqlx(rename = "Type")]
pub data_type: String,
#[sqlx(rename = "Semantic Type")]
pub semantic_type: String,
#[sqlx(rename = "Default")]
pub column_default: Option<String>,
#[sqlx(rename = "Null")]
pub is_nullable: String,
}
@@ -45,6 +50,7 @@ enum SemanticType {
fn semantic_type(str: &str) -> Option<SemanticType> {
match str {
"TIME INDEX" => Some(SemanticType::Timestamp),
"TIMESTAMP" => Some(SemanticType::Timestamp),
"FIELD" => Some(SemanticType::Field),
"TAG" => Some(SemanticType::Tag),
@@ -127,43 +133,43 @@ impl PartialEq<Column> for ColumnEntry {
return false;
}
}
//TODO: Checks `semantic_type`
match semantic_type(&self.semantic_type) {
Some(SemanticType::Tag) => {
if !other
.options
.iter()
.any(|opt| matches!(opt, ColumnOption::PrimaryKey))
{
debug!("ColumnOption::PrimaryKey is not found");
return false;
}
}
Some(SemanticType::Field) => {
if other
.options
.iter()
.any(|opt| matches!(opt, ColumnOption::PrimaryKey | ColumnOption::TimeIndex))
{
debug!("unexpected ColumnOption::PrimaryKey or ColumnOption::TimeIndex");
return false;
}
}
Some(SemanticType::Timestamp) => {
if !other
.options
.iter()
.any(|opt| matches!(opt, ColumnOption::TimeIndex))
{
debug!("ColumnOption::TimeIndex is not found");
return false;
}
}
None => {
debug!("unknown semantic type: {}", self.semantic_type);
return false;
}
};
// //TODO: Checks `semantic_type`
// match semantic_type(&self.semantic_type) {
// Some(SemanticType::Tag) => {
// if !other
// .options
// .iter()
// .any(|opt| matches!(opt, ColumnOption::PrimaryKey))
// {
// debug!("ColumnOption::PrimaryKey is not found");
// return false;
// }
// }
// Some(SemanticType::Field) => {
// if other
// .options
// .iter()
// .any(|opt| matches!(opt, ColumnOption::PrimaryKey | ColumnOption::TimeIndex))
// {
// debug!("unexpected ColumnOption::PrimaryKey or ColumnOption::TimeIndex");
// return false;
// }
// }
// Some(SemanticType::Timestamp) => {
// if !other
// .options
// .iter()
// .any(|opt| matches!(opt, ColumnOption::TimeIndex))
// {
// debug!("ColumnOption::TimeIndex is not found");
// return false;
// }
// }
// None => {
// debug!("unknown semantic type: {}", self.semantic_type);
// return false;
// }
// };
true
}
@@ -211,13 +217,44 @@ where
for<'c> String: Encode<'c, DB> + Type<DB>,
for<'c> &'c str: ColumnIndex<<DB as Database>::Row>,
{
let sql = "SELECT table_schema, table_name, column_name, greptime_data_type as data_type, semantic_type, column_default, is_nullable FROM information_schema.columns WHERE table_schema = ? AND table_name = ?";
sqlx::query_as::<_, ColumnEntry>(sql)
.bind(schema_name.value.to_string())
.bind(table_name.value.to_string())
.fetch_all(e)
// let sql = format!("DESC TABLE {table_name}");
// let rows = sqlx::query(&sql)
// .fetch_all(e)
// .await
// .context(error::ExecuteQuerySnafu { sql })?;
todo!()
}
pub async fn fetch_columns_via_mysql(
db: &Pool<MySql>,
_schema_name: Ident,
table_name: Ident,
) -> Result<Vec<ColumnEntry>> {
let sql = format!("DESC TABLE {table_name}");
let rows = sqlx::query(&sql)
.fetch_all(db)
.await
.context(error::ExecuteQuerySnafu { sql })
.context(error::ExecuteQuerySnafu { sql })?;
Ok(rows
.into_iter()
.map(|row| {
let default_value: String = row.get(3);
let column_default = if default_value.is_empty() {
None
} else {
Some(default_value)
};
ColumnEntry {
column_name: row.get(0),
data_type: row.get(1),
is_nullable: row.get(2),
column_default,
semantic_type: row.get(4),
}
})
.collect::<Vec<_>>())
}
#[cfg(test)]
@@ -233,8 +270,6 @@ mod tests {
fn test_column_eq() {
common_telemetry::init_default_ut_logging();
let column_entry = ColumnEntry {
table_schema: String::new(),
table_name: String::new(),
column_name: "test".to_string(),
data_type: ConcreteDataType::int8_datatype().name(),
semantic_type: "FIELD".to_string(),
@@ -257,8 +292,6 @@ mod tests {
assert!(column_entry == column);
// With default value
let column_entry = ColumnEntry {
table_schema: String::new(),
table_name: String::new(),
column_name: "test".to_string(),
data_type: ConcreteDataType::int8_datatype().to_string(),
semantic_type: "FIELD".to_string(),
@@ -273,8 +306,6 @@ mod tests {
assert!(column_entry == column);
// With default function
let column_entry = ColumnEntry {
table_schema: String::new(),
table_name: String::new(),
column_name: "test".to_string(),
data_type: ConcreteDataType::int8_datatype().to_string(),
semantic_type: "FIELD".to_string(),

65
tests-fuzz/targets/fuzz_alter_table.rs
View File

@@ -21,8 +21,8 @@ use common_telemetry::info;
use libfuzzer_sys::fuzz_target;
use rand::{Rng, SeedableRng};
use rand_chacha::ChaChaRng;
use snafu::ResultExt;
use sqlx::{MySql, Pool};
use snafu::{ensure, ResultExt};
use sqlx::{Executor, MySql, Pool};
use tests_fuzz::context::{TableContext, TableContextRef};
use tests_fuzz::error::{self, Result};
use tests_fuzz::fake::{
@@ -34,10 +34,12 @@ use tests_fuzz::generator::alter_expr::{
AlterExprRenameGeneratorBuilder,
};
use tests_fuzz::generator::create_expr::CreateTableExprGeneratorBuilder;
use tests_fuzz::generator::insert_expr::InsertExprGeneratorBuilder;
use tests_fuzz::generator::Generator;
use tests_fuzz::ir::{droppable_columns, AlterTableExpr, CreateTableExpr};
use tests_fuzz::ir::{droppable_columns, AlterTableExpr, CreateTableExpr, InsertIntoExpr};
use tests_fuzz::translator::mysql::alter_expr::AlterTableExprTranslator;
use tests_fuzz::translator::mysql::create_expr::CreateTableExprTranslator;
use tests_fuzz::translator::mysql::insert_expr::InsertIntoExprTranslator;
use tests_fuzz::translator::DslTranslator;
use tests_fuzz::utils::{init_greptime_connections, Connections};
use tests_fuzz::validator;
@@ -52,19 +54,17 @@ impl FuzzContext {
}
}
#[derive(Clone, Debug)]
#[derive(Clone, Debug, Copy)]
struct FuzzInput {
seed: u64,
actions: usize,
rows: usize,
}
fn generate_create_table_expr<R: Rng + 'static>(rng: &mut R) -> Result<CreateTableExpr> {
let columns = rng.gen_range(2..30);
let create_table_generator = CreateTableExprGeneratorBuilder::default()
.name_generator(Box::new(MappedGenerator::new(
WordGenerator,
merge_two_word_map_fn(random_capitalize_map, uppercase_and_keyword_backtick_map),
)))
.name_generator(Box::new(WordGenerator))
.columns(columns)
.engine("mito")
.build()
@@ -80,10 +80,7 @@ fn generate_alter_table_expr<R: Rng + 'static>(
if rename {
let expr_generator = AlterExprRenameGeneratorBuilder::default()
.table_ctx(table_ctx)
.name_generator(Box::new(MappedGenerator::new(
WordGenerator,
merge_two_word_map_fn(random_capitalize_map, uppercase_and_keyword_backtick_map),
)))
.name_generator(Box::new(WordGenerator))
.build()
.unwrap();
expr_generator.generate(rng)
@@ -112,11 +109,29 @@ impl Arbitrary<'_> for FuzzInput {
let seed = u.int_in_range(u64::MIN..=u64::MAX)?;
let mut rng = ChaChaRng::seed_from_u64(seed);
let actions = rng.gen_range(1..256);
let insertions = rng.gen_range(1..1024);
Ok(FuzzInput { seed, actions })
Ok(FuzzInput {
seed,
actions,
rows: insertions,
})
}
}
fn generate_insert_expr<R: Rng + 'static>(
input: FuzzInput,
rng: &mut R,
table_ctx: TableContextRef,
) -> Result<InsertIntoExpr> {
let insert_generator = InsertExprGeneratorBuilder::default()
.table_ctx(table_ctx)
.rows(input.rows)
.build()
.unwrap();
insert_generator.generate(rng)
}
async fn execute_alter_table(ctx: FuzzContext, input: FuzzInput) -> Result<()> {
info!("input: {input:?}");
let mut rng = ChaChaRng::seed_from_u64(input.seed);
@@ -146,7 +161,7 @@ async fn execute_alter_table(ctx: FuzzContext, input: FuzzInput) -> Result<()> {
table_ctx = Arc::new(Arc::unwrap_or_clone(table_ctx).alter(expr).unwrap());
// Validates columns
let mut column_entries = validator::column::fetch_columns(
let mut column_entries = validator::column::fetch_columns_via_mysql(
&ctx.greptime,
"public".into(),
table_ctx.name.clone(),
@@ -156,6 +171,28 @@ async fn execute_alter_table(ctx: FuzzContext, input: FuzzInput) -> Result<()> {
let mut columns = table_ctx.columns.clone();
columns.sort_by(|a, b| a.name.value.cmp(&b.name.value));
validator::column::assert_eq(&column_entries, &columns)?;
// insertions
let insert_expr = generate_insert_expr(input, &mut rng, table_ctx.clone())?;
let translator = InsertIntoExprTranslator;
let sql = translator.translate(&insert_expr)?;
let result = ctx
.greptime
// unprepared query, see <https://github.com/GreptimeTeam/greptimedb/issues/3500>
.execute(sql.as_str())
.await
.context(error::ExecuteQuerySnafu { sql: &sql })?;
ensure!(
result.rows_affected() == input.rows as u64,
error::AssertSnafu {
reason: format!(
"expected rows affected: {}, actual: {}",
input.rows,
result.rows_affected(),
)
}
);
}
// Cleans up