rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-27 10:20:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: directly invoke Datanode methods in standalone mode (part 1) (#694)

Browse Source 
* refactor: directly invoke Datanode methods in standalone mode

* test: add more unit tests

* fix: get rid of `println` in testing codes

* fix: resolve PR comments

* fix: resolve PR comments

Co-authored-by: luofucong <luofucong@greptime.com>
This commit is contained in:
LFC
2022-12-07 11:37:59 +08:00
committed by GitHub
parent 90c832b33d
commit 833216d317
24 changed files with 738 additions and 676 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/script/src/python/builtins/mod.rs
View File

@@ -15,7 +15,6 @@
//! Builtin module contains GreptimeDB builtin udf/udaf
#[cfg(test)]
#[allow(clippy::print_stdout)]
mod test;
use datafusion_common::{DataFusionError, ScalarValue};

13
src/script/src/python/builtins/test.rs
View File

@@ -18,6 +18,7 @@ use std::io::Read;
use std::path::Path;
use std::sync::Arc;
use common_telemetry::{error, info};
use datatypes::arrow::array::{Float64Array, Int64Array, PrimitiveArray};
use datatypes::arrow::compute::cast::CastOptions;
use datatypes::arrow::datatypes::DataType;
@@ -331,6 +332,8 @@ impl PyValue {
#[test]
fn run_builtin_fn_testcases() {
common_telemetry::init_default_ut_logging();
let loc = Path::new("src/python/builtins/testcases.ron");
let loc = loc.to_str().expect("Fail to parse path");
let mut file = File::open(loc).expect("Fail to open file");
@@ -343,7 +346,7 @@ fn run_builtin_fn_testcases() {
PyVector::make_class(&vm.ctx);
});
for (idx, case) in testcases.into_iter().enumerate() {
print!("Testcase {idx} ...");
info!("Testcase {idx} ...");
cached_vm
.enter(|vm| {
let scope = vm.new_scope_with_builtins();
@@ -368,7 +371,7 @@ fn run_builtin_fn_testcases() {
let err_res = format_py_error(e, vm).to_string();
match case.expect{
Ok(v) => {
println!("\nError:\n{err_res}");
error!("\nError:\n{err_res}");
panic!("Expect Ok: {v:?}, found Error");
},
Err(err) => {
@@ -397,7 +400,6 @@ fn run_builtin_fn_testcases() {
}
};
});
println!(" passed!");
}
}
@@ -443,6 +445,8 @@ fn set_lst_of_vecs_in_scope(
#[allow(unused_must_use)]
#[test]
fn test_vm() {
common_telemetry::init_default_ut_logging();
rustpython_vm::Interpreter::with_init(Default::default(), |vm| {
vm.add_native_module("udf_builtins", Box::new(greptime_builtin::make_module));
// this can be in `.enter()` closure, but for clearity, put it in the `with_init()`
@@ -471,11 +475,10 @@ sin(values)"#,
.map_err(|err| vm.new_syntax_error(&err))
.unwrap();
let res = vm.run_code_obj(code_obj, scope);
println!("{:#?}", res);
match res {
Err(e) => {
let err_res = format_py_error(e, vm).to_string();
println!("Error:\n{err_res}");
error!("Error:\n{err_res}");
}
Ok(obj) => {
let _ser = PyValue::from_py_obj(&obj, vm);

82
src/script/src/python/test.rs
View File

@@ -20,6 +20,7 @@ use std::io::prelude::*;
use std::path::Path;
use std::sync::Arc;
use common_telemetry::{error, info};
use console::style;
use datafusion_common::record_batch::RecordBatch as DfRecordBatch;
use datatypes::arrow::array::PrimitiveArray;
@@ -82,6 +83,8 @@ fn create_sample_recordbatch() -> DfRecordBatch {
/// and exec/parse (depending on the type of predicate) then decide if result is as expected
#[test]
fn run_ron_testcases() {
common_telemetry::init_default_ut_logging();
let loc = Path::new("src/python/testcases.ron");
let loc = loc.to_str().expect("Fail to parse path");
let mut file = File::open(loc).expect("Fail to open file");
@@ -89,9 +92,9 @@ fn run_ron_testcases() {
file.read_to_string(&mut buf)
.expect("Fail to read to string");
let testcases: Vec<TestCase> = from_ron_string(&buf).expect("Fail to convert to testcases");
println!("Read {} testcases from {}", testcases.len(), loc);
info!("Read {} testcases from {}", testcases.len(), loc);
for testcase in testcases {
print!(".ron test {}", testcase.name);
info!(".ron test {}", testcase.name);
match testcase.predicate {
Predicate::ParseIsOk { result } => {
let copr = parse_and_compile_copr(&testcase.code);
@@ -101,21 +104,19 @@ fn run_ron_testcases() {
}
Predicate::ParseIsErr { reason } => {
let copr = parse_and_compile_copr(&testcase.code);
if copr.is_ok() {
eprintln!("Expect to be err, found{copr:#?}");
panic!()
}
assert!(copr.is_err(), "Expect to be err, actual {copr:#?}");
let res = &copr.unwrap_err();
println!(
error!(
"{}",
pretty_print_error_in_src(&testcase.code, res, 0, "<embedded>")
);
let (res, _) = get_error_reason_loc(res);
if !res.contains(&reason) {
eprintln!("{}", testcase.code);
eprintln!("Parse Error, expect \"{reason}\" in \"{res}\", but not found.");
panic!()
}
assert!(
res.contains(&reason),
"{} Parse Error, expect \"{reason}\" in \"{res}\", actual not found.",
testcase.code,
);
}
Predicate::ExecIsOk { fields, columns } => {
let rb = create_sample_recordbatch();
@@ -129,28 +130,25 @@ fn run_ron_testcases() {
.iter()
.zip(&res.schema.arrow_schema().fields)
.map(|(anno, real)| {
if !(anno.datatype.clone().unwrap() == real.data_type
&& anno.is_nullable == real.is_nullable)
{
eprintln!("fields expect to be {anno:#?}, found to be {real:#?}.");
panic!()
}
assert!(
anno.datatype.clone().unwrap() == real.data_type
&& anno.is_nullable == real.is_nullable,
"Fields expected to be {anno:#?}, actual {real:#?}"
);
})
.count();
columns
.iter()
.zip(res.df_recordbatch.columns())
.map(|(anno, real)| {
if !(&anno.ty == real.data_type() && anno.len == real.len()) {
eprintln!(
"Unmatch type or length!Expect [{:#?}; {}], found [{:#?}; {}]",
anno.ty,
anno.len,
real.data_type(),
real.len()
);
panic!()
}
assert!(
&anno.ty == real.data_type() && anno.len == real.len(),
"Type or length not match! Expect [{:#?}; {}], actual [{:#?}; {}]",
anno.ty,
anno.len,
real.data_type(),
real.len()
);
})
.count();
}
@@ -159,28 +157,24 @@ fn run_ron_testcases() {
} => {
let rb = create_sample_recordbatch();
let res = coprocessor::exec_coprocessor(&testcase.code, &rb);
assert!(res.is_err(), "{:#?}\nExpect Err(...), actual Ok(...)", res);
if let Err(res) = res {
println!(
error!(
"{}",
pretty_print_error_in_src(&testcase.code, &res, 1120, "<embedded>")
);
let (reason, _) = get_error_reason_loc(&res);
if !reason.contains(&part_reason) {
eprintln!(
"{}\nExecute error, expect \"{reason}\" in \"{res}\", but not found.",
testcase.code,
reason = style(reason).green(),
res = style(res).red()
);
panic!()
}
} else {
eprintln!("{:#?}\nExpect Err(...), found Ok(...)", res);
panic!();
assert!(
reason.contains(&part_reason),
"{}\nExecute error, expect \"{reason}\" in \"{res}\", actual not found.",
testcase.code,
reason = style(reason).green(),
res = style(res).red()
)
}
}
}
println!(" ... {}", style("ok✅").green());
info!(" ... {}", style("ok✅").green());
}
}
@@ -275,7 +269,7 @@ def calc_rvs(open_time, close):
0,
"copr.py",
);
println!("{res}");
info!("{res}");
} else if let Ok(res) = ret {
dbg!(&res);
} else {
@@ -319,7 +313,7 @@ def a(cpu, mem):
0,
"copr.py",
);
println!("{res}");
info!("{res}");
} else if let Ok(res) = ret {
dbg!(&res);
} else {

6
src/script/src/python/vector.rs
View File

@@ -1039,6 +1039,7 @@ pub mod tests {
use std::sync::Arc;
use common_telemetry::info;
use datatypes::vectors::{Float32Vector, Int32Vector, NullVector};
use rustpython_vm::builtins::PyList;
use rustpython_vm::class::PyClassImpl;
@@ -1170,9 +1171,10 @@ pub mod tests {
}
#[test]
#[allow(clippy::print_stdout)]
// for debug purpose, also this is already a test function so allow print_stdout shouldn't be a problem?
fn test_execute_script() {
common_telemetry::init_default_ut_logging();
fn is_eq<T: std::cmp::PartialEq + rustpython_vm::TryFromObject>(
v: PyResult,
i: T,
@@ -1221,7 +1223,7 @@ pub mod tests {
for (code, pred) in snippet {
let result = execute_script(&interpreter, code, None, pred);
println!(
info!(
"\u{001B}[35m{code}\u{001B}[0m: {:?}{}",
result.clone().map(|v| v.0),
result