diff --git a/Cargo.lock b/Cargo.lock index 491fe43e65..eccb2c2992 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1272,7 +1272,7 @@ checksum = "29c39203181991a7dd4343b8005bd804e7a9a37afb8ac070e43771e8c820bbde" dependencies = [ "chrono", "chrono-tz-build 0.0.3", - "phf 0.11.1", + "phf", ] [[package]] @@ -1283,7 +1283,7 @@ checksum = "cf9cc2b23599e6d7479755f3594285efb3f74a1bdca7a7374948bc831e23a552" dependencies = [ "chrono", "chrono-tz-build 0.1.0", - "phf 0.11.1", + "phf", ] [[package]] @@ -1293,8 +1293,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f509c3a87b33437b05e2458750a0700e5bdd6956176773e6c7d6dd15a283a0c" dependencies = [ "parse-zoneinfo", - "phf 0.11.1", - "phf_codegen 0.11.1", + "phf", + "phf_codegen", ] [[package]] @@ -1304,8 +1304,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9998fb9f7e9b2111641485bf8beb32f92945f97f92a3d061f744cfef335f751" dependencies = [ "parse-zoneinfo", - "phf 0.11.1", - "phf_codegen 0.11.1", + "phf", + "phf_codegen", ] [[package]] @@ -1526,7 +1526,7 @@ dependencies = [ "partition", "query", "rexpect", - "rustyline", + "rustyline 10.1.1", "serde", "servers", "session", @@ -4427,6 +4427,17 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" +[[package]] +name = "libsqlite3-sys" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa" +dependencies = [ + "cc", + "pkg-config", + "vcpkg", +] + [[package]] name = "libz-sys" version = "1.1.8" @@ -4706,6 +4717,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "memoffset" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +dependencies = [ + "autocfg", +] + [[package]] name = "memoffset" version = "0.8.0" @@ -5102,18 +5122,6 @@ dependencies = [ "memoffset 0.6.5", ] -[[package]] -name = "nix" -version = "0.24.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069" -dependencies = [ - "bitflags 1.3.2", - "cfg-if 1.0.0", - "libc", - "memoffset 0.6.5", -] - [[package]] name = "nix" version = "0.25.1" @@ -5137,6 +5145,8 @@ dependencies = [ "bitflags 1.3.2", "cfg-if 1.0.0", "libc", + "memoffset 0.7.1", + "pin-utils", "static_assertions", ] @@ -5183,7 +5193,6 @@ dependencies = [ "autocfg", "num-integer", "num-traits", - "serde", ] [[package]] @@ -5210,7 +5219,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d" dependencies = [ "num-traits", - "serde", ] [[package]] @@ -5805,15 +5813,6 @@ dependencies = [ "x509-certificate", ] -[[package]] -name = "phf" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" -dependencies = [ - "phf_shared 0.10.0", -] - [[package]] name = "phf" version = "0.11.1" @@ -5823,36 +5822,16 @@ dependencies = [ "phf_shared 0.11.1", ] -[[package]] -name = "phf_codegen" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd" -dependencies = [ - "phf_generator 0.10.0", - "phf_shared 0.10.0", -] - [[package]] name = "phf_codegen" version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770" dependencies = [ - "phf_generator 0.11.1", + "phf_generator", "phf_shared 0.11.1", ] -[[package]] -name = "phf_generator" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" -dependencies = [ - "phf_shared 0.10.0", - "rand", -] - [[package]] name = "phf_generator" version = "0.11.1" @@ -7121,8 +7100,8 @@ dependencies = [ [[package]] name = "rustpython-ast" -version = "0.1.0" -source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca" +version = "0.2.0" +source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e" dependencies = [ "num-bigint", "rustpython-common", @@ -7131,8 +7110,8 @@ dependencies = [ [[package]] name = "rustpython-codegen" -version = "0.1.2" -source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca" +version = "0.2.0" +source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e" dependencies = [ "ahash 0.7.6", "bitflags 1.3.2", @@ -7143,17 +7122,19 @@ dependencies = [ "num-traits", "rustpython-ast", "rustpython-compiler-core", - "thiserror", ] [[package]] name = "rustpython-common" -version = "0.0.0" -source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca" +version = "0.2.0" +source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e" dependencies = [ "ascii", + "bitflags 1.3.2", + "bstr 0.2.17", "cfg-if 1.0.0", "hexf-parse", + "itertools", "lexical-parse-float", "libc", "lock_api", @@ -7171,36 +7152,31 @@ dependencies = [ [[package]] name = "rustpython-compiler" -version = "0.1.2" -source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca" +version = "0.2.0" +source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e" dependencies = [ "rustpython-codegen", "rustpython-compiler-core", "rustpython-parser", - "thiserror", ] [[package]] name = "rustpython-compiler-core" -version = "0.1.2" -source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca" +version = "0.2.0" +source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e" dependencies = [ - "bincode", "bitflags 1.3.2", "bstr 0.2.17", "itertools", "lz4_flex", "num-bigint", "num-complex", - "serde", - "static_assertions", - "thiserror", ] [[package]] name = "rustpython-derive" -version = "0.1.2" -source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca" +version = "0.2.0" +source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e" dependencies = [ "rustpython-compiler", "rustpython-derive-impl", @@ -7209,8 +7185,8 @@ dependencies = [ [[package]] name = "rustpython-derive-impl" -version = "0.0.0" -source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca" +version = "0.2.0" +source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e" dependencies = [ "indexmap", "itertools", @@ -7235,8 +7211,8 @@ dependencies = [ [[package]] name = "rustpython-parser" -version = "0.1.2" -source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca" +version = "0.2.0" +source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e" dependencies = [ "ahash 0.7.6", "anyhow", @@ -7246,12 +7222,11 @@ dependencies = [ "log", "num-bigint", "num-traits", - "phf 0.10.1", - "phf_codegen 0.10.0", + "phf", + "phf_codegen", "rustc-hash", "rustpython-ast", "rustpython-compiler-core", - "thiserror", "tiny-keccak", "unic-emoji-char", "unic-ucd-ident", @@ -7260,8 +7235,8 @@ dependencies = [ [[package]] name = "rustpython-pylib" -version = "0.1.0" -source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca" +version = "0.2.0" +source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e" dependencies = [ "glob", "rustpython-compiler-core", @@ -7270,8 +7245,8 @@ dependencies = [ [[package]] name = "rustpython-stdlib" -version = "0.1.2" -source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca" +version = "0.2.0" +source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e" dependencies = [ "adler32", "ahash 0.7.6", @@ -7284,18 +7259,19 @@ dependencies = [ "csv-core", "digest", "dns-lookup", + "dyn-clone", "flate2", "gethostname", "hex", "itertools", - "lexical-parse-float", "libc", + "libsqlite3-sys", "mac_address", "md-5", "memchr", "memmap2", "mt19937", - "nix 0.24.3", + "nix 0.26.2", "num-bigint", "num-complex", "num-integer", @@ -7319,6 +7295,7 @@ dependencies = [ "socket2 0.4.9", "system-configuration", "termios", + "ucd", "unic-char-property", "unic-normal", "unic-ucd-age", @@ -7335,10 +7312,9 @@ dependencies = [ [[package]] name = "rustpython-vm" -version = "0.1.2" -source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca" +version = "0.2.0" +source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e" dependencies = [ - "adler32", "ahash 0.7.6", "ascii", "atty", @@ -7349,12 +7325,10 @@ dependencies = [ "chrono", "crossbeam-utils", "exitcode", - "flate2", "getrandom", "glob", "half 1.8.2", "hex", - "hexf-parse", "indexmap", "is-macro", "itertools", @@ -7362,7 +7336,7 @@ dependencies = [ "log", "memchr", "memoffset 0.6.5", - "nix 0.24.3", + "nix 0.26.2", "num-bigint", "num-complex", "num-integer", @@ -7384,9 +7358,8 @@ dependencies = [ "rustpython-compiler-core", "rustpython-derive", "rustpython-parser", - "rustyline", + "rustyline 11.0.0", "schannel", - "serde", "sre-engine", "static_assertions", "strum", @@ -7437,6 +7410,29 @@ dependencies = [ "winapi", ] +[[package]] +name = "rustyline" +version = "11.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dfc8644681285d1fb67a467fb3021bfea306b99b4146b166a1fe3ada965eece" +dependencies = [ + "bitflags 1.3.2", + "cfg-if 1.0.0", + "clipboard-win", + "dirs-next", + "fd-lock", + "libc", + "log", + "memchr", + "nix 0.26.2", + "radix_trie", + "scopeguard", + "unicode-segmentation", + "unicode-width", + "utf8parse", + "winapi", +] + [[package]] name = "ryu" version = "1.0.13" @@ -8999,7 +8995,7 @@ dependencies = [ "log", "parking_lot", "percent-encoding", - "phf 0.11.1", + "phf", "pin-project-lite", "postgres-protocol", "postgres-types", @@ -9531,6 +9527,12 @@ dependencies = [ "typify-impl 0.0.11", ] +[[package]] +name = "ucd" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe4fa6e588762366f1eb4991ce59ad1b93651d0b769dfb4e4d1c5c4b943d1159" + [[package]] name = "ucd-trie" version = "0.1.5" @@ -9733,9 +9735,11 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "unicode_names2" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "029df4cc8238cefc911704ff8fa210853a0f3bce2694d8f51181dd41ee0f3301" +version = "0.6.0" +source = "git+https://github.com/youknowone/unicode_names2.git?rev=4ce16aa85cbcdd9cc830410f1a72ef9a235f2fde#4ce16aa85cbcdd9cc830410f1a72ef9a235f2fde" +dependencies = [ + "phf", +] [[package]] name = "unindent" diff --git a/src/script/Cargo.toml b/src/script/Cargo.toml index e5320a55e4..9d282befef 100644 --- a/src/script/Cargo.toml +++ b/src/script/Cargo.toml @@ -46,17 +46,17 @@ futures-util.workspace = true once_cell = "1.17.0" paste = { workspace = true, optional = true } query = { path = "../query" } -# TODO(discord9): This is a forked and tweaked version of RustPython, please update it to newest original RustPython After Update toolchain to 1.65 -rustpython-ast = { git = "https://github.com/discord9/RustPython", optional = true, rev = "2e126345" } -rustpython-codegen = { git = "https://github.com/discord9/RustPython", optional = true, rev = "2e126345" } -rustpython-compiler = { git = "https://github.com/discord9/RustPython", optional = true, rev = "2e126345" } -rustpython-compiler-core = { git = "https://github.com/discord9/RustPython", optional = true, rev = "2e126345" } -rustpython-parser = { git = "https://github.com/discord9/RustPython", optional = true, rev = "2e126345" } -rustpython-pylib = { git = "https://github.com/discord9/RustPython", optional = true, rev = "2e126345", features = [ +# TODO(discord9): This is a forked and tweaked version of RustPython, please update it to newest original RustPython After RustPython support GC +rustpython-ast = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412" } +rustpython-codegen = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412" } +rustpython-compiler = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412" } +rustpython-compiler-core = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412" } +rustpython-parser = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412" } +rustpython-pylib = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412", features = [ "freeze-stdlib", ] } -rustpython-stdlib = { git = "https://github.com/discord9/RustPython", optional = true, rev = "2e126345" } -rustpython-vm = { git = "https://github.com/discord9/RustPython", optional = true, rev = "2e126345", features = [ +rustpython-stdlib = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412" } +rustpython-vm = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412", features = [ "default", "codegen", ] } diff --git a/src/script/src/python/error.rs b/src/script/src/python/error.rs index e55869bdea..a1e09931e8 100644 --- a/src/script/src/python/error.rs +++ b/src/script/src/python/error.rs @@ -20,7 +20,7 @@ use datatypes::error::Error as DataTypeError; use query::error::Error as QueryError; use rustpython_codegen::error::CodegenError; use rustpython_parser::ast::Location; -use rustpython_parser::error::ParseError; +use rustpython_parser::ParseError; pub use snafu::ensure; use snafu::prelude::Snafu; use snafu::Location as SnafuLocation; diff --git a/src/script/src/python/ffi_types/copr/compile.rs b/src/script/src/python/ffi_types/copr/compile.rs index 14ec5cd391..c4b38501bd 100644 --- a/src/script/src/python/ffi_types/copr/compile.rs +++ b/src/script/src/python/ffi_types/copr/compile.rs @@ -17,7 +17,7 @@ use rustpython_codegen::compile::compile_top; use rustpython_compiler::{CompileOpts, Mode}; use rustpython_compiler_core::CodeObject; use rustpython_parser::ast::{ArgData, Located, Location}; -use rustpython_parser::{ast, parser}; +use rustpython_parser::{ast, parse, Mode as ParseMode}; use snafu::ResultExt; use crate::fail_parse_error; @@ -95,8 +95,7 @@ pub fn compile_script( script: &str, ) -> Result { // note that it's important to use `parser::Mode::Interactive` so the ast can be compile to return a result instead of return None in eval mode - let mut top = - parser::parse(script, parser::Mode::Interactive, "").context(PyParseSnafu)?; + let mut top = parse(script, ParseMode::Interactive, "").context(PyParseSnafu)?; // erase decorator if let ast::Mod::Interactive { body } = &mut top { let stmts = body; diff --git a/src/script/src/python/ffi_types/copr/parse.rs b/src/script/src/python/ffi_types/copr/parse.rs index 880b2e5890..537e91f54d 100644 --- a/src/script/src/python/ffi_types/copr/parse.rs +++ b/src/script/src/python/ffi_types/copr/parse.rs @@ -18,7 +18,7 @@ use std::sync::Arc; use datatypes::prelude::ConcreteDataType; use query::QueryEngineRef; use rustpython_parser::ast::{Arguments, Location}; -use rustpython_parser::{ast, parser}; +use rustpython_parser::{ast, parse_program}; #[cfg(test)] use serde::Deserialize; use snafu::{OptionExt, ResultExt}; @@ -443,7 +443,7 @@ pub fn parse_and_compile_copr( script: &str, query_engine: Option, ) -> Result { - let python_ast = parser::parse_program(script, "").context(PyParseSnafu)?; + let python_ast = parse_program(script, "").context(PyParseSnafu)?; let mut coprocessor = None; diff --git a/src/script/src/python/ffi_types/vector.rs b/src/script/src/python/ffi_types/vector.rs index f3e0f8aefc..7b8378f25b 100644 --- a/src/script/src/python/ffi_types/vector.rs +++ b/src/script/src/python/ffi_types/vector.rs @@ -38,7 +38,7 @@ use rustpython_vm::{ VirtualMachine, }; -use crate::python::utils::is_instance; +use crate::python::rspython::utils::is_instance; /// The Main FFI type `PyVector` that is used both in RustPython and PyO3 #[cfg_attr(feature = "pyo3_backend", pyo3class(name = "vector"))] diff --git a/src/script/src/python/ffi_types/vector/tests.rs b/src/script/src/python/ffi_types/vector/tests.rs index 8ee4f6f7cc..a0cd1e3bcc 100644 --- a/src/script/src/python/ffi_types/vector/tests.rs +++ b/src/script/src/python/ffi_types/vector/tests.rs @@ -155,8 +155,10 @@ fn eval_pyo3(testcase: TestCase, locals: HashMap) { } fn eval_rspy(testcase: TestCase, locals: HashMap) { - vm::Interpreter::without_stdlib(Default::default()).enter(|vm| { + vm::Interpreter::with_init(Default::default(), |vm| { PyVector::make_class(&vm.ctx); + }) + .enter(|vm| { let scope = vm.new_scope_with_builtins(); locals.into_iter().for_each(|(k, v)| { scope @@ -166,10 +168,17 @@ fn eval_rspy(testcase: TestCase, locals: HashMap) { .unwrap(); }); let code_obj = vm - .compile(&testcase.eval, Mode::Eval, "".to_owned()) + .compile(&testcase.eval, Mode::Eval, "".to_string()) .map_err(|err| vm.new_syntax_error(&err)) .unwrap(); - let obj = vm.run_code_obj(code_obj, scope).unwrap(); + let obj = vm + .run_code_obj(code_obj, scope) + .map_err(|e| { + let mut output = String::new(); + vm.write_exception(&mut output, &e).unwrap(); + (e, output) + }) + .unwrap(); let v = obj.downcast::().unwrap(); let result_arr = v.to_arrow_array(); let expect_arr = testcase.result.to_arrow_array(); diff --git a/src/script/src/python/rspython.rs b/src/script/src/python/rspython.rs index 341e21da40..710145425a 100644 --- a/src/script/src/python/rspython.rs +++ b/src/script/src/python/rspython.rs @@ -19,7 +19,7 @@ pub(crate) mod vector_impl; pub(crate) mod builtins; mod dataframe_impl; -mod utils; +pub(crate) mod utils; #[cfg(test)] pub(crate) use copr_impl::init_interpreter; diff --git a/src/script/src/python/rspython/builtins.rs b/src/script/src/python/rspython/builtins.rs index 1ee77c6100..fc78c61c2e 100644 --- a/src/script/src/python/rspython/builtins.rs +++ b/src/script/src/python/rspython/builtins.rs @@ -30,7 +30,7 @@ use rustpython_vm::builtins::{PyBaseExceptionRef, PyBool, PyFloat, PyInt, PyList use rustpython_vm::{pymodule, AsObject, PyObjectRef, PyPayload, PyResult, VirtualMachine}; use crate::python::ffi_types::PyVector; -use crate::python::utils::is_instance; +use crate::python::rspython::utils::is_instance; pub fn init_greptime_builtins(module_name: &str, vm: &mut VirtualMachine) { vm.add_native_module( @@ -356,7 +356,7 @@ pub(crate) mod greptime_builtin { #[pyfunction] fn col(name: String, vm: &VirtualMachine) -> PyExprRef { let expr: PyExpr = DfExpr::Column(datafusion_common::Column::from_name(name)).into(); - expr.into_ref(vm) + expr.into_ref(&vm.ctx) } #[pyfunction] @@ -366,7 +366,7 @@ pub(crate) mod greptime_builtin { .try_to_scalar_value(&val.data_type()) .map_err(|e| vm.new_runtime_error(format!("{e}")))?; let expr: PyExpr = DfExpr::Literal(scalar_val).into(); - Ok(expr.into_ref(vm)) + Ok(expr.into_ref(&vm.ctx)) } // the main binding code, due to proc macro things, can't directly use a simpler macro diff --git a/src/script/src/python/rspython/builtins/test.rs b/src/script/src/python/rspython/builtins/test.rs index 58f333dd86..e97f506105 100644 --- a/src/script/src/python/rspython/builtins/test.rs +++ b/src/script/src/python/rspython/builtins/test.rs @@ -33,8 +33,8 @@ use serde::{Deserialize, Serialize}; use super::*; use crate::python::ffi_types::PyVector; -use crate::python::utils::{format_py_error, is_instance}; - +use crate::python::rspython::utils::is_instance; +use crate::python::utils::format_py_error; #[test] fn convert_scalar_to_py_obj_and_back() { rustpython_vm::Interpreter::with_init(Default::default(), |vm| { diff --git a/src/script/src/python/rspython/copr_impl.rs b/src/script/src/python/rspython/copr_impl.rs index 576f0fa112..6f4c179273 100644 --- a/src/script/src/python/rspython/copr_impl.rs +++ b/src/script/src/python/rspython/copr_impl.rs @@ -213,7 +213,7 @@ pub(crate) fn init_interpreter() -> Arc { // so according to this issue: // https://github.com/RustPython/RustPython/issues/4292 // add this line for stdlib, so rustpython can found stdlib's python part in bytecode format - vm.add_frozen(rustpython_pylib::frozen_stdlib()); + vm.add_frozen(rustpython_pylib::FROZEN_STDLIB); // add our own custom datatype and module PyVector::make_class(&vm.ctx); PyQueryEngine::make_class(&vm.ctx); diff --git a/src/script/src/python/rspython/dataframe_impl.rs b/src/script/src/python/rspython/dataframe_impl.rs index e269db409a..deeabc0306 100644 --- a/src/script/src/python/rspython/dataframe_impl.rs +++ b/src/script/src/python/rspython/dataframe_impl.rs @@ -28,8 +28,10 @@ pub(crate) mod data_frame { use datafusion::dataframe::DataFrame as DfDataFrame; use datafusion::execution::context::SessionContext; use datafusion_expr::Expr as DfExpr; + use rustpython_vm::convert::ToPyResult; use rustpython_vm::function::PyComparisonValue; - use rustpython_vm::types::{Comparable, PyComparisonOp}; + use rustpython_vm::protocol::PyNumberMethods; + use rustpython_vm::types::{AsNumber, Comparable, PyComparisonOp}; use rustpython_vm::{ pyclass as rspyclass, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; @@ -40,6 +42,7 @@ pub(crate) mod data_frame { use crate::python::rspython::builtins::greptime_builtin::{ lit, query as get_query_engine, PyDataFrame, }; + use crate::python::rspython::utils::obj_cast_to; use crate::python::utils::block_on_async; impl From for PyDataFrame { @@ -313,7 +316,20 @@ pub(crate) mod data_frame { } } - #[rspyclass(with(Comparable))] + impl AsNumber for PyExpr { + fn as_number() -> &'static PyNumberMethods { + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + and: Some(|a, b, vm| PyExpr::and(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)), + or: Some(|a, b, vm| PyExpr::or(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)), + invert: Some(|a, vm| PyExpr::invert((*a).to_owned(), vm).to_pyresult(vm)), + + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } + } + + #[rspyclass(with(Comparable, AsNumber))] impl PyExpr { fn richcompare( &self, @@ -342,18 +358,23 @@ pub(crate) mod data_frame { } #[pymethod(magic)] - fn and(&self, other: PyExprRef) -> PyResult { - Ok(self.inner.clone().and(other.inner.clone()).into()) + fn and(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = obj_cast_to::(zelf, vm)?; + let other = obj_cast_to::(other, vm)?; + Ok(zelf.inner.clone().and(other.inner.clone()).into()) } #[pymethod(magic)] - fn or(&self, other: PyExprRef) -> PyResult { - Ok(self.inner.clone().or(other.inner.clone()).into()) + fn or(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = obj_cast_to::(zelf, vm)?; + let other = obj_cast_to::(other, vm)?; + Ok(zelf.inner.clone().or(other.inner.clone()).into()) } /// `~` operator, return `!self` #[pymethod(magic)] - fn invert(&self) -> PyResult { - Ok(self.inner.clone().not().into()) + fn invert(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = obj_cast_to::(zelf, vm)?; + Ok(zelf.inner.clone().not().into()) } /// sort ascending&nulls_first diff --git a/src/script/src/python/rspython/test.rs b/src/script/src/python/rspython/test.rs index f82c037e69..9277651168 100644 --- a/src/script/src/python/rspython/test.rs +++ b/src/script/src/python/rspython/test.rs @@ -25,7 +25,7 @@ use datatypes::data_type::{ConcreteDataType, DataType}; use datatypes::schema::{ColumnSchema, Schema}; use datatypes::vectors::{Float32Vector, Float64Vector, Int64Vector, VectorRef}; use ron::from_str as from_ron_string; -use rustpython_parser::parser; +use rustpython_parser::{parse, Mode}; use serde::{Deserialize, Serialize}; use crate::python::error::{get_error_reason_loc, pretty_print_error_in_src, visualize_loc, Error}; @@ -181,7 +181,7 @@ fn test_type_anno() { def a(cpu, mem: vector[f64])->(vector[f64|None], vector[f64], vector[_], vector[ _ | None]): return cpu + mem, cpu - mem, cpu * mem, cpu / mem "#; - let pyast = parser::parse(python_source, parser::Mode::Interactive, "").unwrap(); + let pyast = parse(python_source, Mode::Interactive, "").unwrap(); let copr = parse_and_compile_copr(python_source, None); dbg!(copr); } diff --git a/src/script/src/python/rspython/utils.rs b/src/script/src/python/rspython/utils.rs index c230670731..e1ee104b46 100644 --- a/src/script/src/python/rspython/utils.rs +++ b/src/script/src/python/rspython/utils.rs @@ -22,7 +22,8 @@ use datatypes::vectors::{ BooleanVector, Float64Vector, Helper, Int64Vector, NullVector, StringVector, VectorRef, }; use rustpython_vm::builtins::{PyBaseExceptionRef, PyBool, PyFloat, PyInt, PyList, PyStr}; -use rustpython_vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine}; +use rustpython_vm::object::PyObjectPayload; +use rustpython_vm::{PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine}; use snafu::{OptionExt, ResultExt}; use crate::python::error; @@ -33,7 +34,21 @@ use crate::python::rspython::builtins::try_into_columnar_value; /// use `rustpython`'s `is_instance` method to check if a PyObject is a instance of class. /// if `PyResult` is Err, then this function return `false` pub fn is_instance(obj: &PyObjectRef, vm: &VirtualMachine) -> bool { - obj.is_instance(T::class(vm).into(), vm).unwrap_or(false) + obj.is_instance(T::class(&vm.ctx).into(), vm) + .unwrap_or(false) +} + +pub fn obj_cast_to( + obj: PyObjectRef, + vm: &VirtualMachine, +) -> PyResult> { + obj.downcast::().map_err(|e| { + vm.new_type_error(format!( + "Can't cast object into {}, actual type: {}", + std::any::type_name::(), + e.class().name() + )) + }) } pub fn format_py_error(excep: PyBaseExceptionRef, vm: &VirtualMachine) -> error::Error { diff --git a/src/script/src/python/rspython/vector_impl.rs b/src/script/src/python/rspython/vector_impl.rs index fe25119b74..2f373c1c3b 100644 --- a/src/script/src/python/rspython/vector_impl.rs +++ b/src/script/src/python/rspython/vector_impl.rs @@ -25,18 +25,21 @@ use datatypes::value::{self, OrderedFloat}; use datatypes::vectors::Helper; use once_cell::sync::Lazy; use rustpython_vm::builtins::{PyBaseExceptionRef, PyBool, PyBytes, PyFloat, PyInt, PyNone, PyStr}; +use rustpython_vm::convert::ToPyResult; use rustpython_vm::function::{Either, OptionalArg, PyComparisonValue}; -use rustpython_vm::protocol::{PyMappingMethods, PySequenceMethods}; -use rustpython_vm::types::{AsMapping, AsSequence, Comparable, PyComparisonOp}; +use rustpython_vm::protocol::{PyMappingMethods, PyNumberMethods, PySequenceMethods}; +use rustpython_vm::types::{ + AsMapping, AsNumber, AsSequence, Comparable, PyComparisonOp, Representable, +}; use rustpython_vm::{ - atomic_func, pyclass as rspyclass, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + atomic_func, pyclass as rspyclass, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, }; use crate::python::ffi_types::vector::{ arrow_rfloordiv, arrow_rsub, arrow_rtruediv, rspy_is_pyobj_scalar, wrap_result, PyVector, }; -use crate::python::utils::is_instance; +use crate::python::rspython::utils::{is_instance, obj_cast_to}; /// PyVectors' rustpython specify methods fn to_type_error(vm: &'_ VirtualMachine) -> impl FnOnce(String) -> PyBaseExceptionRef + '_ { @@ -45,7 +48,7 @@ fn to_type_error(vm: &'_ VirtualMachine) -> impl FnOnce(String) -> PyBaseExcepti pub(crate) type PyVectorRef = PyRef; /// PyVector type wraps a greptime vector, impl multiply/div/add/sub opeerators etc. -#[rspyclass(with(AsMapping, AsSequence, Comparable))] +#[rspyclass(with(AsMapping, AsSequence, Comparable, AsNumber, Representable))] impl PyVector { #[pymethod] pub(crate) fn new( @@ -86,29 +89,32 @@ impl PyVector { #[pymethod(name = "__radd__")] #[pymethod(magic)] - fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn add(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = obj_cast_to::(zelf, vm)?; if rspy_is_pyobj_scalar(&other, vm) { - self.rspy_scalar_arith_op(other, None, wrap_result(arithmetic::add_dyn), vm) + zelf.rspy_scalar_arith_op(other, None, wrap_result(arithmetic::add_dyn), vm) } else { - self.rspy_vector_arith_op(other, None, wrap_result(arithmetic::add_dyn), vm) + zelf.rspy_vector_arith_op(other, None, wrap_result(arithmetic::add_dyn), vm) } } #[pymethod(magic)] - fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn sub(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = obj_cast_to::(zelf, vm)?; if rspy_is_pyobj_scalar(&other, vm) { - self.rspy_scalar_arith_op(other, None, wrap_result(arithmetic::subtract_dyn), vm) + zelf.rspy_scalar_arith_op(other, None, wrap_result(arithmetic::subtract_dyn), vm) } else { - self.rspy_vector_arith_op(other, None, wrap_result(arithmetic::subtract_dyn), vm) + zelf.rspy_vector_arith_op(other, None, wrap_result(arithmetic::subtract_dyn), vm) } } #[pymethod(magic)] - fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn rsub(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = obj_cast_to::(zelf, vm)?; if rspy_is_pyobj_scalar(&other, vm) { - self.rspy_scalar_arith_op(other, None, arrow_rsub, vm) + zelf.rspy_scalar_arith_op(other, None, arrow_rsub, vm) } else { - self.rspy_vector_arith_op( + zelf.rspy_vector_arith_op( other, None, wrap_result(|a, b| arithmetic::subtract_dyn(b, a)), @@ -119,25 +125,27 @@ impl PyVector { #[pymethod(name = "__rmul__")] #[pymethod(magic)] - fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn mul(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = obj_cast_to::(zelf, vm)?; if rspy_is_pyobj_scalar(&other, vm) { - self.rspy_scalar_arith_op(other, None, wrap_result(arithmetic::multiply_dyn), vm) + zelf.rspy_scalar_arith_op(other, None, wrap_result(arithmetic::multiply_dyn), vm) } else { - self.rspy_vector_arith_op(other, None, wrap_result(arithmetic::multiply_dyn), vm) + zelf.rspy_vector_arith_op(other, None, wrap_result(arithmetic::multiply_dyn), vm) } } #[pymethod(magic)] - fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn truediv(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = obj_cast_to::(zelf, vm)?; if rspy_is_pyobj_scalar(&other, vm) { - self.rspy_scalar_arith_op( + zelf.rspy_scalar_arith_op( other, Some(ArrowDataType::Float64), wrap_result(arithmetic::divide_dyn), vm, ) } else { - self.rspy_vector_arith_op( + zelf.rspy_vector_arith_op( other, Some(ArrowDataType::Float64), wrap_result(arithmetic::divide_dyn), @@ -161,16 +169,17 @@ impl PyVector { } #[pymethod(magic)] - fn floordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + fn floordiv(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = obj_cast_to::(zelf, vm)?; if rspy_is_pyobj_scalar(&other, vm) { - self.rspy_scalar_arith_op( + zelf.rspy_scalar_arith_op( other, Some(ArrowDataType::Int64), wrap_result(arithmetic::divide_dyn), vm, ) } else { - self.rspy_vector_arith_op( + zelf.rspy_vector_arith_op( other, Some(ArrowDataType::Int64), wrap_result(arithmetic::divide_dyn), @@ -194,19 +203,33 @@ impl PyVector { } } - #[pymethod(magic)] - fn and(&self, other: PyVectorRef, vm: &VirtualMachine) -> PyResult { - Self::vector_and(self, &other).map_err(to_type_error(vm)) + fn obj_to_vector(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult> { + obj.downcast::().map_err(|e| { + vm.new_type_error(format!( + "Can't cast right operand into PyVector, actual type: {}", + e.class().name() + )) + }) } #[pymethod(magic)] - fn or(&self, other: PyVectorRef, vm: &VirtualMachine) -> PyResult { - Self::vector_or(self, &other).map_err(to_type_error(vm)) + fn and(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = Self::obj_to_vector(zelf, vm)?; + let other = Self::obj_to_vector(other, vm)?; + Self::vector_and(&zelf, &other).map_err(to_type_error(vm)) } #[pymethod(magic)] - fn invert(&self, vm: &VirtualMachine) -> PyResult { - Self::vector_invert(self).map_err(to_type_error(vm)) + fn or(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = Self::obj_to_vector(zelf, vm)?; + let other = Self::obj_to_vector(other, vm)?; + Self::vector_or(&zelf, &other).map_err(to_type_error(vm)) + } + + #[pymethod(magic)] + fn invert(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult { + let zelf = Self::obj_to_vector(zelf, vm)?; + Self::vector_invert(&zelf).map_err(to_type_error(vm)) } #[pymethod(name = "__len__")] @@ -262,6 +285,40 @@ impl PyVector { } } +impl Representable for PyVector { + #[inline] + fn repr_str(zelf: &Py, _vm: &VirtualMachine) -> PyResult { + Ok(format!("{:#?}", *zelf)) + } +} + +impl AsNumber for PyVector { + fn as_number() -> &'static PyNumberMethods { + // FIXME(discord9): have to use `&PyObject.to_owned()` here + // because it seems to be the only way to convert a `&PyObject` to `PyObjectRef`. + static AS_NUMBER: PyNumberMethods = PyNumberMethods { + and: Some(|a, b, vm| PyVector::and(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)), + or: Some(|a, b, vm| PyVector::or(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)), + invert: Some(|a, vm| PyVector::invert((*a).to_owned(), vm).to_pyresult(vm)), + add: Some(|a, b, vm| PyVector::add(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)), + subtract: Some(|a, b, vm| { + PyVector::sub(a.to_owned(), b.to_owned(), vm).to_pyresult(vm) + }), + multiply: Some(|a, b, vm| { + PyVector::mul(a.to_owned(), b.to_owned(), vm).to_pyresult(vm) + }), + true_divide: Some(|a, b, vm| { + PyVector::truediv(a.to_owned(), b.to_owned(), vm).to_pyresult(vm) + }), + floor_divide: Some(|a, b, vm| { + PyVector::floordiv(a.to_owned(), b.to_owned(), vm).to_pyresult(vm) + }), + ..PyNumberMethods::NOT_IMPLEMENTED + }; + &AS_NUMBER + } +} + impl AsMapping for PyVector { fn as_mapping() -> &'static PyMappingMethods { static AS_MAPPING: PyMappingMethods = PyMappingMethods { diff --git a/src/script/src/python/utils.rs b/src/script/src/python/utils.rs index beeac92923..709e87e74a 100644 --- a/src/script/src/python/utils.rs +++ b/src/script/src/python/utils.rs @@ -15,17 +15,11 @@ use futures::Future; use once_cell::sync::OnceCell; use rustpython_vm::builtins::PyBaseExceptionRef; -use rustpython_vm::{PyObjectRef, PyPayload, VirtualMachine}; +use rustpython_vm::VirtualMachine; use tokio::runtime::Runtime; use crate::python::error; -/// use `rustpython`'s `is_instance` method to check if a PyObject is a instance of class. -/// if `PyResult` is Err, then this function return `false` -pub fn is_instance(obj: &PyObjectRef, vm: &VirtualMachine) -> bool { - obj.is_instance(T::class(vm).into(), vm).unwrap_or(false) -} - pub fn format_py_error(excep: PyBaseExceptionRef, vm: &VirtualMachine) -> error::Error { let mut msg = String::new(); if let Err(e) = vm.write_exception(&mut msg, &excep) {