rust/greptimedb
1
0
Fork 0
mirror of https://github.com/GreptimeTeam/greptimedb.git synced 2026-05-18 14:00:39 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore: update RustPython depend (#1406)

Browse Source 
* chore: update RustPython to newer version

* chore: bump ver

* chore: PR advices
This commit is contained in:
discord9
2023-04-18 15:39:57 +08:00
committed by GitHub
parent 1c65987026
commit de8b889701
16 changed files with 268 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

192
Cargo.lock generated
View File

@@ -1272,7 +1272,7 @@ checksum = "29c39203181991a7dd4343b8005bd804e7a9a37afb8ac070e43771e8c820bbde"
dependencies = [
"chrono",
"chrono-tz-build 0.0.3",
"phf 0.11.1",
"phf",
]
[[package]]
@@ -1283,7 +1283,7 @@ checksum = "cf9cc2b23599e6d7479755f3594285efb3f74a1bdca7a7374948bc831e23a552"
dependencies = [
"chrono",
"chrono-tz-build 0.1.0",
"phf 0.11.1",
"phf",
]
[[package]]
@@ -1293,8 +1293,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f509c3a87b33437b05e2458750a0700e5bdd6956176773e6c7d6dd15a283a0c"
dependencies = [
"parse-zoneinfo",
"phf 0.11.1",
"phf_codegen 0.11.1",
"phf",
"phf_codegen",
]
[[package]]
@@ -1304,8 +1304,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9998fb9f7e9b2111641485bf8beb32f92945f97f92a3d061f744cfef335f751"
dependencies = [
"parse-zoneinfo",
"phf 0.11.1",
"phf_codegen 0.11.1",
"phf",
"phf_codegen",
]
[[package]]
@@ -1526,7 +1526,7 @@ dependencies = [
"partition",
"query",
"rexpect",
"rustyline",
"rustyline 10.1.1",
"serde",
"servers",
"session",
@@ -4427,6 +4427,17 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb"
[[package]]
name = "libsqlite3-sys"
version = "0.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29f835d03d717946d28b1d1ed632eb6f0e24a299388ee623d0c23118d3e8a7fa"
dependencies = [
"cc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "libz-sys"
version = "1.1.8"
@@ -4706,6 +4717,15 @@ dependencies = [
"autocfg",
]
[[package]]
name = "memoffset"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
dependencies = [
"autocfg",
]
[[package]]
name = "memoffset"
version = "0.8.0"
@@ -5102,18 +5122,6 @@ dependencies = [
"memoffset 0.6.5",
]
[[package]]
name = "nix"
version = "0.24.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa52e972a9a719cecb6864fb88568781eb706bac2cd1d4f04a648542dbf78069"
dependencies = [
"bitflags 1.3.2",
"cfg-if 1.0.0",
"libc",
"memoffset 0.6.5",
]
[[package]]
name = "nix"
version = "0.25.1"
@@ -5137,6 +5145,8 @@ dependencies = [
"bitflags 1.3.2",
"cfg-if 1.0.0",
"libc",
"memoffset 0.7.1",
"pin-utils",
"static_assertions",
]
@@ -5183,7 +5193,6 @@ dependencies = [
"autocfg",
"num-integer",
"num-traits",
"serde",
]
[[package]]
@@ -5210,7 +5219,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02e0d21255c828d6f128a1e41534206671e8c3ea0c62f32291e808dc82cff17d"
dependencies = [
"num-traits",
"serde",
]
[[package]]
@@ -5805,15 +5813,6 @@ dependencies = [
"x509-certificate",
]
[[package]]
name = "phf"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259"
dependencies = [
"phf_shared 0.10.0",
]
[[package]]
name = "phf"
version = "0.11.1"
@@ -5823,36 +5822,16 @@ dependencies = [
"phf_shared 0.11.1",
]
[[package]]
name = "phf_codegen"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd"
dependencies = [
"phf_generator 0.10.0",
"phf_shared 0.10.0",
]
[[package]]
name = "phf_codegen"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a56ac890c5e3ca598bbdeaa99964edb5b0258a583a9eb6ef4e89fc85d9224770"
dependencies = [
"phf_generator 0.11.1",
"phf_generator",
"phf_shared 0.11.1",
]
[[package]]
name = "phf_generator"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"
dependencies = [
"phf_shared 0.10.0",
"rand",
]
[[package]]
name = "phf_generator"
version = "0.11.1"
@@ -7121,8 +7100,8 @@ dependencies = [
[[package]]
name = "rustpython-ast"
version = "0.1.0"
source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca"
version = "0.2.0"
source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e"
dependencies = [
"num-bigint",
"rustpython-common",
@@ -7131,8 +7110,8 @@ dependencies = [
[[package]]
name = "rustpython-codegen"
version = "0.1.2"
source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca"
version = "0.2.0"
source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e"
dependencies = [
"ahash 0.7.6",
"bitflags 1.3.2",
@@ -7143,17 +7122,19 @@ dependencies = [
"num-traits",
"rustpython-ast",
"rustpython-compiler-core",
"thiserror",
]
[[package]]
name = "rustpython-common"
version = "0.0.0"
source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca"
version = "0.2.0"
source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e"
dependencies = [
"ascii",
"bitflags 1.3.2",
"bstr 0.2.17",
"cfg-if 1.0.0",
"hexf-parse",
"itertools",
"lexical-parse-float",
"libc",
"lock_api",
@@ -7171,36 +7152,31 @@ dependencies = [
[[package]]
name = "rustpython-compiler"
version = "0.1.2"
source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca"
version = "0.2.0"
source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e"
dependencies = [
"rustpython-codegen",
"rustpython-compiler-core",
"rustpython-parser",
"thiserror",
]
[[package]]
name = "rustpython-compiler-core"
version = "0.1.2"
source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca"
version = "0.2.0"
source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e"
dependencies = [
"bincode",
"bitflags 1.3.2",
"bstr 0.2.17",
"itertools",
"lz4_flex",
"num-bigint",
"num-complex",
"serde",
"static_assertions",
"thiserror",
]
[[package]]
name = "rustpython-derive"
version = "0.1.2"
source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca"
version = "0.2.0"
source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e"
dependencies = [
"rustpython-compiler",
"rustpython-derive-impl",
@@ -7209,8 +7185,8 @@ dependencies = [
[[package]]
name = "rustpython-derive-impl"
version = "0.0.0"
source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca"
version = "0.2.0"
source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e"
dependencies = [
"indexmap",
"itertools",
@@ -7235,8 +7211,8 @@ dependencies = [
[[package]]
name = "rustpython-parser"
version = "0.1.2"
source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca"
version = "0.2.0"
source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e"
dependencies = [
"ahash 0.7.6",
"anyhow",
@@ -7246,12 +7222,11 @@ dependencies = [
"log",
"num-bigint",
"num-traits",
"phf 0.10.1",
"phf_codegen 0.10.0",
"phf",
"phf_codegen",
"rustc-hash",
"rustpython-ast",
"rustpython-compiler-core",
"thiserror",
"tiny-keccak",
"unic-emoji-char",
"unic-ucd-ident",
@@ -7260,8 +7235,8 @@ dependencies = [
[[package]]
name = "rustpython-pylib"
version = "0.1.0"
source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca"
version = "0.2.0"
source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e"
dependencies = [
"glob",
"rustpython-compiler-core",
@@ -7270,8 +7245,8 @@ dependencies = [
[[package]]
name = "rustpython-stdlib"
version = "0.1.2"
source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca"
version = "0.2.0"
source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e"
dependencies = [
"adler32",
"ahash 0.7.6",
@@ -7284,18 +7259,19 @@ dependencies = [
"csv-core",
"digest",
"dns-lookup",
"dyn-clone",
"flate2",
"gethostname",
"hex",
"itertools",
"lexical-parse-float",
"libc",
"libsqlite3-sys",
"mac_address",
"md-5",
"memchr",
"memmap2",
"mt19937",
"nix 0.24.3",
"nix 0.26.2",
"num-bigint",
"num-complex",
"num-integer",
@@ -7319,6 +7295,7 @@ dependencies = [
"socket2 0.4.9",
"system-configuration",
"termios",
"ucd",
"unic-char-property",
"unic-normal",
"unic-ucd-age",
@@ -7335,10 +7312,9 @@ dependencies = [
[[package]]
name = "rustpython-vm"
version = "0.1.2"
source = "git+https://github.com/discord9/RustPython?rev=2e126345#2e12634569d01674724490193eb9638f056e51ca"
version = "0.2.0"
source = "git+https://github.com/discord9/RustPython?rev=9ed5137412#9ed51374125b5f1a9e5cee5dd7e27023b8591f1e"
dependencies = [
"adler32",
"ahash 0.7.6",
"ascii",
"atty",
@@ -7349,12 +7325,10 @@ dependencies = [
"chrono",
"crossbeam-utils",
"exitcode",
"flate2",
"getrandom",
"glob",
"half 1.8.2",
"hex",
"hexf-parse",
"indexmap",
"is-macro",
"itertools",
@@ -7362,7 +7336,7 @@ dependencies = [
"log",
"memchr",
"memoffset 0.6.5",
"nix 0.24.3",
"nix 0.26.2",
"num-bigint",
"num-complex",
"num-integer",
@@ -7384,9 +7358,8 @@ dependencies = [
"rustpython-compiler-core",
"rustpython-derive",
"rustpython-parser",
"rustyline",
"rustyline 11.0.0",
"schannel",
"serde",
"sre-engine",
"static_assertions",
"strum",
@@ -7437,6 +7410,29 @@ dependencies = [
"winapi",
]
[[package]]
name = "rustyline"
version = "11.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5dfc8644681285d1fb67a467fb3021bfea306b99b4146b166a1fe3ada965eece"
dependencies = [
"bitflags 1.3.2",
"cfg-if 1.0.0",
"clipboard-win",
"dirs-next",
"fd-lock",
"libc",
"log",
"memchr",
"nix 0.26.2",
"radix_trie",
"scopeguard",
"unicode-segmentation",
"unicode-width",
"utf8parse",
"winapi",
]
[[package]]
name = "ryu"
version = "1.0.13"
@@ -8999,7 +8995,7 @@ dependencies = [
"log",
"parking_lot",
"percent-encoding",
"phf 0.11.1",
"phf",
"pin-project-lite",
"postgres-protocol",
"postgres-types",
@@ -9531,6 +9527,12 @@ dependencies = [
"typify-impl 0.0.11",
]
[[package]]
name = "ucd"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe4fa6e588762366f1eb4991ce59ad1b93651d0b769dfb4e4d1c5c4b943d1159"
[[package]]
name = "ucd-trie"
version = "0.1.5"
@@ -9733,9 +9735,11 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"
[[package]]
name = "unicode_names2"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "029df4cc8238cefc911704ff8fa210853a0f3bce2694d8f51181dd41ee0f3301"
version = "0.6.0"
source = "git+https://github.com/youknowone/unicode_names2.git?rev=4ce16aa85cbcdd9cc830410f1a72ef9a235f2fde#4ce16aa85cbcdd9cc830410f1a72ef9a235f2fde"
dependencies = [
"phf",
]
[[package]]
name = "unindent"

18
src/script/Cargo.toml
View File

@@ -46,17 +46,17 @@ futures-util.workspace = true
once_cell = "1.17.0"
paste = { workspace = true, optional = true }
query = { path = "../query" }
# TODO(discord9): This is a forked and tweaked version of RustPython, please update it to newest original RustPython After Update toolchain to 1.65
rustpython-ast = { git = "https://github.com/discord9/RustPython", optional = true, rev = "2e126345" }
rustpython-codegen = { git = "https://github.com/discord9/RustPython", optional = true, rev = "2e126345" }
rustpython-compiler = { git = "https://github.com/discord9/RustPython", optional = true, rev = "2e126345" }
rustpython-compiler-core = { git = "https://github.com/discord9/RustPython", optional = true, rev = "2e126345" }
rustpython-parser = { git = "https://github.com/discord9/RustPython", optional = true, rev = "2e126345" }
rustpython-pylib = { git = "https://github.com/discord9/RustPython", optional = true, rev = "2e126345", features = [
# TODO(discord9): This is a forked and tweaked version of RustPython, please update it to newest original RustPython After RustPython support GC
rustpython-ast = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412" }
rustpython-codegen = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412" }
rustpython-compiler = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412" }
rustpython-compiler-core = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412" }
rustpython-parser = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412" }
rustpython-pylib = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412", features = [
"freeze-stdlib",
] }
rustpython-stdlib = { git = "https://github.com/discord9/RustPython", optional = true, rev = "2e126345" }
rustpython-vm = { git = "https://github.com/discord9/RustPython", optional = true, rev = "2e126345", features = [
rustpython-stdlib = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412" }
rustpython-vm = { git = "https://github.com/discord9/RustPython", optional = true, rev = "9ed5137412", features = [
"default",
"codegen",
] }

2
src/script/src/python/error.rs
View File

@@ -20,7 +20,7 @@ use datatypes::error::Error as DataTypeError;
use query::error::Error as QueryError;
use rustpython_codegen::error::CodegenError;
use rustpython_parser::ast::Location;
use rustpython_parser::error::ParseError;
use rustpython_parser::ParseError;
pub use snafu::ensure;
use snafu::prelude::Snafu;
use snafu::Location as SnafuLocation;

5
src/script/src/python/ffi_types/copr/compile.rs
View File

@@ -17,7 +17,7 @@ use rustpython_codegen::compile::compile_top;
use rustpython_compiler::{CompileOpts, Mode};
use rustpython_compiler_core::CodeObject;
use rustpython_parser::ast::{ArgData, Located, Location};
use rustpython_parser::{ast, parser};
use rustpython_parser::{ast, parse, Mode as ParseMode};
use snafu::ResultExt;
use crate::fail_parse_error;
@@ -95,8 +95,7 @@ pub fn compile_script(
script: &str,
) -> Result<CodeObject> {
// note that it's important to use `parser::Mode::Interactive` so the ast can be compile to return a result instead of return None in eval mode
let mut top =
parser::parse(script, parser::Mode::Interactive, "<embedded>").context(PyParseSnafu)?;
let mut top = parse(script, ParseMode::Interactive, "<embedded>").context(PyParseSnafu)?;
// erase decorator
if let ast::Mod::Interactive { body } = &mut top {
let stmts = body;

4
src/script/src/python/ffi_types/copr/parse.rs
View File

@@ -18,7 +18,7 @@ use std::sync::Arc;
use datatypes::prelude::ConcreteDataType;
use query::QueryEngineRef;
use rustpython_parser::ast::{Arguments, Location};
use rustpython_parser::{ast, parser};
use rustpython_parser::{ast, parse_program};
#[cfg(test)]
use serde::Deserialize;
use snafu::{OptionExt, ResultExt};
@@ -443,7 +443,7 @@ pub fn parse_and_compile_copr(
script: &str,
query_engine: Option<QueryEngineRef>,
) -> Result<Coprocessor> {
let python_ast = parser::parse_program(script, "<embedded>").context(PyParseSnafu)?;
let python_ast = parse_program(script, "<embedded>").context(PyParseSnafu)?;
let mut coprocessor = None;

2
src/script/src/python/ffi_types/vector.rs
View File

@@ -38,7 +38,7 @@ use rustpython_vm::{
VirtualMachine,
};
use crate::python::utils::is_instance;
use crate::python::rspython::utils::is_instance;
/// The Main FFI type `PyVector` that is used both in RustPython and PyO3
#[cfg_attr(feature = "pyo3_backend", pyo3class(name = "vector"))]

15
src/script/src/python/ffi_types/vector/tests.rs
View File

@@ -155,8 +155,10 @@ fn eval_pyo3(testcase: TestCase, locals: HashMap<String, PyVector>) {
}
fn eval_rspy(testcase: TestCase, locals: HashMap<String, PyVector>) {
vm::Interpreter::without_stdlib(Default::default()).enter(|vm| {
vm::Interpreter::with_init(Default::default(), |vm| {
PyVector::make_class(&vm.ctx);
})
.enter(|vm| {
let scope = vm.new_scope_with_builtins();
locals.into_iter().for_each(|(k, v)| {
scope
@@ -166,10 +168,17 @@ fn eval_rspy(testcase: TestCase, locals: HashMap<String, PyVector>) {
.unwrap();
});
let code_obj = vm
.compile(&testcase.eval, Mode::Eval, "<embedded>".to_owned())
.compile(&testcase.eval, Mode::Eval, "<embedded>".to_string())
.map_err(|err| vm.new_syntax_error(&err))
.unwrap();
let obj = vm.run_code_obj(code_obj, scope).unwrap();
let obj = vm
.run_code_obj(code_obj, scope)
.map_err(|e| {
let mut output = String::new();
vm.write_exception(&mut output, &e).unwrap();
(e, output)
})
.unwrap();
let v = obj.downcast::<PyVector>().unwrap();
let result_arr = v.to_arrow_array();
let expect_arr = testcase.result.to_arrow_array();

2
src/script/src/python/rspython.rs
View File

@@ -19,7 +19,7 @@ pub(crate) mod vector_impl;
pub(crate) mod builtins;
mod dataframe_impl;
mod utils;
pub(crate) mod utils;
#[cfg(test)]
pub(crate) use copr_impl::init_interpreter;

6
src/script/src/python/rspython/builtins.rs
View File

@@ -30,7 +30,7 @@ use rustpython_vm::builtins::{PyBaseExceptionRef, PyBool, PyFloat, PyInt, PyList
use rustpython_vm::{pymodule, AsObject, PyObjectRef, PyPayload, PyResult, VirtualMachine};
use crate::python::ffi_types::PyVector;
use crate::python::utils::is_instance;
use crate::python::rspython::utils::is_instance;
pub fn init_greptime_builtins(module_name: &str, vm: &mut VirtualMachine) {
vm.add_native_module(
@@ -356,7 +356,7 @@ pub(crate) mod greptime_builtin {
#[pyfunction]
fn col(name: String, vm: &VirtualMachine) -> PyExprRef {
let expr: PyExpr = DfExpr::Column(datafusion_common::Column::from_name(name)).into();
expr.into_ref(vm)
expr.into_ref(&vm.ctx)
}
#[pyfunction]
@@ -366,7 +366,7 @@ pub(crate) mod greptime_builtin {
.try_to_scalar_value(&val.data_type())
.map_err(|e| vm.new_runtime_error(format!("{e}")))?;
let expr: PyExpr = DfExpr::Literal(scalar_val).into();
Ok(expr.into_ref(vm))
Ok(expr.into_ref(&vm.ctx))
}
// the main binding code, due to proc macro things, can't directly use a simpler macro

4
src/script/src/python/rspython/builtins/test.rs
View File

@@ -33,8 +33,8 @@ use serde::{Deserialize, Serialize};
use super::*;
use crate::python::ffi_types::PyVector;
use crate::python::utils::{format_py_error, is_instance};
use crate::python::rspython::utils::is_instance;
use crate::python::utils::format_py_error;
#[test]
fn convert_scalar_to_py_obj_and_back() {
rustpython_vm::Interpreter::with_init(Default::default(), |vm| {

2
src/script/src/python/rspython/copr_impl.rs
View File

@@ -213,7 +213,7 @@ pub(crate) fn init_interpreter() -> Arc<Interpreter> {
// so according to this issue:
// https://github.com/RustPython/RustPython/issues/4292
// add this line for stdlib, so rustpython can found stdlib's python part in bytecode format
vm.add_frozen(rustpython_pylib::frozen_stdlib());
vm.add_frozen(rustpython_pylib::FROZEN_STDLIB);
// add our own custom datatype and module
PyVector::make_class(&vm.ctx);
PyQueryEngine::make_class(&vm.ctx);

37
src/script/src/python/rspython/dataframe_impl.rs
View File

@@ -28,8 +28,10 @@ pub(crate) mod data_frame {
use datafusion::dataframe::DataFrame as DfDataFrame;
use datafusion::execution::context::SessionContext;
use datafusion_expr::Expr as DfExpr;
use rustpython_vm::convert::ToPyResult;
use rustpython_vm::function::PyComparisonValue;
use rustpython_vm::types::{Comparable, PyComparisonOp};
use rustpython_vm::protocol::PyNumberMethods;
use rustpython_vm::types::{AsNumber, Comparable, PyComparisonOp};
use rustpython_vm::{
pyclass as rspyclass, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
};
@@ -40,6 +42,7 @@ pub(crate) mod data_frame {
use crate::python::rspython::builtins::greptime_builtin::{
lit, query as get_query_engine, PyDataFrame,
};
use crate::python::rspython::utils::obj_cast_to;
use crate::python::utils::block_on_async;
impl From<DfDataFrame> for PyDataFrame {
@@ -313,7 +316,20 @@ pub(crate) mod data_frame {
}
}
#[rspyclass(with(Comparable))]
impl AsNumber for PyExpr {
fn as_number() -> &'static PyNumberMethods {
static AS_NUMBER: PyNumberMethods = PyNumberMethods {
and: Some(|a, b, vm| PyExpr::and(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)),
or: Some(|a, b, vm| PyExpr::or(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)),
invert: Some(|a, vm| PyExpr::invert((*a).to_owned(), vm).to_pyresult(vm)),
..PyNumberMethods::NOT_IMPLEMENTED
};
&AS_NUMBER
}
}
#[rspyclass(with(Comparable, AsNumber))]
impl PyExpr {
fn richcompare(
&self,
@@ -342,18 +358,23 @@ pub(crate) mod data_frame {
}
#[pymethod(magic)]
fn and(&self, other: PyExprRef) -> PyResult<PyExpr> {
Ok(self.inner.clone().and(other.inner.clone()).into())
fn and(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyExpr> {
let zelf = obj_cast_to::<Self>(zelf, vm)?;
let other = obj_cast_to::<Self>(other, vm)?;
Ok(zelf.inner.clone().and(other.inner.clone()).into())
}
#[pymethod(magic)]
fn or(&self, other: PyExprRef) -> PyResult<PyExpr> {
Ok(self.inner.clone().or(other.inner.clone()).into())
fn or(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyExpr> {
let zelf = obj_cast_to::<Self>(zelf, vm)?;
let other = obj_cast_to::<Self>(other, vm)?;
Ok(zelf.inner.clone().or(other.inner.clone()).into())
}
/// `~` operator, return `!self`
#[pymethod(magic)]
fn invert(&self) -> PyResult<PyExpr> {
Ok(self.inner.clone().not().into())
fn invert(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyExpr> {
let zelf = obj_cast_to::<Self>(zelf, vm)?;
Ok(zelf.inner.clone().not().into())
}
/// sort ascending&nulls_first

4
src/script/src/python/rspython/test.rs
View File

@@ -25,7 +25,7 @@ use datatypes::data_type::{ConcreteDataType, DataType};
use datatypes::schema::{ColumnSchema, Schema};
use datatypes::vectors::{Float32Vector, Float64Vector, Int64Vector, VectorRef};
use ron::from_str as from_ron_string;
use rustpython_parser::parser;
use rustpython_parser::{parse, Mode};
use serde::{Deserialize, Serialize};
use crate::python::error::{get_error_reason_loc, pretty_print_error_in_src, visualize_loc, Error};
@@ -181,7 +181,7 @@ fn test_type_anno() {
def a(cpu, mem: vector[f64])->(vector[f64|None], vector[f64], vector[_], vector[ _ | None]):
return cpu + mem, cpu - mem, cpu * mem, cpu / mem
"#;
let pyast = parser::parse(python_source, parser::Mode::Interactive, "<embedded>").unwrap();
let pyast = parse(python_source, Mode::Interactive, "<embedded>").unwrap();
let copr = parse_and_compile_copr(python_source, None);
dbg!(copr);
}

19
src/script/src/python/rspython/utils.rs
View File

@@ -22,7 +22,8 @@ use datatypes::vectors::{
BooleanVector, Float64Vector, Helper, Int64Vector, NullVector, StringVector, VectorRef,
};
use rustpython_vm::builtins::{PyBaseExceptionRef, PyBool, PyFloat, PyInt, PyList, PyStr};
use rustpython_vm::{PyObjectRef, PyPayload, PyResult, VirtualMachine};
use rustpython_vm::object::PyObjectPayload;
use rustpython_vm::{PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine};
use snafu::{OptionExt, ResultExt};
use crate::python::error;
@@ -33,7 +34,21 @@ use crate::python::rspython::builtins::try_into_columnar_value;
/// use `rustpython`'s `is_instance` method to check if a PyObject is a instance of class.
/// if `PyResult` is Err, then this function return `false`
pub fn is_instance<T: PyPayload>(obj: &PyObjectRef, vm: &VirtualMachine) -> bool {
obj.is_instance(T::class(vm).into(), vm).unwrap_or(false)
obj.is_instance(T::class(&vm.ctx).into(), vm)
.unwrap_or(false)
}
pub fn obj_cast_to<T: PyObjectPayload>(
obj: PyObjectRef,
vm: &VirtualMachine,
) -> PyResult<PyRef<T>> {
obj.downcast::<T>().map_err(|e| {
vm.new_type_error(format!(
"Can't cast object into {}, actual type: {}",
std::any::type_name::<T>(),
e.class().name()
))
})
}
pub fn format_py_error(excep: PyBaseExceptionRef, vm: &VirtualMachine) -> error::Error {

117
src/script/src/python/rspython/vector_impl.rs
View File

@@ -25,18 +25,21 @@ use datatypes::value::{self, OrderedFloat};
use datatypes::vectors::Helper;
use once_cell::sync::Lazy;
use rustpython_vm::builtins::{PyBaseExceptionRef, PyBool, PyBytes, PyFloat, PyInt, PyNone, PyStr};
use rustpython_vm::convert::ToPyResult;
use rustpython_vm::function::{Either, OptionalArg, PyComparisonValue};
use rustpython_vm::protocol::{PyMappingMethods, PySequenceMethods};
use rustpython_vm::types::{AsMapping, AsSequence, Comparable, PyComparisonOp};
use rustpython_vm::protocol::{PyMappingMethods, PyNumberMethods, PySequenceMethods};
use rustpython_vm::types::{
AsMapping, AsNumber, AsSequence, Comparable, PyComparisonOp, Representable,
};
use rustpython_vm::{
atomic_func, pyclass as rspyclass, PyObject, PyObjectRef, PyPayload, PyRef, PyResult,
atomic_func, pyclass as rspyclass, Py, PyObject, PyObjectRef, PyPayload, PyRef, PyResult,
VirtualMachine,
};
use crate::python::ffi_types::vector::{
arrow_rfloordiv, arrow_rsub, arrow_rtruediv, rspy_is_pyobj_scalar, wrap_result, PyVector,
};
use crate::python::utils::is_instance;
use crate::python::rspython::utils::{is_instance, obj_cast_to};
/// PyVectors' rustpython specify methods
fn to_type_error(vm: &'_ VirtualMachine) -> impl FnOnce(String) -> PyBaseExceptionRef + '_ {
@@ -45,7 +48,7 @@ fn to_type_error(vm: &'_ VirtualMachine) -> impl FnOnce(String) -> PyBaseExcepti
pub(crate) type PyVectorRef = PyRef<PyVector>;
/// PyVector type wraps a greptime vector, impl multiply/div/add/sub opeerators etc.
#[rspyclass(with(AsMapping, AsSequence, Comparable))]
#[rspyclass(with(AsMapping, AsSequence, Comparable, AsNumber, Representable))]
impl PyVector {
#[pymethod]
pub(crate) fn new(
@@ -86,29 +89,32 @@ impl PyVector {
#[pymethod(name = "__radd__")]
#[pymethod(magic)]
fn add(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyVector> {
fn add(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyVector> {
let zelf = obj_cast_to::<PyVector>(zelf, vm)?;
if rspy_is_pyobj_scalar(&other, vm) {
self.rspy_scalar_arith_op(other, None, wrap_result(arithmetic::add_dyn), vm)
zelf.rspy_scalar_arith_op(other, None, wrap_result(arithmetic::add_dyn), vm)
} else {
self.rspy_vector_arith_op(other, None, wrap_result(arithmetic::add_dyn), vm)
zelf.rspy_vector_arith_op(other, None, wrap_result(arithmetic::add_dyn), vm)
}
}
#[pymethod(magic)]
fn sub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyVector> {
fn sub(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyVector> {
let zelf = obj_cast_to::<PyVector>(zelf, vm)?;
if rspy_is_pyobj_scalar(&other, vm) {
self.rspy_scalar_arith_op(other, None, wrap_result(arithmetic::subtract_dyn), vm)
zelf.rspy_scalar_arith_op(other, None, wrap_result(arithmetic::subtract_dyn), vm)
} else {
self.rspy_vector_arith_op(other, None, wrap_result(arithmetic::subtract_dyn), vm)
zelf.rspy_vector_arith_op(other, None, wrap_result(arithmetic::subtract_dyn), vm)
}
}
#[pymethod(magic)]
fn rsub(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyVector> {
fn rsub(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyVector> {
let zelf = obj_cast_to::<PyVector>(zelf, vm)?;
if rspy_is_pyobj_scalar(&other, vm) {
self.rspy_scalar_arith_op(other, None, arrow_rsub, vm)
zelf.rspy_scalar_arith_op(other, None, arrow_rsub, vm)
} else {
self.rspy_vector_arith_op(
zelf.rspy_vector_arith_op(
other,
None,
wrap_result(|a, b| arithmetic::subtract_dyn(b, a)),
@@ -119,25 +125,27 @@ impl PyVector {
#[pymethod(name = "__rmul__")]
#[pymethod(magic)]
fn mul(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyVector> {
fn mul(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyVector> {
let zelf = obj_cast_to::<PyVector>(zelf, vm)?;
if rspy_is_pyobj_scalar(&other, vm) {
self.rspy_scalar_arith_op(other, None, wrap_result(arithmetic::multiply_dyn), vm)
zelf.rspy_scalar_arith_op(other, None, wrap_result(arithmetic::multiply_dyn), vm)
} else {
self.rspy_vector_arith_op(other, None, wrap_result(arithmetic::multiply_dyn), vm)
zelf.rspy_vector_arith_op(other, None, wrap_result(arithmetic::multiply_dyn), vm)
}
}
#[pymethod(magic)]
fn truediv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyVector> {
fn truediv(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyVector> {
let zelf = obj_cast_to::<PyVector>(zelf, vm)?;
if rspy_is_pyobj_scalar(&other, vm) {
self.rspy_scalar_arith_op(
zelf.rspy_scalar_arith_op(
other,
Some(ArrowDataType::Float64),
wrap_result(arithmetic::divide_dyn),
vm,
)
} else {
self.rspy_vector_arith_op(
zelf.rspy_vector_arith_op(
other,
Some(ArrowDataType::Float64),
wrap_result(arithmetic::divide_dyn),
@@ -161,16 +169,17 @@ impl PyVector {
}
#[pymethod(magic)]
fn floordiv(&self, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyVector> {
fn floordiv(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyVector> {
let zelf = obj_cast_to::<PyVector>(zelf, vm)?;
if rspy_is_pyobj_scalar(&other, vm) {
self.rspy_scalar_arith_op(
zelf.rspy_scalar_arith_op(
other,
Some(ArrowDataType::Int64),
wrap_result(arithmetic::divide_dyn),
vm,
)
} else {
self.rspy_vector_arith_op(
zelf.rspy_vector_arith_op(
other,
Some(ArrowDataType::Int64),
wrap_result(arithmetic::divide_dyn),
@@ -194,19 +203,33 @@ impl PyVector {
}
}
#[pymethod(magic)]
fn and(&self, other: PyVectorRef, vm: &VirtualMachine) -> PyResult<PyVector> {
Self::vector_and(self, &other).map_err(to_type_error(vm))
fn obj_to_vector(obj: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyRef<PyVector>> {
obj.downcast::<PyVector>().map_err(|e| {
vm.new_type_error(format!(
"Can't cast right operand into PyVector, actual type: {}",
e.class().name()
))
})
}
#[pymethod(magic)]
fn or(&self, other: PyVectorRef, vm: &VirtualMachine) -> PyResult<PyVector> {
Self::vector_or(self, &other).map_err(to_type_error(vm))
fn and(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyVector> {
let zelf = Self::obj_to_vector(zelf, vm)?;
let other = Self::obj_to_vector(other, vm)?;
Self::vector_and(&zelf, &other).map_err(to_type_error(vm))
}
#[pymethod(magic)]
fn invert(&self, vm: &VirtualMachine) -> PyResult<PyVector> {
Self::vector_invert(self).map_err(to_type_error(vm))
fn or(zelf: PyObjectRef, other: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyVector> {
let zelf = Self::obj_to_vector(zelf, vm)?;
let other = Self::obj_to_vector(other, vm)?;
Self::vector_or(&zelf, &other).map_err(to_type_error(vm))
}
#[pymethod(magic)]
fn invert(zelf: PyObjectRef, vm: &VirtualMachine) -> PyResult<PyVector> {
let zelf = Self::obj_to_vector(zelf, vm)?;
Self::vector_invert(&zelf).map_err(to_type_error(vm))
}
#[pymethod(name = "__len__")]
@@ -262,6 +285,40 @@ impl PyVector {
}
}
impl Representable for PyVector {
#[inline]
fn repr_str(zelf: &Py<Self>, _vm: &VirtualMachine) -> PyResult<String> {
Ok(format!("{:#?}", *zelf))
}
}
impl AsNumber for PyVector {
fn as_number() -> &'static PyNumberMethods {
// FIXME(discord9): have to use `&PyObject.to_owned()` here
// because it seems to be the only way to convert a `&PyObject` to `PyObjectRef`.
static AS_NUMBER: PyNumberMethods = PyNumberMethods {
and: Some(|a, b, vm| PyVector::and(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)),
or: Some(|a, b, vm| PyVector::or(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)),
invert: Some(|a, vm| PyVector::invert((*a).to_owned(), vm).to_pyresult(vm)),
add: Some(|a, b, vm| PyVector::add(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)),
subtract: Some(|a, b, vm| {
PyVector::sub(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)
}),
multiply: Some(|a, b, vm| {
PyVector::mul(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)
}),
true_divide: Some(|a, b, vm| {
PyVector::truediv(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)
}),
floor_divide: Some(|a, b, vm| {
PyVector::floordiv(a.to_owned(), b.to_owned(), vm).to_pyresult(vm)
}),
..PyNumberMethods::NOT_IMPLEMENTED
};
&AS_NUMBER
}
}
impl AsMapping for PyVector {
fn as_mapping() -> &'static PyMappingMethods {
static AS_MAPPING: PyMappingMethods = PyMappingMethods {

8
src/script/src/python/utils.rs
View File

@@ -15,17 +15,11 @@
use futures::Future;
use once_cell::sync::OnceCell;
use rustpython_vm::builtins::PyBaseExceptionRef;
use rustpython_vm::{PyObjectRef, PyPayload, VirtualMachine};
use rustpython_vm::VirtualMachine;
use tokio::runtime::Runtime;
use crate::python::error;
/// use `rustpython`'s `is_instance` method to check if a PyObject is a instance of class.
/// if `PyResult` is Err, then this function return `false`
pub fn is_instance<T: PyPayload>(obj: &PyObjectRef, vm: &VirtualMachine) -> bool {
obj.is_instance(T::class(vm).into(), vm).unwrap_or(false)
}
pub fn format_py_error(excep: PyBaseExceptionRef, vm: &VirtualMachine) -> error::Error {
let mut msg = String::new();
if let Err(e) = vm.write_exception(&mut msg, &excep) {