From 6fe205f3b5379582a0e948f83a7887cc8387e0c9 Mon Sep 17 00:00:00 2001 From: discord9 <55937128+discord9@users.noreply.github.com> Date: Fri, 30 Dec 2022 16:55:43 +0800 Subject: [PATCH] chore: Update RustPython(With GC) (#809) * chore: use newest RustPython * chore: use Garbage collected RustPython Fork * style: format toml --- Cargo.lock | 41 +++++++++++------ src/script/Cargo.toml | 18 ++++---- src/script/src/python/coprocessor/compile.rs | 2 +- src/script/src/python/vector.rs | 47 ++++++++++++-------- 4 files changed, 67 insertions(+), 41 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a2eb40eeee..8b9c74195d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4318,9 +4318,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" [[package]] name = "oorandom" @@ -5766,7 +5766,7 @@ dependencies = [ [[package]] name = "rustpython-ast" version = "0.1.0" -source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8" +source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6" dependencies = [ "num-bigint", "rustpython-common", @@ -5776,9 +5776,10 @@ dependencies = [ [[package]] name = "rustpython-codegen" version = "0.1.2" -source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8" +source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6" dependencies = [ "ahash 0.7.6", + "bitflags", "indexmap", "itertools", "log", @@ -5792,7 +5793,7 @@ dependencies = [ [[package]] name = "rustpython-common" version = "0.0.0" -source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8" +source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6" dependencies = [ "ascii", "cfg-if 1.0.0", @@ -5815,7 +5816,7 @@ dependencies = [ [[package]] name = "rustpython-compiler" version = "0.1.2" -source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8" +source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6" dependencies = [ "rustpython-codegen", "rustpython-compiler-core", @@ -5826,7 +5827,7 @@ dependencies = [ [[package]] name = "rustpython-compiler-core" version = "0.1.2" -source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8" +source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6" dependencies = [ "bincode 1.3.3", "bitflags", @@ -5843,7 +5844,17 @@ dependencies = [ [[package]] name = "rustpython-derive" version = "0.1.2" -source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8" +source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6" +dependencies = [ + "rustpython-compiler", + "rustpython-derive-impl", + "syn", +] + +[[package]] +name = "rustpython-derive-impl" +version = "0.0.0" +source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6" dependencies = [ "indexmap", "itertools", @@ -5851,8 +5862,6 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "rustpython-codegen", - "rustpython-compiler", "rustpython-compiler-core", "rustpython-doc", "syn", @@ -5871,7 +5880,7 @@ dependencies = [ [[package]] name = "rustpython-parser" version = "0.1.2" -source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8" +source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6" dependencies = [ "ahash 0.7.6", "anyhow", @@ -5883,6 +5892,7 @@ dependencies = [ "num-traits", "phf 0.10.1", "phf_codegen 0.10.0", + "rustc-hash", "rustpython-ast", "rustpython-compiler-core", "thiserror", @@ -5895,7 +5905,7 @@ dependencies = [ [[package]] name = "rustpython-pylib" version = "0.1.0" -source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8" +source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6" dependencies = [ "glob", "rustpython-compiler-core", @@ -5905,7 +5915,7 @@ dependencies = [ [[package]] name = "rustpython-stdlib" version = "0.1.2" -source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8" +source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6" dependencies = [ "adler32", "ahash 0.7.6", @@ -5933,6 +5943,7 @@ dependencies = [ "num-bigint", "num-complex", "num-integer", + "num-rational", "num-traits", "num_enum", "once_cell", @@ -5969,7 +5980,7 @@ dependencies = [ [[package]] name = "rustpython-vm" version = "0.1.2" -source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8" +source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6" dependencies = [ "adler32", "ahash 0.7.6", @@ -6214,6 +6225,7 @@ dependencies = [ "common-telemetry", "common-time", "console", + "crossbeam-utils", "datafusion", "datafusion-common", "datafusion-expr", @@ -6223,6 +6235,7 @@ dependencies = [ "futures-util", "log-store", "mito", + "once_cell", "paste", "query", "ron", diff --git a/src/script/Cargo.toml b/src/script/Cargo.toml index 12397204d8..45ac4a7264 100644 --- a/src/script/Cargo.toml +++ b/src/script/Cargo.toml @@ -33,6 +33,7 @@ common-recordbatch = { path = "../common/recordbatch" } common-telemetry = { path = "../common/telemetry" } common-time = { path = "../common/time" } console = "0.15" +crossbeam-utils = "0.8.14" datafusion = { workspace = true, optional = true } datafusion-common = { workspace = true, optional = true } datafusion-expr = { workspace = true, optional = true } @@ -40,19 +41,20 @@ datafusion-physical-expr = { workspace = true, optional = true } datatypes = { path = "../datatypes" } futures.workspace = true futures-util = "0.3" +once_cell = "1.17.0" paste = { workspace = true, optional = true } query = { path = "../query" } # TODO(discord9): This is a forked and tweaked version of RustPython, please update it to newest original RustPython After Update toolchain to 1.65 -rustpython-ast = { git = "https://github.com/discord9/RustPython", optional = true, rev = "183e8dab" } -rustpython-codegen = { git = "https://github.com/discord9/RustPython", optional = true, rev = "183e8dab" } -rustpython-compiler = { git = "https://github.com/discord9/RustPython", optional = true, rev = "183e8dab" } -rustpython-compiler-core = { git = "https://github.com/discord9/RustPython", optional = true, rev = "183e8dab" } -rustpython-parser = { git = "https://github.com/discord9/RustPython", optional = true, rev = "183e8dab" } -rustpython-pylib = { git = "https://github.com/discord9/RustPython", optional = true, rev = "183e8dab", features = [ +rustpython-ast = { git = "https://github.com/discord9/RustPython", optional = true, rev = "f89b1537" } +rustpython-codegen = { git = "https://github.com/discord9/RustPython", optional = true, rev = "f89b1537" } +rustpython-compiler = { git = "https://github.com/discord9/RustPython", optional = true, rev = "f89b1537" } +rustpython-compiler-core = { git = "https://github.com/discord9/RustPython", optional = true, rev = "f89b1537" } +rustpython-parser = { git = "https://github.com/discord9/RustPython", optional = true, rev = "f89b1537" } +rustpython-pylib = { git = "https://github.com/discord9/RustPython", optional = true, rev = "f89b1537", features = [ "freeze-stdlib", ] } -rustpython-stdlib = { git = "https://github.com/discord9/RustPython", optional = true, rev = "183e8dab" } -rustpython-vm = { git = "https://github.com/discord9/RustPython", optional = true, rev = "183e8dab", features = [ +rustpython-stdlib = { git = "https://github.com/discord9/RustPython", optional = true, rev = "f89b1537" } +rustpython-vm = { git = "https://github.com/discord9/RustPython", optional = true, rev = "f89b1537", features = [ "default", "codegen", ] } diff --git a/src/script/src/python/coprocessor/compile.rs b/src/script/src/python/coprocessor/compile.rs index ddb3f1a914..ad991c78f1 100644 --- a/src/script/src/python/coprocessor/compile.rs +++ b/src/script/src/python/coprocessor/compile.rs @@ -25,7 +25,7 @@ use crate::python::coprocessor::parse::{ret_parse_error, DecoratorArgs}; use crate::python::error::{PyCompileSnafu, PyParseSnafu, Result}; fn create_located(node: T, loc: Location) -> Located { - Located::new(loc, node) + Located::new(loc, loc, node) } /// generate a call to the coprocessor function diff --git a/src/script/src/python/vector.rs b/src/script/src/python/vector.rs index 7ec61fbd4c..484d826ef1 100644 --- a/src/script/src/python/vector.rs +++ b/src/script/src/python/vector.rs @@ -19,6 +19,7 @@ use std::sync::Arc; use common_time::date::Date; use common_time::datetime::DateTime; use common_time::timestamp::Timestamp; +use crossbeam_utils::atomic::AtomicCell; use datatypes::arrow::array::{ Array, ArrayRef, BooleanArray, Float64Array, Int64Array, UInt64Array, }; @@ -30,13 +31,15 @@ use datatypes::data_type::{ConcreteDataType, DataType}; use datatypes::prelude::Value; use datatypes::value::{self, OrderedFloat}; use datatypes::vectors::{Helper, NullVector, VectorRef}; +use once_cell::sync::Lazy; use rustpython_vm::builtins::{PyBaseExceptionRef, PyBool, PyBytes, PyFloat, PyInt, PyNone, PyStr}; use rustpython_vm::function::{Either, OptionalArg, PyComparisonValue}; use rustpython_vm::protocol::{PyMappingMethods, PySequenceMethods}; use rustpython_vm::sliceable::{SaturatedSlice, SequenceIndex, SequenceIndexOp}; use rustpython_vm::types::{AsMapping, AsSequence, Comparable, PyComparisonOp}; use rustpython_vm::{ - pyclass, AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine, + atomic_func, pyclass, AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, + VirtualMachine, }; use crate::python::utils::{is_instance, PyVectorRef}; @@ -931,25 +934,33 @@ fn get_concrete_type(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult &'static PyMappingMethods { + static AS_MAPPING: PyMappingMethods = PyMappingMethods { + length: atomic_func!(|mapping, _vm| Ok(PyVector::mapping_downcast(mapping).len())), + subscript: atomic_func!( + |mapping, needle, vm| PyVector::mapping_downcast(mapping)._getitem(needle, vm) + ), + ass_subscript: AtomicCell::new(None), + }; + &AS_MAPPING + } } impl AsSequence for PyVector { - const AS_SEQUENCE: PySequenceMethods = PySequenceMethods { - length: Some(|seq, _vm| Ok(Self::sequence_downcast(seq).len())), - item: Some(|seq, i, vm| { - let zelf = Self::sequence_downcast(seq); - zelf.getitem_by_index(i, vm) - }), - ass_item: Some(|_seq, _i, _value, vm| { - Err(vm.new_type_error("PyVector object doesn't support item assigns".to_owned())) - }), - ..PySequenceMethods::NOT_IMPLEMENTED - }; + fn as_sequence() -> &'static PySequenceMethods { + static AS_SEQUENCE: Lazy = Lazy::new(|| PySequenceMethods { + length: atomic_func!(|seq, _vm| Ok(PyVector::sequence_downcast(seq).len())), + item: atomic_func!(|seq, i, vm| { + let zelf = PyVector::sequence_downcast(seq); + zelf.getitem_by_index(i, vm) + }), + ass_item: atomic_func!(|_seq, _i, _value, vm| { + Err(vm.new_type_error("PyVector object doesn't support item assigns".to_owned())) + }), + ..PySequenceMethods::NOT_IMPLEMENTED + }); + &AS_SEQUENCE + } } impl Comparable for PyVector { @@ -1069,7 +1080,7 @@ pub mod tests { let a = PyVector::from(a); let a = a.into_pyobject(vm); assert!(PySequence::find_methods(&a, vm).is_some()); - assert!(PySequence::new(&a, vm).is_some()); + assert!(PySequence::try_protocol(&a, vm).is_ok()); }) }