chore: Update RustPython(With GC) (#809)

* chore: use newest RustPython * chore: use Garbage collected RustPython Fork * style: format toml
2026-01-06 13:22:57 +00:00 · 2022-12-30 16:55:43 +08:00
parent d13de0aeba
commit 6fe205f3b5
4 changed files with 67 additions and 41 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4318,9 +4318,9 @@ dependencies = [

 [[package]]
 name = "once_cell"
-version = "1.16.0"
+version = "1.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860"
+checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"

 [[package]]
 name = "oorandom"
@@ -5766,7 +5766,7 @@ dependencies = [
 [[package]]
 name = "rustpython-ast"
 version = "0.1.0"
-source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8"
+source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6"
 dependencies = [
 "num-bigint",
 "rustpython-common",
@@ -5776,9 +5776,10 @@ dependencies = [
 [[package]]
 name = "rustpython-codegen"
 version = "0.1.2"
-source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8"
+source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6"
 dependencies = [
 "ahash 0.7.6",
+ "bitflags",
 "indexmap",
 "itertools",
 "log",
@@ -5792,7 +5793,7 @@ dependencies = [
 [[package]]
 name = "rustpython-common"
 version = "0.0.0"
-source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8"
+source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6"
 dependencies = [
 "ascii",
 "cfg-if 1.0.0",
@@ -5815,7 +5816,7 @@ dependencies = [
 [[package]]
 name = "rustpython-compiler"
 version = "0.1.2"
-source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8"
+source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6"
 dependencies = [
 "rustpython-codegen",
 "rustpython-compiler-core",
@@ -5826,7 +5827,7 @@ dependencies = [
 [[package]]
 name = "rustpython-compiler-core"
 version = "0.1.2"
-source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8"
+source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6"
 dependencies = [
 "bincode 1.3.3",
 "bitflags",
@@ -5843,7 +5844,17 @@ dependencies = [
 [[package]]
 name = "rustpython-derive"
 version = "0.1.2"
-source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8"
+source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6"
+dependencies = [
+ "rustpython-compiler",
+ "rustpython-derive-impl",
+ "syn",
+]
+
+[[package]]
+name = "rustpython-derive-impl"
+version = "0.0.0"
+source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6"
 dependencies = [
 "indexmap",
 "itertools",
@@ -5851,8 +5862,6 @@ dependencies = [
 "once_cell",
 "proc-macro2",
 "quote",
- "rustpython-codegen",
- "rustpython-compiler",
 "rustpython-compiler-core",
 "rustpython-doc",
 "syn",
@@ -5871,7 +5880,7 @@ dependencies = [
 [[package]]
 name = "rustpython-parser"
 version = "0.1.2"
-source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8"
+source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6"
 dependencies = [
 "ahash 0.7.6",
 "anyhow",
@@ -5883,6 +5892,7 @@ dependencies = [
 "num-traits",
 "phf 0.10.1",
 "phf_codegen 0.10.0",
+ "rustc-hash",
 "rustpython-ast",
 "rustpython-compiler-core",
 "thiserror",
@@ -5895,7 +5905,7 @@ dependencies = [
 [[package]]
 name = "rustpython-pylib"
 version = "0.1.0"
-source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8"
+source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6"
 dependencies = [
 "glob",
 "rustpython-compiler-core",
@@ -5905,7 +5915,7 @@ dependencies = [
 [[package]]
 name = "rustpython-stdlib"
 version = "0.1.2"
-source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8"
+source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6"
 dependencies = [
 "adler32",
 "ahash 0.7.6",
@@ -5933,6 +5943,7 @@ dependencies = [
 "num-bigint",
 "num-complex",
 "num-integer",
+ "num-rational",
 "num-traits",
 "num_enum",
 "once_cell",
@@ -5969,7 +5980,7 @@ dependencies = [
 [[package]]
 name = "rustpython-vm"
 version = "0.1.2"
-source = "git+https://github.com/discord9/RustPython?rev=183e8dab#183e8dabe0027e31630368e36c6be83b5f9cb3f8"
+source = "git+https://github.com/discord9/RustPython?rev=f89b1537#f89b1537b9c789ff566717d1c2ad3d0777cbf5d6"
 dependencies = [
 "adler32",
 "ahash 0.7.6",
@@ -6214,6 +6225,7 @@ dependencies = [
 "common-telemetry",
 "common-time",
 "console",
+ "crossbeam-utils",
 "datafusion",
 "datafusion-common",
 "datafusion-expr",
@@ -6223,6 +6235,7 @@ dependencies = [
 "futures-util",
 "log-store",
 "mito",
+ "once_cell",
 "paste",
 "query",
 "ron",
--- a/src/script/Cargo.toml
+++ b/src/script/Cargo.toml
@@ -33,6 +33,7 @@ common-recordbatch = { path = "../common/recordbatch" }
 common-telemetry = { path = "../common/telemetry" }
 common-time = { path = "../common/time" }
 console = "0.15"
+crossbeam-utils = "0.8.14"
 datafusion = { workspace = true, optional = true }
 datafusion-common = { workspace = true, optional = true }
 datafusion-expr = { workspace = true, optional = true }
@@ -40,19 +41,20 @@ datafusion-physical-expr = { workspace = true, optional = true }
 datatypes = { path = "../datatypes" }
 futures.workspace = true
 futures-util = "0.3"
+once_cell = "1.17.0"
 paste = { workspace = true, optional = true }
 query = { path = "../query" }
 # TODO(discord9): This is a forked and tweaked version of RustPython, please update it to newest original RustPython After Update toolchain to 1.65
-rustpython-ast = { git = "https://github.com/discord9/RustPython", optional = true, rev = "183e8dab" }
-rustpython-codegen = { git = "https://github.com/discord9/RustPython", optional = true, rev = "183e8dab" }
-rustpython-compiler = { git = "https://github.com/discord9/RustPython", optional = true, rev = "183e8dab" }
-rustpython-compiler-core = { git = "https://github.com/discord9/RustPython", optional = true, rev = "183e8dab" }
-rustpython-parser = { git = "https://github.com/discord9/RustPython", optional = true, rev = "183e8dab" }
-rustpython-pylib = { git = "https://github.com/discord9/RustPython", optional = true, rev = "183e8dab", features = [
+rustpython-ast = { git = "https://github.com/discord9/RustPython", optional = true, rev = "f89b1537" }
+rustpython-codegen = { git = "https://github.com/discord9/RustPython", optional = true, rev = "f89b1537" }
+rustpython-compiler = { git = "https://github.com/discord9/RustPython", optional = true, rev = "f89b1537" }
+rustpython-compiler-core = { git = "https://github.com/discord9/RustPython", optional = true, rev = "f89b1537" }
+rustpython-parser = { git = "https://github.com/discord9/RustPython", optional = true, rev = "f89b1537" }
+rustpython-pylib = { git = "https://github.com/discord9/RustPython", optional = true, rev = "f89b1537", features = [
    "freeze-stdlib",
 ] }
-rustpython-stdlib = { git = "https://github.com/discord9/RustPython", optional = true, rev = "183e8dab" }
-rustpython-vm = { git = "https://github.com/discord9/RustPython", optional = true, rev = "183e8dab", features = [
+rustpython-stdlib = { git = "https://github.com/discord9/RustPython", optional = true, rev = "f89b1537" }
+rustpython-vm = { git = "https://github.com/discord9/RustPython", optional = true, rev = "f89b1537", features = [
    "default",
    "codegen",
 ] }
--- a/src/script/src/python/coprocessor/compile.rs
+++ b/src/script/src/python/coprocessor/compile.rs
@@ -25,7 +25,7 @@ use crate::python::coprocessor::parse::{ret_parse_error, DecoratorArgs};
 use crate::python::error::{PyCompileSnafu, PyParseSnafu, Result};

 fn create_located<T>(node: T, loc: Location) -> Located<T> {
-    Located::new(loc, node)
+    Located::new(loc, loc, node)
 }

 /// generate a call to the coprocessor function
--- a/src/script/src/python/vector.rs
+++ b/src/script/src/python/vector.rs
@@ -19,6 +19,7 @@ use std::sync::Arc;
 use common_time::date::Date;
 use common_time::datetime::DateTime;
 use common_time::timestamp::Timestamp;
+use crossbeam_utils::atomic::AtomicCell;
 use datatypes::arrow::array::{
    Array, ArrayRef, BooleanArray, Float64Array, Int64Array, UInt64Array,
 };
@@ -30,13 +31,15 @@ use datatypes::data_type::{ConcreteDataType, DataType};
 use datatypes::prelude::Value;
 use datatypes::value::{self, OrderedFloat};
 use datatypes::vectors::{Helper, NullVector, VectorRef};
+use once_cell::sync::Lazy;
 use rustpython_vm::builtins::{PyBaseExceptionRef, PyBool, PyBytes, PyFloat, PyInt, PyNone, PyStr};
 use rustpython_vm::function::{Either, OptionalArg, PyComparisonValue};
 use rustpython_vm::protocol::{PyMappingMethods, PySequenceMethods};
 use rustpython_vm::sliceable::{SaturatedSlice, SequenceIndex, SequenceIndexOp};
 use rustpython_vm::types::{AsMapping, AsSequence, Comparable, PyComparisonOp};
 use rustpython_vm::{
-    pyclass, AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult, VirtualMachine,
+    atomic_func, pyclass, AsObject, PyObject, PyObjectRef, PyPayload, PyRef, PyResult,
+    VirtualMachine,
 };

 use crate::python::utils::{is_instance, PyVectorRef};
@@ -931,25 +934,33 @@ fn get_concrete_type(obj: &PyObjectRef, vm: &VirtualMachine) -> PyResult<Concret
 }

 impl AsMapping for PyVector {
-    const AS_MAPPING: PyMappingMethods = PyMappingMethods {
-        length: Some(|mapping, _vm| Ok(Self::mapping_downcast(mapping).len())),
-        subscript: Some(|mapping, needle, vm| Self::mapping_downcast(mapping)._getitem(needle, vm)),
-        ass_subscript: None,
-    };
+    fn as_mapping() -> &'static PyMappingMethods {
+        static AS_MAPPING: PyMappingMethods = PyMappingMethods {
+            length: atomic_func!(|mapping, _vm| Ok(PyVector::mapping_downcast(mapping).len())),
+            subscript: atomic_func!(
+                |mapping, needle, vm| PyVector::mapping_downcast(mapping)._getitem(needle, vm)
+            ),
+            ass_subscript: AtomicCell::new(None),
+        };
+        &AS_MAPPING
+    }
 }

 impl AsSequence for PyVector {
-    const AS_SEQUENCE: PySequenceMethods = PySequenceMethods {
-        length: Some(|seq, _vm| Ok(Self::sequence_downcast(seq).len())),
-        item: Some(|seq, i, vm| {
-            let zelf = Self::sequence_downcast(seq);
-            zelf.getitem_by_index(i, vm)
-        }),
-        ass_item: Some(|_seq, _i, _value, vm| {
-            Err(vm.new_type_error("PyVector object doesn't support item assigns".to_owned()))
-        }),
-        ..PySequenceMethods::NOT_IMPLEMENTED
-    };
+    fn as_sequence() -> &'static PySequenceMethods {
+        static AS_SEQUENCE: Lazy<PySequenceMethods> = Lazy::new(|| PySequenceMethods {
+            length: atomic_func!(|seq, _vm| Ok(PyVector::sequence_downcast(seq).len())),
+            item: atomic_func!(|seq, i, vm| {
+                let zelf = PyVector::sequence_downcast(seq);
+                zelf.getitem_by_index(i, vm)
+            }),
+            ass_item: atomic_func!(|_seq, _i, _value, vm| {
+                Err(vm.new_type_error("PyVector object doesn't support item assigns".to_owned()))
+            }),
+            ..PySequenceMethods::NOT_IMPLEMENTED
+        });
+        &AS_SEQUENCE
+    }
 }

 impl Comparable for PyVector {
@@ -1069,7 +1080,7 @@ pub mod tests {
            let a = PyVector::from(a);
            let a = a.into_pyobject(vm);
            assert!(PySequence::find_methods(&a, vm).is_some());
-            assert!(PySequence::new(&a, vm).is_some());
+            assert!(PySequence::try_protocol(&a, vm).is_ok());
        })
    }