chore: update lance dependency to v2.0.0-beta.8 (#2907)

## Summary
- bump Lance crates to v2.0.0-beta.8 and align
arrow/datafusion/regex/half and PyO3 dependencies
- update Rust/Python bindings for upstream API changes (namespace/table
requests, query select columns, storage option providers)
- verified with cargo clippy --workspace --tests --all-features -D
warnings and cargo fmt --all

Triggered by refs/tags/v2.0.0-beta.8.

---------

Signed-off-by: BubbleCal <bubble-cal@outlook.com>
Co-authored-by: BubbleCal <bubble-cal@outlook.com>

python/src/storage_options.rs

@@ -17,11 +17,12 @@ use pyo3::types::PyDict;
 /// Internal wrapper around a Python object implementing StorageOptionsProvider
 pub struct PyStorageOptionsProvider {
     /// The Python object implementing fetch_storage_options()
-    inner: PyObject,
+    inner: Py<PyAny>,
 }
 
 impl Clone for PyStorageOptionsProvider {
     fn clone(&self) -> Self {
+        #[allow(deprecated)]
         Python::with_gil(|py| Self {
             inner: self.inner.clone_ref(py),
         })
@@ -29,7 +30,8 @@ impl Clone for PyStorageOptionsProvider {
 }
 
 impl PyStorageOptionsProvider {
-    pub fn new(obj: PyObject) -> PyResult<Self> {
+    pub fn new(obj: Py<PyAny>) -> PyResult<Self> {
+        #[allow(deprecated)]
         Python::with_gil(|py| {
             // Verify the object has a fetch_storage_options method
             if !obj.bind(py).hasattr("fetch_storage_options")? {
@@ -37,7 +39,9 @@ impl PyStorageOptionsProvider {
                     "StorageOptionsProvider must implement fetch_storage_options() method",
                 ));
             }
-            Ok(Self { inner: obj })
+            Ok(Self {
+                inner: obj.clone_ref(py),
+            })
         })
     }
 }
@@ -60,6 +64,7 @@ impl StorageOptionsProvider for PyStorageOptionsProviderWrapper {
         let py_provider = self.py_provider.clone();
 
         tokio::task::spawn_blocking(move || {
+            #[allow(deprecated)]
             Python::with_gil(|py| {
                 // Call the Python fetch_storage_options method
                 let result = py_provider
@@ -119,6 +124,7 @@ impl StorageOptionsProvider for PyStorageOptionsProviderWrapper {
     }
 
     fn provider_id(&self) -> String {
+        #[allow(deprecated)]
         Python::with_gil(|py| {
             // Call provider_id() method on the Python object
             let obj = self.py_provider.inner.bind(py);
@@ -143,7 +149,7 @@ impl std::fmt::Debug for PyStorageOptionsProviderWrapper {
 /// This is the main entry point for converting Python StorageOptionsProvider objects
 /// to Rust trait objects that can be used by the Lance ecosystem.
 pub fn py_object_to_storage_options_provider(
-    py_obj: PyObject,
+    py_obj: Py<PyAny>,
 ) -> PyResult<Arc<dyn StorageOptionsProvider>> {
     let py_provider = PyStorageOptionsProvider::new(py_obj)?;
     Ok(Arc::new(PyStorageOptionsProviderWrapper::new(py_provider)))