fix(python): use namespace-backed rust connection for namespace tables (#3286)

So far, I have been using a hacky approach that creates and opens
namespace-backed table, by getting its location and use a temporary
lancedb connection to create or open it. This was working for features
like credentials vending but is no longer fully working for the managed
versioning feature, recently geneva tests have been failing here and
there and various patches are not addressing the root cause. This PR
fully fixes this and implements proper rust binding for it.
Specifically:

- build a real Rust namespace-backed connection from the Python
namespace client
- route namespace table create/open through that connection instead of
resolved-location temp connections
- keep namespace client naming consistent in the Rust bridge and
preserve federated namespace + DuckDB behavior

python/src/connection.rs

@@ -1,11 +1,17 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
-use std::{collections::HashMap, sync::Arc, time::Duration};
+use std::{
+    collections::{HashMap, HashSet},
+    sync::Arc,
+    time::Duration,
+};
 
 use arrow::{datatypes::Schema, ffi_stream::ArrowArrayStreamReader, pyarrow::FromPyArrow};
 use lancedb::{
     connection::Connection as LanceConnection,
+    connection::NamespaceClientPushdownOperation,
+    database::namespace::LanceNamespaceDatabase,
     database::{CreateTableMode, Database, ReadConsistency},
 };
 use pyo3::{
@@ -39,6 +45,29 @@ impl Connection {
     }
 }
 
+fn parse_namespace_client_pushdown_operations(
+    operations: Option<Vec<String>>,
+) -> PyResult<HashSet<NamespaceClientPushdownOperation>> {
+    let mut parsed = HashSet::new();
+    for operation in operations.unwrap_or_default() {
+        match operation.as_str() {
+            "QueryTable" => {
+                parsed.insert(NamespaceClientPushdownOperation::QueryTable);
+            }
+            "CreateTable" => {
+                parsed.insert(NamespaceClientPushdownOperation::CreateTable);
+            }
+            _ => {
+                return Err(PyValueError::new_err(format!(
+                    "Invalid pushdown operation: {}",
+                    operation
+                )));
+            }
+        }
+    }
+    Ok(parsed)
+}
+
 impl Connection {
     fn parse_create_mode_str(mode: &str) -> PyResult<CreateTableMode> {
         match mode {
@@ -538,6 +567,52 @@ pub fn connect(
     })
 }
 
+#[pyfunction]
+#[pyo3(signature = (
+    namespace_client,
+    read_consistency_interval=None,
+    storage_options=None,
+    session=None,
+    namespace_client_pushdown_operations=None,
+    namespace_client_impl=None,
+    namespace_client_properties=None,
+))]
+#[allow(clippy::too_many_arguments)]
+pub fn connect_namespace_client(
+    py: Python<'_>,
+    namespace_client: Py<PyAny>,
+    read_consistency_interval: Option<f64>,
+    storage_options: Option<HashMap<String, String>>,
+    session: Option<crate::session::Session>,
+    namespace_client_pushdown_operations: Option<Vec<String>>,
+    namespace_client_impl: Option<String>,
+    namespace_client_properties: Option<HashMap<String, String>>,
+) -> PyResult<Connection> {
+    let namespace_client = extract_namespace_arc(py, namespace_client)?;
+    let read_consistency_interval = read_consistency_interval.map(Duration::from_secs_f64);
+    let namespace_client_pushdown_operations =
+        parse_namespace_client_pushdown_operations(namespace_client_pushdown_operations)?;
+    let ns_impl = namespace_client_impl.unwrap_or_else(|| "python".to_string());
+    let ns_properties = namespace_client_properties.unwrap_or_default();
+    let storage_options = storage_options.unwrap_or_default();
+    let session = session.map(|s| s.inner.clone());
+
+    let database = LanceNamespaceDatabase::from_namespace_client(
+        namespace_client,
+        ns_impl,
+        ns_properties,
+        storage_options,
+        read_consistency_interval,
+        session,
+        namespace_client_pushdown_operations,
+    );
+
+    Ok(Connection::new(LanceConnection::new(
+        Arc::new(database),
+        Arc::new(lancedb::embeddings::MemoryRegistry::new()),
+    )))
+}
+
 #[derive(FromPyObject)]
 pub struct PyClientConfig {
     user_agent: String,

python/src/lib.rs

@@ -2,7 +2,7 @@
 // SPDX-FileCopyrightText: Copyright The LanceDB Authors
 
 use arrow::RecordBatchStream;
-use connection::{Connection, connect};
+use connection::{Connection, connect, connect_namespace_client};
 use env_logger::Env;
 use expr::{PyExpr, expr_col, expr_func, expr_lit};
 use index::IndexConfig;
@@ -58,6 +58,7 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_class::<PyPermutationReader>()?;
     m.add_class::<PyExpr>()?;
     m.add_function(wrap_pyfunction!(connect, m)?)?;
+    m.add_function(wrap_pyfunction!(connect_namespace_client, m)?)?;
     m.add_function(wrap_pyfunction!(permutation::async_permutation_builder, m)?)?;
     m.add_function(wrap_pyfunction!(util::validate_table_name, m)?)?;
     m.add_function(wrap_pyfunction!(query::fts_query_to_json, m)?)?;