feat: support namespace credentials vending (#2778)

Based on https://github.com/lancedb/lance/pull/4984

1. Bump to 1.0.0-beta.2
2. Use DirectoryNamespace in lance to perform all testing in python and
rust for much better coverage
3. Refactor `ListingDatabase` to be able to accept location and
namespace. This is because we have to leverage listing database (local
lancedb connection) for using namespace, namespace only resolves the
location and storage options but we don't want to bind all the way to
rust since user will plug-in namespace from python side. And thus
`ListingDatabase` needs to be able to accept location and namespace that
are created from namespace connection.
4. For credentials vending, we also pass storage options provider all
the way to rust layer, and the rust layer calls back to the python
function to fetch next storage option. This is exactly the same thing we
did in pylance.

python/python/tests/test_db.py

@@ -781,58 +781,6 @@ def test_local_drop_namespace_not_supported(tmp_path):
         db.drop_namespace(["test_namespace"])
 
 
-def test_local_table_operations_with_namespace_raise_error(tmp_path):
-    """
-    Test that table operations with namespace parameter
-    raise ValueError in local mode.
-    """
-    db = lancedb.connect(tmp_path)
-
-    # Create some test data
-    data = [{"vector": [1.0, 2.0], "item": "test"}]
-    schema = pa.schema(
-        [pa.field("vector", pa.list_(pa.float32(), 2)), pa.field("item", pa.string())]
-    )
-
-    # Test create_table with namespace - should raise ValueError
-    with pytest.raises(
-        NotImplementedError,
-        match="Namespace parameter is not supported for listing database",
-    ):
-        db.create_table(
-            "test_table_with_ns", data=data, schema=schema, namespace=["test_ns"]
-        )
-
-    # Create table normally for other tests
-    db.create_table("test_table", data=data, schema=schema)
-    assert "test_table" in db.table_names()
-
-    # Test open_table with namespace - should raise ValueError
-    with pytest.raises(
-        NotImplementedError,
-        match="Namespace parameter is not supported for listing database",
-    ):
-        db.open_table("test_table", namespace=["test_ns"])
-
-    # Test table_names with namespace - should raise ValueError
-    with pytest.raises(
-        NotImplementedError,
-        match="Namespace parameter is not supported for listing database",
-    ):
-        list(db.table_names(namespace=["test_ns"]))
-
-    # Test drop_table with namespace - should raise ValueError
-    with pytest.raises(
-        NotImplementedError,
-        match="Namespace parameter is not supported for listing database",
-    ):
-        db.drop_table("test_table", namespace=["test_ns"])
-
-    # Test table_names without namespace - should work normally
-    tables_root = list(db.table_names())
-    assert "test_table" in tables_root
-
-
 def test_clone_table_latest_version(tmp_path):
     """Test cloning a table with the latest version (default behavior)"""
     import os