From 741cb9ad4e7b15aaa611a6c64415bd67c9200e9d Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Wed, 28 Oct 2026 16:08:43 -0700 Subject: [PATCH] fix(python): implement remote namespace and table existence checks (#4725) Remote Python connections inherited the unsupported-operation stubs for `namespace_exists` and `table_exists`, so both raised `NotImplementedError` even when namespace and table operations worked. Add the missing remote overrides, async methods, and PyO3 bindings using the existing namespace describe and table open paths. Only an explicit not-found result returns `False`; permission, transport, and server errors propagate. This preserves authentication, dynamic headers, retries, and nested namespace handling, and rechecks cached tables before reporting that they exist. The async namespace helper also recognizes native `NamespaceNotFound` errors so missing local namespaces return `False`. The table and namespace guides document Python remote support and distinguish it from the unsupported `exists` routes used by direct REST namespace clients. Validation: - All 40 focused regression cases pass, covering successful and missing lookups, local and remote namespaces, dynamic headers, permission/server errors, invalid table identifiers, and cached tables that disappear. - The original inherited stubs reproduce the reported `NotImplementedError`; the local namespace regression reproduced the native missing-namespace `RuntimeError` before its fix. - Full remote connection suite: 255 passed. Local namespace tests: 14 passed. - `cargo clippy --quiet -p lancedb-python --features remote --lib`, `cargo fmt --all`, Python Ruff lint, and Python formatting checks pass. - Root `ruff check .` reports 48 existing errors in documentation examples under `docs/web-tests`; the Python subtree passes and those unrelated files are unchanged. Fixes #4722 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> --- docs/web/management/namespace.mdx | 7 +- docs/web/management/table.mdx | 23 ++++--- python/python/lancedb/_lancedb.pyi | 2 + python/python/lancedb/db.py | 43 ++++++++++++ python/python/lancedb/remote/db.py | 30 ++++++++ python/python/tests/test_db.py | 18 +++++ python/python/tests/test_remote_db.py | 98 ++++++++++++++++++++++++++- python/src/connection.rs | 47 +++++++++++++ 8 files changed, 257 insertions(+), 11 deletions(-) diff --git a/docs/web/management/namespace.mdx b/docs/web/management/namespace.mdx index 0d80831c9..e5222ab84 100644 --- a/docs/web/management/namespace.mdx +++ b/docs/web/management/namespace.mdx @@ -243,8 +243,11 @@ let db = lancedb::connect_namespace("rest", properties) ``` [LanceDB Enterprise](/enterprise) operates a REST namespace server on top of the Lance format, so any REST client that can speak the REST namespace API -contract can be used to interact with it. It does not serve the contract's `exists` routes, so checks such as `namespace_exists` -fail against it. For authentication examples in LanceDB Enterprise, visit +contract can be used to interact with it. It does not serve the contract's `exists` routes, +so existence checks made through a direct REST namespace client fail against it. +Python remote connections created with `lancedb.connect("db://...")` or +`lancedb.connect_async("db://...")` support `namespace_exists` and `table_exists` +through the existing describe routes instead. For authentication examples in LanceDB Enterprise, visit the [Namespaces in SDKs](#namespaces-in-lancedb-enterprise) page. ## Best practices {#best-practices} diff --git a/docs/web/management/table.mdx b/docs/web/management/table.mdx index bf256ac97..a28f63d40 100644 --- a/docs/web/management/table.mdx +++ b/docs/web/management/table.mdx @@ -214,14 +214,21 @@ await db.tableNames(); Each accepts `namespace_path` to act inside a [namespace](/management/namespace) rather than at the database root. -To find out whether a table exists, open it and handle the error, or look for -its name in a listing. A missing table raises `ValueError` on a local directory -or a deployment, and `lance_namespace.TableNotFoundError` on a connection from -`connect_namespace`. A listing returns one page of names at a time, so check -every page. Python's `table_exists` is no substitute: a local directory or a -deployment raises `NotImplementedError` for it, and on a connection from -`connect_namespace("dir", ...)` or `connect_namespace("rest", ...)` it fails with -an `AttributeError`. TypeScript has no `tableExists`. +For Python remote connections created with `lancedb.connect("db://...")`, use +`db.table_exists(["documents"])` to check whether a table exists. For an async +remote connection, use `await db.table_exists(["documents"])`. Include namespace +segments before the table name, for example `["analytics", "documents"]`. These +checks return `False` when the table is missing and propagate other errors, +including permission errors. + +Local synchronous directory connections still raise `NotImplementedError` for +`table_exists`. On a connection from `connect_namespace("dir", ...)` or +`connect_namespace("rest", ...)`, it fails with an `AttributeError`. For these +connections, open the table and handle the error, or look for its name in a +listing. A missing table raises `ValueError` on a local directory and +`lance_namespace.TableNotFoundError` on a connection from `connect_namespace`. +A listing returns one page of names at a time, so check every page. TypeScript +has no `tableExists`. ## Rename and clone tables {#rename-clone} diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 66fdd3a7c..2b8d1655a 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -137,6 +137,8 @@ class Connection(object): self, namespace_path: List[str], ) -> DescribeNamespaceResponse: ... + async def namespace_exists(self, namespace_id: List[str]) -> bool: ... + async def table_exists(self, table_id: List[str]) -> bool: ... def list_tables( self, namespace_path: Optional[List[str]] = None, diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index 09e032607..32f7dc43c 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -2436,6 +2436,49 @@ class AsyncConnection(object): result = await self._inner.describe_namespace(namespace_path) return DescribeNamespaceResponse(**result) + async def namespace_exists(self, namespace_id: List[str]) -> bool: + """Check if a namespace exists. + + Parameters + ---------- + namespace_id: List[str] + The namespace identifier to check. An empty list represents root. + + Returns + ------- + bool + True if the namespace exists, False if it is not found. + Other errors, including permission errors, are propagated. + + Examples + -------- + >>> async def check_namespace(conn): + ... return await conn.namespace_exists(["analytics"]) + """ + return await self._inner.namespace_exists(namespace_id) + + async def table_exists(self, table_id: List[str]) -> bool: + """Check if a table exists. + + Parameters + ---------- + table_id: List[str] + The full identifier, including namespace segments and table name. + Must contain at least the table name. + + Returns + ------- + bool + True if the table exists, False if it is not found. + Other errors, including permission errors, are propagated. + + Examples + -------- + >>> async def check_table(conn): + ... return await conn.table_exists(["analytics", "events"]) + """ + return await self._inner.table_exists(table_id) + def list_tables( self, namespace_path: Optional[List[str]] = None, diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index e2defa216..d97e6526b 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -390,6 +390,36 @@ class RemoteDBConnection(DBConnection): """ return LOOP.run(self._conn.describe_namespace(namespace_path=namespace_path)) + @override + def namespace_exists(self, namespace_id: List[str]) -> bool: + """Check if a namespace exists. + + ``namespace_id`` contains the namespace segments; an empty list represents + root. Returns False only if the namespace is not found. Other errors, + including permission errors, are propagated. + + Examples + -------- + >>> def check_namespace(conn): + ... return conn.namespace_exists(["analytics"]) + """ + return LOOP.run(self._conn.namespace_exists(namespace_id)) + + @override + def table_exists(self, table_id: List[str]) -> bool: + """Check if a table exists. + + ``table_id`` contains the namespace segments followed by the table name. + Returns False only if the table is not found. Other errors, including + permission errors, are propagated. + + Examples + -------- + >>> def check_table(conn): + ... return conn.table_exists(["analytics", "events"]) + """ + return LOOP.run(self._conn.table_exists(table_id)) + @override def list_tables( self, diff --git a/python/python/tests/test_db.py b/python/python/tests/test_db.py index af63987aa..64bf928c0 100644 --- a/python/python/tests/test_db.py +++ b/python/python/tests/test_db.py @@ -1149,6 +1149,24 @@ def test_local_namespace_operations(tmp_path): assert db.list_namespaces().namespaces == [] +@pytest.mark.asyncio +async def test_local_async_namespace_exists(tmp_path): + db = await lancedb.connect_async(tmp_path) + await db.create_namespace(["existing"]) + await db.create_namespace(["existing", "nested"]) + + assert await db.namespace_exists([]) is True + assert await db.namespace_exists(["existing"]) is True + assert await db.namespace_exists(["existing", "nested"]) is True + assert await db.namespace_exists(["missing"]) is False + assert await db.namespace_exists(["existing", "missing"]) is False + assert await db.namespace_exists(["missing", "nested"]) is False + + await db.drop_namespace(["existing", "nested"]) + assert await db.namespace_exists(["existing", "nested"]) is False + assert await db.namespace_exists(["existing"]) is True + + def test_local_sync_namespace_uses_rust_without_python_client(tmp_path, monkeypatch): """Sync local namespace operations should avoid the Python namespace client.""" db = lancedb.connect(tmp_path) diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 32fec4b57..0b32376e8 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -54,7 +54,7 @@ def test_connect_rejects_invalid_cloud_dns_hostname(db_name): @contextlib.contextmanager -def mock_lancedb_connection(handler): +def mock_lancedb_connection(handler, **client_config): with http.server.HTTPServer( ("localhost", 0), make_mock_http_handler(handler) ) as server: @@ -71,6 +71,7 @@ def mock_lancedb_connection(handler): "timeout_config": { "connect_timeout": 1, }, + **client_config, }, ) @@ -110,6 +111,101 @@ async def mock_lancedb_connection_async(handler, **client_config): handle.join() +@pytest.fixture +def existence_handler(method, identifier, status): + def handler(request): + resource = "namespace" if method == "namespace_exists" else "table" + resource_id = "$".join(identifier) or "$" + suffix = "/" if resource == "table" else "" + assert request.command == "POST" + assert request.path == f"/v1/{resource}/{resource_id}/describe{suffix}" + assert request.headers["x-api-key"] == "fake" + assert request.headers["X-Exists-Test"] == "present" + request.send_response(status) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(b"{}" if status == 200 else b"lookup failed") + + return handler + + +@pytest.mark.parametrize( + "method,identifier", + [ + ("namespace_exists", []), + ("namespace_exists", ["ns"]), + ("namespace_exists", ["parent", "ns"]), + ("table_exists", ["table"]), + ("table_exists", ["parent", "ns", "table"]), + ], +) +@pytest.mark.parametrize("status", [200, 404, 403, 507]) +def test_remote_exists(method, identifier, status, existence_handler): + from lancedb.remote.header import StaticHeaderProvider + + provider = StaticHeaderProvider({"X-Exists-Test": "present"}) + with mock_lancedb_connection(existence_handler, header_provider=provider) as db: + if status in (200, 404): + assert getattr(db, method)(identifier) is (status == 200) + else: + with pytest.raises(HttpError) as exc_info: + getattr(db, method)(identifier) + assert exc_info.value.status_code == status + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "method,identifier", + [ + ("namespace_exists", []), + ("namespace_exists", ["parent", "ns"]), + ("table_exists", ["table"]), + ("table_exists", ["parent", "ns", "table"]), + ], +) +@pytest.mark.parametrize("status", [200, 404, 403, 507]) +async def test_async_remote_exists(method, identifier, status, existence_handler): + from lancedb.remote.header import StaticHeaderProvider + + provider = StaticHeaderProvider({"X-Exists-Test": "present"}) + async with mock_lancedb_connection_async( + existence_handler, header_provider=provider + ) as db: + if status in (200, 404): + assert await getattr(db, method)(identifier) is (status == 200) + else: + with pytest.raises(HttpError) as exc_info: + await getattr(db, method)(identifier) + assert exc_info.value.status_code == status + + +@pytest.mark.parametrize("table_id", [[], ["invalid/name"]]) +def test_remote_table_exists_rejects_invalid_identifier(table_id): + requests = [] + with mock_lancedb_connection(lambda request: requests.append(request.path)) as db: + with pytest.raises(ValueError): + db.table_exists(table_id) + assert requests == [] + + +def test_remote_table_exists_rechecks_cached_table(): + requests = [] + + def handler(request): + requests.append(request.path) + request.send_response(200 if len(requests) == 1 else 404) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(b"{}") + + with mock_lancedb_connection(handler) as db: + table_id = ["ns", "table"] + assert db.table_exists(table_id) is True + assert db.table_exists(table_id) is False + assert table_id == ["ns", "table"] + assert requests == ["/v1/table/ns$table/describe/"] * 2 + + def paginated_table_listing_handler(request): url = urlparse(request.path) assert url.path in { diff --git a/python/src/connection.rs b/python/src/connection.rs index 2cdf71f4a..54ea615de 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -711,6 +711,53 @@ impl Connection { }) } + pub fn namespace_exists( + self_: PyRef<'_, Self>, + namespace_id: Vec, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + future_into_py(self_.py(), async move { + let request = lance_namespace::models::DescribeNamespaceRequest { + id: Some(namespace_id), + ..Default::default() + }; + match inner.describe_namespace(request).await { + Ok(_) => Ok(true), + Err(lancedb::Error::Http { + status_code: Some(status), + .. + }) if status.as_u16() == 404 => Ok(false), + Err(lancedb::Error::Lance { + source: lance_core::Error::Namespace { ref source, .. }, + }) if matches!( + source.downcast_ref::(), + Some(lance_namespace::NamespaceError::NamespaceNotFound { .. }) + ) => + { + Ok(false) + } + Err(err) => Err(err).infer_error(), + } + }) + } + + pub fn table_exists( + self_: PyRef<'_, Self>, + mut table_id: Vec, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let name = table_id + .pop() + .ok_or_else(|| PyValueError::new_err("table_id must include a table name"))?; + future_into_py(self_.py(), async move { + match inner.open_table(name).namespace(table_id).execute().await { + Ok(_) => Ok(true), + Err(lancedb::Error::TableNotFound { .. }) => Ok(false), + Err(err) => Err(err).infer_error(), + } + }) + } + #[pyo3(signature = (namespace_path=None, page_token=None, limit=None))] pub fn list_tables( &self,