From e622254108f15e159627d8a1d8e42e983e45cd81 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 11 Nov 2026 23:32:19 -0800 Subject: [PATCH] fix(python): use direct lookups for table existence checks (#4974) Table existence checks should not depend on listing every table or searching only the first 100 names. Use a direct lookup for `name in db` and implement `table_exists(table_id)` on synchronous local connections, matching the remote and asynchronous APIs. Native namespace connections use their initialized Rust connection for existence checks; custom namespace clients retain their existing callback. Missing tables return `False`; other lookup errors propagate. ```python if "events" in db: db.drop_table("events") exists = db.table_exists(["analytics", "events"]) all_names = list(db.list_tables()) ``` Clarify that `.tables` contains one page and update deprecation guidance with safe replacements. Add regression coverage for lookups beyond the first page, nested namespaces, native and custom namespace clients, missing tables, permission errors, and lookups that do not list tables. Include an unpublished draft for the table-management guide until the new SDK support ships. --- docs/src/python/python.md | 17 +++++++ docs/web/management/table.mdx | 23 ++++++++- python/python/lancedb/db.py | 56 +++++++++++++++++---- python/python/lancedb/listing.py | 8 +++ python/python/lancedb/namespace.py | 30 ++++++++--- python/python/lancedb/remote/db.py | 13 +++-- python/python/tests/test_db.py | 46 +++++++++++++---- python/python/tests/test_namespace.py | 41 +++++++++++++++ python/python/tests/test_remote_db.py | 26 ++++++++++ python/python/tests/test_sync_api_parity.py | 25 ++++++--- 10 files changed, 246 insertions(+), 39 deletions(-) diff --git a/docs/src/python/python.md b/docs/src/python/python.md index eccaae6ce..8de5b5517 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -42,6 +42,18 @@ table names. `db.list_tables().tables` is a list of the first page; `limit` controls its page size and `page_token` resumes at a saved token. The same patterns apply to `list_namespaces()`. Other resource listings use `page_limit`. +For existence checks, use `db.table_exists(["events"])` or `"events" in db`. +These look up the table directly and work even when the database has more than +100 tables. For a table in a namespace, include the namespace segments: + +```python +exists = db.table_exists(["analytics", "events"]) +``` + +When migrating from `name in db.table_names()`, use `name in db` or +`db.table_exists([name])`. Checking `name in db.list_tables().tables` only +searches the first page and can miss an existing table. + Resource listings (`list_storage_integrations`, `list_secrets`, `list_views`, `list_jobs`, `list_functions`, and `list_materialized_views`) return lazy iterators. Use `list(db.list_views())` to collect synchronous results, or `[name async for name in db.list_views()]` with @@ -1055,6 +1067,11 @@ the page list does not consume the iterator. `page.page_token` is the token for the next request; drain cached results before using it to resume in a new listing. The same behavior applies to namespaces. +Use `await db.table_exists(["events"])` to check for a table in the root +namespace, or `await db.table_exists(["analytics", "events"])` for a table in +a namespace. Checking membership in an awaited listing's `.tables` attribute +only searches one page. + ::: lancedb.listing.AsyncTableListing ::: lancedb.listing.AsyncNamespaceListing diff --git a/docs/web/management/table.mdx b/docs/web/management/table.mdx index 2a05b4a0f..23fdbdb4a 100644 --- a/docs/web/management/table.mdx +++ b/docs/web/management/table.mdx @@ -201,7 +201,7 @@ setting. See [Limits](/basics/limits#schema). ```python Python icon="python" tbl = db.open_table("documents") -db.table_names() +names = list(db.list_tables()) ``` ```typescript TypeScript icon="square-js" @@ -230,6 +230,27 @@ listing. A missing table raises `ValueError` on a local directory and A listing returns one page of names at a time, so check every page. TypeScript has no `tableExists`. +{/* +Draft for the table-existence lookup release (#4974). Once released, replace +the two table-existence paragraphs above with this guidance: + +For Python local, remote, and namespace connections, use +`db.table_exists(["documents"])` or `"documents" in db` to check whether a root +table exists. For an async connection, use +`await db.table_exists(["documents"])`. Include namespace segments before the +table name, for example `["analytics", "documents"]`. These checks return +`False` when the table is missing and propagate other errors, including +permission errors. They look up the table directly, regardless of the number +of tables in the database. + +When migrating from `name in db.table_names()`, use `name in db` or +`db.table_exists([name])`. To collect every table name, use +`list(db.list_tables())`, or `[name async for name in db.list_tables()]` on an +async connection. The `.tables` attribute holds only one page (100 names by +default), so membership checks against it can miss existing tables. TypeScript +has no `tableExists`. +*/} + ## Rename and clone tables {#rename-clone} diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index 73c8305ff..84ebb41a1 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -322,6 +322,9 @@ class DBConnection(EnforceOverrides): def table_exists(self, table_id: List[str]) -> bool: """Check if a table exists. + Use this method instead of checking membership in + ``list_tables().tables``, which contains only one page of table names. + Parameters ---------- table_id: List[str] @@ -331,12 +334,20 @@ class DBConnection(EnforceOverrides): Returns ------- bool - True if the table exists, False otherwise. + True if the table exists, False if it is not found. + Other errors, including permission errors, are propagated. Raises ------ NotImplementedError - If the connection type does not support namespace operations. + If the connection type does not support table existence checks. + + Examples + -------- + >>> def check_table(conn): + ... root_exists = conn.table_exists(["events"]) + ... nested_exists = conn.table_exists(["analytics", "events"]) + ... return root_exists, nested_exists """ raise NotImplementedError( "Namespace operations are not supported for this connection type" @@ -348,7 +359,11 @@ class DBConnection(EnforceOverrides): page_token: Optional[str] = None, limit: Optional[int] = None, ) -> TableListing: - """List tables in this database, one page at a time. + """Iterate over all tables in this database, fetching pages as needed. + + ``list_tables().tables`` contains only the first page (100 names by + default). Use ``list(list_tables())`` to collect all names, or + ``table_exists([name])`` to check whether a particular table exists. Parameters ---------- @@ -934,7 +949,7 @@ class DBConnection(EnforceOverrides): def __contains__(self, name: str) -> bool: """Return whether the root namespace contains the named table.""" - return name in self._all_table_names() + return self.table_exists([name]) def namespace_client(self) -> LanceNamespace: """Get the equivalent namespace client for this connection. @@ -1849,6 +1864,10 @@ class LanceDBConnection(DBConnection): ) -> DescribeNamespaceResponse: return LOOP.run(self._conn.describe_namespace(namespace_path=namespace_path)) + @override + def table_exists(self, table_id: List[str]) -> bool: + return LOOP.run(self._conn.table_exists(table_id)) + @override def list_tables( self, @@ -1856,7 +1875,11 @@ class LanceDBConnection(DBConnection): page_token: Optional[str] = None, limit: Optional[int] = None, ) -> TableListing: - """List tables in this database, one page at a time. + """Iterate over all tables in this database, fetching pages as needed. + + The ``tables`` attribute contains only the current page (100 names by + default). Iterate over the listing to visit all pages, or use + ``table_exists([name])`` to check for a particular table. Parameters ---------- @@ -1894,7 +1917,9 @@ class LanceDBConnection(DBConnection): """Get the names of all tables in the database. The names are sorted. .. deprecated:: - Use :meth:`list_tables` instead, which provides proper pagination support. + Iterate over :meth:`list_tables` to get all names, or use + :meth:`table_exists` to check for a particular table. The listing's + ``tables`` attribute contains only one page. Parameters ---------- @@ -1914,7 +1939,8 @@ class LanceDBConnection(DBConnection): import warnings warnings.warn( - "table_names() is deprecated, use list_tables() instead", + "table_names() is deprecated, iterate list_tables() for all names " + "or use table_exists([name]) to check existence", DeprecationWarning, stacklevel=2, ) @@ -2878,6 +2904,9 @@ class AsyncConnection(object): async def table_exists(self, table_id: List[str]) -> bool: """Check if a table exists. + Use this method instead of checking membership in an awaited listing's + ``tables`` attribute, which contains only one page of table names. + Parameters ---------- table_id: List[str] @@ -2903,7 +2932,11 @@ class AsyncConnection(object): page_token: Optional[str] = None, limit: Optional[int] = None, ) -> AsyncTableListing: - """List tables in this database, one page at a time. + """Iterate over all tables in this database, fetching pages as needed. + + The ``tables`` attribute contains only the current page (100 names by + default). Iterate over the listing to visit all pages, or use + ``table_exists([name])`` to check for a particular table. Parameters ---------- @@ -2942,7 +2975,9 @@ class AsyncConnection(object): """List all tables in this database, in sorted order .. deprecated:: - Use :meth:`list_tables` instead, which provides proper pagination support. + Iterate over :meth:`list_tables` to get all names, or use + :meth:`table_exists` to check for a particular table. The listing's + ``tables`` attribute contains only one page. Parameters ---------- @@ -2966,7 +3001,8 @@ class AsyncConnection(object): import warnings warnings.warn( - "table_names() is deprecated, use list_tables() instead", + "table_names() is deprecated, iterate list_tables() for all names " + "or use table_exists([name]) to check existence", DeprecationWarning, stacklevel=2, ) diff --git a/python/python/lancedb/listing.py b/python/python/lancedb/listing.py index 8abc9d85b..59aba1431 100644 --- a/python/python/lancedb/listing.py +++ b/python/python/lancedb/listing.py @@ -175,6 +175,9 @@ class _AsyncPagedNameListing(AsyncIterator[str]): class AsyncTableListing(_AsyncPagedNameListing): """Awaitable table iterator; awaiting loads one page. + Iteration fetches all pages. The ``tables`` attribute contains only the + current page, so use ``await db.table_exists([name])`` for existence checks. + For example:: page = await db.list_tables(limit=20) @@ -229,6 +232,10 @@ class _PagedNameListing(Iterator[str]): class TableListing(_PagedNameListing): """Table iterator retaining the current page as a list. + Iteration fetches all pages. The ``tables`` attribute contains only the + current page, so use ``db.table_exists([name])`` or ``name in db`` for + existence checks. Use ``list(db.list_tables())`` to collect all names. + For example:: page = db.list_tables(limit=20) @@ -239,6 +246,7 @@ class TableListing(_PagedNameListing): @property def tables(self) -> list[str]: + """A copy of the table names in the most recently fetched page.""" return self._inner.tables diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py index 3356b363c..aa9025641 100644 --- a/python/python/lancedb/namespace.py +++ b/python/python/lancedb/namespace.py @@ -579,12 +579,15 @@ class LanceNamespaceDBConnection(DBConnection): If limit is None, return all table names; zero returns an empty list. .. deprecated:: - Use :meth:`list_tables` instead, which provides proper pagination support. + Iterate over :meth:`list_tables` to get all names, or use + :meth:`table_exists` to check for a particular table. The listing's + ``tables`` attribute contains only one page. """ import warnings warnings.warn( - "table_names() is deprecated, use list_tables() instead", + "table_names() is deprecated, iterate list_tables() for all names " + "or use table_exists([name]) to check existence", DeprecationWarning, stacklevel=2, ) @@ -1015,6 +1018,8 @@ class LanceNamespaceDBConnection(DBConnection): bool True if the table exists, False otherwise. """ + if self._route_pushdown_to_rust: + return LOOP.run(self._inner.table_exists(table_id)) request = TableExistsRequest(id=table_id) try: self._namespace_client.table_exists(request) @@ -1030,7 +1035,11 @@ class LanceNamespaceDBConnection(DBConnection): limit: Optional[int] = None, ) -> TableListing: """ - List all tables in this database with pagination support. + Iterate over all tables in this database, fetching pages as needed. + + The ``tables`` attribute contains only the current page (100 names by + default). Iterate over the listing to visit all pages, or use + ``table_exists([name])`` to check for a particular table. Parameters ---------- @@ -1227,12 +1236,15 @@ class AsyncLanceNamespaceDBConnection: If limit is None, return all table names; zero returns an empty list. .. deprecated:: - Use :meth:`list_tables` instead, which provides proper pagination support. + Iterate over :meth:`list_tables` to get all names, or use + :meth:`table_exists` to check for a particular table. The listing's + ``tables`` attribute contains only one page. """ import warnings warnings.warn( - "table_names() is deprecated, use list_tables() instead", + "table_names() is deprecated, iterate list_tables() for all names " + "or use table_exists([name]) to check existence", DeprecationWarning, stacklevel=2, ) @@ -1614,6 +1626,8 @@ class AsyncLanceNamespaceDBConnection: bool True if the table exists, False otherwise. """ + if self._route_pushdown_to_rust: + return await self._inner.table_exists(table_id) request = TableExistsRequest(id=table_id) try: self._namespace_client.table_exists(request) @@ -1628,7 +1642,11 @@ class AsyncLanceNamespaceDBConnection: limit: Optional[int] = None, ) -> AsyncTableListing: """ - List all tables in this database with pagination support. + Iterate over all tables in this database, fetching pages as needed. + + The ``tables`` attribute contains only the current page (100 names by + default). Iterate over the listing to visit all pages, or use + ``table_exists([name])`` to check for a particular table. Parameters ---------- diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index ebceba16b..21ad37b36 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -430,7 +430,11 @@ class RemoteDBConnection(DBConnection): page_token: Optional[str] = None, limit: Optional[int] = None, ) -> TableListing: - """List a page of tables in this database. + """Iterate over all tables in this database, fetching pages as needed. + + The ``tables`` attribute contains only the current page (100 names by + default). Iterate over the listing to visit all pages, or use + ``table_exists([name])`` to check for a particular table. Parameters ---------- @@ -468,7 +472,9 @@ class RemoteDBConnection(DBConnection): """List the names of all tables in the database. .. deprecated:: - Use :meth:`list_tables` instead, which provides proper pagination support. + Iterate over :meth:`list_tables` to get all names, or use + :meth:`table_exists` to check for a particular table. The listing's + ``tables`` attribute contains only one page. Parameters ---------- @@ -488,7 +494,8 @@ class RemoteDBConnection(DBConnection): import warnings warnings.warn( - "table_names() is deprecated, use list_tables() instead", + "table_names() is deprecated, iterate list_tables() for all names " + "or use table_exists([name]) to check existence", DeprecationWarning, stacklevel=2, ) diff --git a/python/python/tests/test_db.py b/python/python/tests/test_db.py index 64bf928c0..80a98bc18 100644 --- a/python/python/tests/test_db.py +++ b/python/python/tests/test_db.py @@ -282,9 +282,7 @@ async def test_table_listing_defaults_and_zero_limit(tmp_path): assert empty.page_token is None -def test_db_contains_stops_after_matching_table_page( - tmp_db: lancedb.DBConnection, monkeypatch -): +def test_db_len_includes_all_table_pages(tmp_db: lancedb.DBConnection, monkeypatch): calls = [] pages = { None: SimpleNamespace(tables=["table_0", "table_1"], page_token="next"), @@ -297,18 +295,44 @@ def test_db_contains_stops_after_matching_table_page( monkeypatch.setattr(tmp_db, "list_tables", list_tables) - assert "table_1" in tmp_db - assert calls == [None] - - calls.clear() - assert "table_2" in tmp_db - assert calls == [None, "next"] - - calls.clear() assert len(tmp_db) == 3 assert calls == [None, "next"] +@pytest.mark.asyncio +@pytest.mark.parametrize("namespace_path", [[], ["analytics", "nested"]]) +async def test_local_table_exists_without_listing( + tmp_path, monkeypatch, namespace_path +): + db = lancedb.connect(tmp_path) + for depth in range(1, len(namespace_path) + 1): + db.create_namespace(namespace_path[:depth]) + db.create_table("events", data=[{"id": 1}], namespace_path=namespace_path) + if not namespace_path: + for idx in range(100): + (tmp_path / f"a{idx:03}.lance").mkdir() + assert "events" not in db.list_tables().tables + adb = await lancedb.connect_async(tmp_path) + + def fail_listing(*args, **kwargs): + raise AssertionError("Existence checks must not list tables") + + monkeypatch.setattr(db, "list_tables", fail_listing) + monkeypatch.setattr(adb, "list_tables", fail_listing) + table_id = namespace_path + ["events"] + assert db.table_exists(table_id) is True + assert await adb.table_exists(table_id) is True + assert db.table_exists(namespace_path + ["missing"]) is False + assert await adb.table_exists(namespace_path + ["missing"]) is False + assert ("events" in db) is (not namespace_path) + assert "missing" not in db + assert table_id == namespace_path + ["events"] + + db.drop_table("events", namespace_path=namespace_path) + assert db.table_exists(table_id) is False + assert await adb.table_exists(table_id) is False + + @pytest.mark.asyncio async def test_table_names_async(tmp_path): db = lancedb.connect(tmp_path) diff --git a/python/python/tests/test_namespace.py b/python/python/tests/test_namespace.py index 5eb9c02bd..9bce1059d 100644 --- a/python/python/tests/test_namespace.py +++ b/python/python/tests/test_namespace.py @@ -34,6 +34,27 @@ PUSHDOWN_DATA = pa.table( ) +@pytest.mark.asyncio +@pytest.mark.parametrize("table_id", [["events"], ["analytics", "events"], ["missing"]]) +async def test_custom_namespace_table_exists_uses_namespace_api(table_id): + class ExistenceNamespace(LanceNamespace): + def namespace_id(self): + return "existence-test" + + def table_exists(self, request): + if request.id == ["missing"]: + raise TableNotFoundError("Table not found") + + def describe_table(self, request): + raise AssertionError("Custom existence checks must not open tables") + + namespace = ExistenceNamespace() + db = LanceNamespaceDBConnection(namespace) + adb = lancedb.AsyncLanceNamespaceDBConnection(namespace) + assert db.table_exists(table_id) is (table_id != ["missing"]) + assert await adb.table_exists(table_id) is (table_id != ["missing"]) + + class TestNamespaceCreateTableOptions: @pytest.mark.parametrize("version", ["legacy", "stable"]) @pytest.mark.parametrize("empty", [False, True]) @@ -411,8 +432,16 @@ class TestNamespaceConnection: assert "test_ns" in db.list_namespaces().namespaces schema = pa.schema([pa.field("id", pa.int64())]) + db.create_table("root_table", schema=schema) + assert db.table_exists(["root_table"]) is True + assert "root_table" in db + assert db.table_exists(["missing"]) is False + assert "missing" not in db table = db.create_table("test_table", schema=schema, namespace_path=["test_ns"]) assert table.namespace == ["test_ns"] + assert db.table_exists(["test_ns", "test_table"]) is True + assert db.table_exists(["test_ns", "missing"]) is False + assert "test_table" not in db assert "test_table" in db.table_names(namespace_path=["test_ns"]) assert "test_table" in db.list_tables(namespace_path=["test_ns"]).tables @@ -420,7 +449,10 @@ class TestNamespaceConnection: assert opened.namespace == ["test_ns"] db.drop_table("test_table", namespace_path=["test_ns"]) + assert db.table_exists(["test_ns", "test_table"]) is False assert db.list_tables(namespace_path=["test_ns"]).tables == [] + db.drop_table("root_table") + assert "root_table" not in db db.drop_namespace(["test_ns"]) assert "test_ns" not in db.list_namespaces().namespaces @@ -942,12 +974,18 @@ class TestAsyncNamespaceConnection: assert "test_ns" in (await db.list_namespaces()).namespaces schema = pa.schema([pa.field("id", pa.int64())]) + await db.create_table("root_table", schema=schema) + assert await db.table_exists(["root_table"]) is True + assert await db.table_exists(["missing"]) is False table = await db.create_table( "test_table", schema=schema, namespace_path=["test_ns"] ) assert table._namespace_path == ["test_ns"] assert table._namespace_client is None assert table._route_pushdown_to_rust is True + assert await db.table_exists(["test_ns", "test_table"]) is True + assert await db.table_exists(["test_ns", "missing"]) is False + assert await db.table_exists(["test_table"]) is False assert "test_table" in await db.table_names(namespace_path=["test_ns"]) assert "test_table" in (await db.list_tables(namespace_path=["test_ns"])).tables @@ -955,7 +993,10 @@ class TestAsyncNamespaceConnection: assert opened._namespace_path == ["test_ns"] await db.drop_table("test_table", namespace_path=["test_ns"]) + assert await db.table_exists(["test_ns", "test_table"]) is False assert (await db.list_tables(namespace_path=["test_ns"])).tables == [] + await db.drop_table("root_table") + assert await db.table_exists(["root_table"]) is False await db.drop_namespace(["test_ns"]) assert "test_ns" not in (await db.list_namespaces()).namespaces diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 407596d26..952437051 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -22,6 +22,7 @@ import time from urllib.parse import parse_qs, urlparse from unittest.mock import MagicMock, patch import uuid +import warnings from packaging.version import Version from lance_namespace.errors import TableNotFoundError @@ -461,6 +462,31 @@ def test_remote_table_exists_rechecks_cached_table(): assert requests == ["/v1/table/ns$table/describe/"] * 2 +@pytest.mark.parametrize("status", [200, 404, 403]) +def test_remote_membership_uses_direct_lookup(status): + requests = [] + + def handler(request): + requests.append(request.path) + assert request.command == "POST" + assert request.path == "/v1/table/t129/describe/" + request.send_response(status) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(b"{}" if status == 200 else b"lookup failed") + + with mock_lancedb_connection(handler) as db: + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + if status in (200, 404): + assert ("t129" in db) is (status == 200) + else: + with pytest.raises(HttpError) as exc_info: + "t129" in db + assert exc_info.value.status_code == status + assert requests == ["/v1/table/t129/describe/"] + + def paginated_table_listing_handler(request): url = urlparse(request.path) assert url.path in { diff --git a/python/python/tests/test_sync_api_parity.py b/python/python/tests/test_sync_api_parity.py index a12efd0c9..230fb45c6 100644 --- a/python/python/tests/test_sync_api_parity.py +++ b/python/python/tests/test_sync_api_parity.py @@ -58,8 +58,8 @@ def test_legacy_table_replace_field_metadata(legacy_table): ] -def respond(request, body): - request.send_response(200) +def respond(request, body, status=200): + request.send_response(status) request.send_header("Content-Type", "application/json") request.end_headers() request.wfile.write(json.dumps(body).encode()) @@ -78,12 +78,20 @@ def test_remote_unsupported_properties(attribute): @pytest.mark.parametrize("tables", [[], ["first", "last"]]) -def test_remote_len_and_contains_follow_pages(tables): +def test_remote_len_paginates_and_contains_looks_up_table(tables): requests = [] def handler(request): requests.append(request.path) - query = parse_qs(urlparse(request.path).query) + url = urlparse(request.path) + if request.command == "POST": + name = url.path.split("/")[3] + assert url.path == f"/v1/table/{name}/describe/" + respond(request, {}, status=200 if name in tables else 404) + return + assert request.command == "GET" + assert url.path == "/v1/namespace/$/table/list" + query = parse_qs(url.query) if tables and "page_token" not in query: respond(request, {"tables": [tables[0]], "page_token": "opaque-next"}) else: @@ -93,11 +101,12 @@ def test_remote_len_and_contains_follow_pages(tables): with mock_lancedb_connection(handler) as db: assert len(db) == len(tables) - assert ("last" in db) == bool(tables) - assert "missing" not in db + assert len(requests) == (2 if tables else 1) requests.clear() - assert ("first" in db) == bool(tables) - assert len(requests) == 1 + for name in ("last", "missing", "first"): + assert (name in db) is (name in tables) + assert requests == [f"/v1/table/{name}/describe/"] + requests.clear() @pytest.mark.parametrize("namespace", [[], ["parent", "child"]])