fix(python): use direct lookups for table existence checks (#4974)

Table existence checks should not depend on listing every table or
searching only the first 100 names. Use a direct lookup for `name in db`
and implement `table_exists(table_id)` on synchronous local connections,
matching the remote and asynchronous APIs. Native namespace connections
use their initialized Rust connection for existence checks; custom
namespace clients retain their existing callback. Missing tables return
`False`; other lookup errors propagate.

```python
if "events" in db:
    db.drop_table("events")

exists = db.table_exists(["analytics", "events"])
all_names = list(db.list_tables())
```

Clarify that `.tables` contains one page and update deprecation guidance
with safe replacements. Add regression coverage for lookups beyond the
first page, nested namespaces, native and custom namespace clients,
missing tables, permission errors, and lookups that do not list tables.
Include an unpublished draft for the table-management guide until the
new SDK support ships.

python/python/lancedb/db.py

@@ -322,6 +322,9 @@ class DBConnection(EnforceOverrides):
     def table_exists(self, table_id: List[str]) -> bool:
         """Check if a table exists.
 
+        Use this method instead of checking membership in
+        ``list_tables().tables``, which contains only one page of table names.
+
         Parameters
         ----------
         table_id: List[str]
@@ -331,12 +334,20 @@ class DBConnection(EnforceOverrides):
         Returns
         -------
         bool
-            True if the table exists, False otherwise.
+            True if the table exists, False if it is not found.
+            Other errors, including permission errors, are propagated.
 
         Raises
         ------
         NotImplementedError
-            If the connection type does not support namespace operations.
+            If the connection type does not support table existence checks.
+
+        Examples
+        --------
+        >>> def check_table(conn):
+        ...     root_exists = conn.table_exists(["events"])
+        ...     nested_exists = conn.table_exists(["analytics", "events"])
+        ...     return root_exists, nested_exists
         """
         raise NotImplementedError(
             "Namespace operations are not supported for this connection type"
@@ -348,7 +359,11 @@ class DBConnection(EnforceOverrides):
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
     ) -> TableListing:
-        """List tables in this database, one page at a time.
+        """Iterate over all tables in this database, fetching pages as needed.
+
+        ``list_tables().tables`` contains only the first page (100 names by
+        default). Use ``list(list_tables())`` to collect all names, or
+        ``table_exists([name])`` to check whether a particular table exists.
 
         Parameters
         ----------
@@ -934,7 +949,7 @@ class DBConnection(EnforceOverrides):
 
     def __contains__(self, name: str) -> bool:
         """Return whether the root namespace contains the named table."""
-        return name in self._all_table_names()
+        return self.table_exists([name])
 
     def namespace_client(self) -> LanceNamespace:
         """Get the equivalent namespace client for this connection.
@@ -1849,6 +1864,10 @@ class LanceDBConnection(DBConnection):
     ) -> DescribeNamespaceResponse:
         return LOOP.run(self._conn.describe_namespace(namespace_path=namespace_path))
 
+    @override
+    def table_exists(self, table_id: List[str]) -> bool:
+        return LOOP.run(self._conn.table_exists(table_id))
+
     @override
     def list_tables(
         self,
@@ -1856,7 +1875,11 @@ class LanceDBConnection(DBConnection):
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
     ) -> TableListing:
-        """List tables in this database, one page at a time.
+        """Iterate over all tables in this database, fetching pages as needed.
+
+        The ``tables`` attribute contains only the current page (100 names by
+        default). Iterate over the listing to visit all pages, or use
+        ``table_exists([name])`` to check for a particular table.
 
         Parameters
         ----------
@@ -1894,7 +1917,9 @@ class LanceDBConnection(DBConnection):
         """Get the names of all tables in the database. The names are sorted.
 
         .. deprecated::
-            Use :meth:`list_tables` instead, which provides proper pagination support.
+            Iterate over :meth:`list_tables` to get all names, or use
+            :meth:`table_exists` to check for a particular table. The listing's
+            ``tables`` attribute contains only one page.
 
         Parameters
         ----------
@@ -1914,7 +1939,8 @@ class LanceDBConnection(DBConnection):
         import warnings
 
         warnings.warn(
-            "table_names() is deprecated, use list_tables() instead",
+            "table_names() is deprecated, iterate list_tables() for all names "
+            "or use table_exists([name]) to check existence",
             DeprecationWarning,
             stacklevel=2,
         )
@@ -2878,6 +2904,9 @@ class AsyncConnection(object):
     async def table_exists(self, table_id: List[str]) -> bool:
         """Check if a table exists.
 
+        Use this method instead of checking membership in an awaited listing's
+        ``tables`` attribute, which contains only one page of table names.
+
         Parameters
         ----------
         table_id: List[str]
@@ -2903,7 +2932,11 @@ class AsyncConnection(object):
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
     ) -> AsyncTableListing:
-        """List tables in this database, one page at a time.
+        """Iterate over all tables in this database, fetching pages as needed.
+
+        The ``tables`` attribute contains only the current page (100 names by
+        default). Iterate over the listing to visit all pages, or use
+        ``table_exists([name])`` to check for a particular table.
 
         Parameters
         ----------
@@ -2942,7 +2975,9 @@ class AsyncConnection(object):
         """List all tables in this database, in sorted order
 
         .. deprecated::
-            Use :meth:`list_tables` instead, which provides proper pagination support.
+            Iterate over :meth:`list_tables` to get all names, or use
+            :meth:`table_exists` to check for a particular table. The listing's
+            ``tables`` attribute contains only one page.
 
         Parameters
         ----------
@@ -2966,7 +3001,8 @@ class AsyncConnection(object):
         import warnings
 
         warnings.warn(
-            "table_names() is deprecated, use list_tables() instead",
+            "table_names() is deprecated, iterate list_tables() for all names "
+            "or use table_exists([name]) to check existence",
             DeprecationWarning,
             stacklevel=2,
         )

python/python/lancedb/listing.py

@@ -175,6 +175,9 @@ class _AsyncPagedNameListing(AsyncIterator[str]):
 class AsyncTableListing(_AsyncPagedNameListing):
     """Awaitable table iterator; awaiting loads one page.
 
+    Iteration fetches all pages. The ``tables`` attribute contains only the
+    current page, so use ``await db.table_exists([name])`` for existence checks.
+
     For example::
 
         page = await db.list_tables(limit=20)
@@ -229,6 +232,10 @@ class _PagedNameListing(Iterator[str]):
 class TableListing(_PagedNameListing):
     """Table iterator retaining the current page as a list.
 
+    Iteration fetches all pages. The ``tables`` attribute contains only the
+    current page, so use ``db.table_exists([name])`` or ``name in db`` for
+    existence checks. Use ``list(db.list_tables())`` to collect all names.
+
     For example::
 
         page = db.list_tables(limit=20)
@@ -239,6 +246,7 @@ class TableListing(_PagedNameListing):
 
     @property
     def tables(self) -> list[str]:
+        """A copy of the table names in the most recently fetched page."""
         return self._inner.tables
 
 

python/python/lancedb/namespace.py

@@ -579,12 +579,15 @@ class LanceNamespaceDBConnection(DBConnection):
         If limit is None, return all table names; zero returns an empty list.
 
         .. deprecated::
-            Use :meth:`list_tables` instead, which provides proper pagination support.
+            Iterate over :meth:`list_tables` to get all names, or use
+            :meth:`table_exists` to check for a particular table. The listing's
+            ``tables`` attribute contains only one page.
         """
         import warnings
 
         warnings.warn(
-            "table_names() is deprecated, use list_tables() instead",
+            "table_names() is deprecated, iterate list_tables() for all names "
+            "or use table_exists([name]) to check existence",
             DeprecationWarning,
             stacklevel=2,
         )
@@ -1015,6 +1018,8 @@ class LanceNamespaceDBConnection(DBConnection):
         bool
             True if the table exists, False otherwise.
         """
+        if self._route_pushdown_to_rust:
+            return LOOP.run(self._inner.table_exists(table_id))
         request = TableExistsRequest(id=table_id)
         try:
             self._namespace_client.table_exists(request)
@@ -1030,7 +1035,11 @@ class LanceNamespaceDBConnection(DBConnection):
         limit: Optional[int] = None,
     ) -> TableListing:
         """
-        List all tables in this database with pagination support.
+        Iterate over all tables in this database, fetching pages as needed.
+
+        The ``tables`` attribute contains only the current page (100 names by
+        default). Iterate over the listing to visit all pages, or use
+        ``table_exists([name])`` to check for a particular table.
 
         Parameters
         ----------
@@ -1227,12 +1236,15 @@ class AsyncLanceNamespaceDBConnection:
         If limit is None, return all table names; zero returns an empty list.
 
         .. deprecated::
-            Use :meth:`list_tables` instead, which provides proper pagination support.
+            Iterate over :meth:`list_tables` to get all names, or use
+            :meth:`table_exists` to check for a particular table. The listing's
+            ``tables`` attribute contains only one page.
         """
         import warnings
 
         warnings.warn(
-            "table_names() is deprecated, use list_tables() instead",
+            "table_names() is deprecated, iterate list_tables() for all names "
+            "or use table_exists([name]) to check existence",
             DeprecationWarning,
             stacklevel=2,
         )
@@ -1614,6 +1626,8 @@ class AsyncLanceNamespaceDBConnection:
         bool
             True if the table exists, False otherwise.
         """
+        if self._route_pushdown_to_rust:
+            return await self._inner.table_exists(table_id)
         request = TableExistsRequest(id=table_id)
         try:
             self._namespace_client.table_exists(request)
@@ -1628,7 +1642,11 @@ class AsyncLanceNamespaceDBConnection:
         limit: Optional[int] = None,
     ) -> AsyncTableListing:
         """
-        List all tables in this database with pagination support.
+        Iterate over all tables in this database, fetching pages as needed.
+
+        The ``tables`` attribute contains only the current page (100 names by
+        default). Iterate over the listing to visit all pages, or use
+        ``table_exists([name])`` to check for a particular table.
 
         Parameters
         ----------

python/python/lancedb/remote/db.py

@@ -430,7 +430,11 @@ class RemoteDBConnection(DBConnection):
         page_token: Optional[str] = None,
         limit: Optional[int] = None,
     ) -> TableListing:
-        """List a page of tables in this database.
+        """Iterate over all tables in this database, fetching pages as needed.
+
+        The ``tables`` attribute contains only the current page (100 names by
+        default). Iterate over the listing to visit all pages, or use
+        ``table_exists([name])`` to check for a particular table.
 
         Parameters
         ----------
@@ -468,7 +472,9 @@ class RemoteDBConnection(DBConnection):
         """List the names of all tables in the database.
 
         .. deprecated::
-            Use :meth:`list_tables` instead, which provides proper pagination support.
+            Iterate over :meth:`list_tables` to get all names, or use
+            :meth:`table_exists` to check for a particular table. The listing's
+            ``tables`` attribute contains only one page.
 
         Parameters
         ----------
@@ -488,7 +494,8 @@ class RemoteDBConnection(DBConnection):
         import warnings
 
         warnings.warn(
-            "table_names() is deprecated, use list_tables() instead",
+            "table_names() is deprecated, iterate list_tables() for all names "
+            "or use table_exists([name]) to check existence",
             DeprecationWarning,
             stacklevel=2,
         )

python/python/tests/test_db.py

@@ -282,9 +282,7 @@ async def test_table_listing_defaults_and_zero_limit(tmp_path):
     assert empty.page_token is None
 
 
-def test_db_contains_stops_after_matching_table_page(
-    tmp_db: lancedb.DBConnection, monkeypatch
-):
+def test_db_len_includes_all_table_pages(tmp_db: lancedb.DBConnection, monkeypatch):
     calls = []
     pages = {
         None: SimpleNamespace(tables=["table_0", "table_1"], page_token="next"),
@@ -297,18 +295,44 @@ def test_db_contains_stops_after_matching_table_page(
 
     monkeypatch.setattr(tmp_db, "list_tables", list_tables)
 
-    assert "table_1" in tmp_db
-    assert calls == [None]
-
-    calls.clear()
-    assert "table_2" in tmp_db
-    assert calls == [None, "next"]
-
-    calls.clear()
     assert len(tmp_db) == 3
     assert calls == [None, "next"]
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("namespace_path", [[], ["analytics", "nested"]])
+async def test_local_table_exists_without_listing(
+    tmp_path, monkeypatch, namespace_path
+):
+    db = lancedb.connect(tmp_path)
+    for depth in range(1, len(namespace_path) + 1):
+        db.create_namespace(namespace_path[:depth])
+    db.create_table("events", data=[{"id": 1}], namespace_path=namespace_path)
+    if not namespace_path:
+        for idx in range(100):
+            (tmp_path / f"a{idx:03}.lance").mkdir()
+        assert "events" not in db.list_tables().tables
+    adb = await lancedb.connect_async(tmp_path)
+
+    def fail_listing(*args, **kwargs):
+        raise AssertionError("Existence checks must not list tables")
+
+    monkeypatch.setattr(db, "list_tables", fail_listing)
+    monkeypatch.setattr(adb, "list_tables", fail_listing)
+    table_id = namespace_path + ["events"]
+    assert db.table_exists(table_id) is True
+    assert await adb.table_exists(table_id) is True
+    assert db.table_exists(namespace_path + ["missing"]) is False
+    assert await adb.table_exists(namespace_path + ["missing"]) is False
+    assert ("events" in db) is (not namespace_path)
+    assert "missing" not in db
+    assert table_id == namespace_path + ["events"]
+
+    db.drop_table("events", namespace_path=namespace_path)
+    assert db.table_exists(table_id) is False
+    assert await adb.table_exists(table_id) is False
+
+
 @pytest.mark.asyncio
 async def test_table_names_async(tmp_path):
     db = lancedb.connect(tmp_path)

python/python/tests/test_namespace.py

@@ -34,6 +34,27 @@ PUSHDOWN_DATA = pa.table(
 )
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("table_id", [["events"], ["analytics", "events"], ["missing"]])
+async def test_custom_namespace_table_exists_uses_namespace_api(table_id):
+    class ExistenceNamespace(LanceNamespace):
+        def namespace_id(self):
+            return "existence-test"
+
+        def table_exists(self, request):
+            if request.id == ["missing"]:
+                raise TableNotFoundError("Table not found")
+
+        def describe_table(self, request):
+            raise AssertionError("Custom existence checks must not open tables")
+
+    namespace = ExistenceNamespace()
+    db = LanceNamespaceDBConnection(namespace)
+    adb = lancedb.AsyncLanceNamespaceDBConnection(namespace)
+    assert db.table_exists(table_id) is (table_id != ["missing"])
+    assert await adb.table_exists(table_id) is (table_id != ["missing"])
+
+
 class TestNamespaceCreateTableOptions:
     @pytest.mark.parametrize("version", ["legacy", "stable"])
     @pytest.mark.parametrize("empty", [False, True])
@@ -411,8 +432,16 @@ class TestNamespaceConnection:
         assert "test_ns" in db.list_namespaces().namespaces
 
         schema = pa.schema([pa.field("id", pa.int64())])
+        db.create_table("root_table", schema=schema)
+        assert db.table_exists(["root_table"]) is True
+        assert "root_table" in db
+        assert db.table_exists(["missing"]) is False
+        assert "missing" not in db
         table = db.create_table("test_table", schema=schema, namespace_path=["test_ns"])
         assert table.namespace == ["test_ns"]
+        assert db.table_exists(["test_ns", "test_table"]) is True
+        assert db.table_exists(["test_ns", "missing"]) is False
+        assert "test_table" not in db
         assert "test_table" in db.table_names(namespace_path=["test_ns"])
         assert "test_table" in db.list_tables(namespace_path=["test_ns"]).tables
 
@@ -420,7 +449,10 @@ class TestNamespaceConnection:
         assert opened.namespace == ["test_ns"]
 
         db.drop_table("test_table", namespace_path=["test_ns"])
+        assert db.table_exists(["test_ns", "test_table"]) is False
         assert db.list_tables(namespace_path=["test_ns"]).tables == []
+        db.drop_table("root_table")
+        assert "root_table" not in db
         db.drop_namespace(["test_ns"])
         assert "test_ns" not in db.list_namespaces().namespaces
 
@@ -942,12 +974,18 @@ class TestAsyncNamespaceConnection:
         assert "test_ns" in (await db.list_namespaces()).namespaces
 
         schema = pa.schema([pa.field("id", pa.int64())])
+        await db.create_table("root_table", schema=schema)
+        assert await db.table_exists(["root_table"]) is True
+        assert await db.table_exists(["missing"]) is False
         table = await db.create_table(
             "test_table", schema=schema, namespace_path=["test_ns"]
         )
         assert table._namespace_path == ["test_ns"]
         assert table._namespace_client is None
         assert table._route_pushdown_to_rust is True
+        assert await db.table_exists(["test_ns", "test_table"]) is True
+        assert await db.table_exists(["test_ns", "missing"]) is False
+        assert await db.table_exists(["test_table"]) is False
         assert "test_table" in await db.table_names(namespace_path=["test_ns"])
         assert "test_table" in (await db.list_tables(namespace_path=["test_ns"])).tables
 
@@ -955,7 +993,10 @@ class TestAsyncNamespaceConnection:
         assert opened._namespace_path == ["test_ns"]
 
         await db.drop_table("test_table", namespace_path=["test_ns"])
+        assert await db.table_exists(["test_ns", "test_table"]) is False
         assert (await db.list_tables(namespace_path=["test_ns"])).tables == []
+        await db.drop_table("root_table")
+        assert await db.table_exists(["root_table"]) is False
         await db.drop_namespace(["test_ns"])
         assert "test_ns" not in (await db.list_namespaces()).namespaces
 

python/python/tests/test_remote_db.py

@@ -22,6 +22,7 @@ import time
 from urllib.parse import parse_qs, urlparse
 from unittest.mock import MagicMock, patch
 import uuid
+import warnings
 from packaging.version import Version
 from lance_namespace.errors import TableNotFoundError
 
@@ -461,6 +462,31 @@ def test_remote_table_exists_rechecks_cached_table():
     assert requests == ["/v1/table/ns$table/describe/"] * 2
 
 
+@pytest.mark.parametrize("status", [200, 404, 403])
+def test_remote_membership_uses_direct_lookup(status):
+    requests = []
+
+    def handler(request):
+        requests.append(request.path)
+        assert request.command == "POST"
+        assert request.path == "/v1/table/t129/describe/"
+        request.send_response(status)
+        request.send_header("Content-Type", "application/json")
+        request.end_headers()
+        request.wfile.write(b"{}" if status == 200 else b"lookup failed")
+
+    with mock_lancedb_connection(handler) as db:
+        with warnings.catch_warnings():
+            warnings.simplefilter("error", DeprecationWarning)
+            if status in (200, 404):
+                assert ("t129" in db) is (status == 200)
+            else:
+                with pytest.raises(HttpError) as exc_info:
+                    "t129" in db
+                assert exc_info.value.status_code == status
+    assert requests == ["/v1/table/t129/describe/"]
+
+
 def paginated_table_listing_handler(request):
     url = urlparse(request.path)
     assert url.path in {

python/python/tests/test_sync_api_parity.py

@@ -58,8 +58,8 @@ def test_legacy_table_replace_field_metadata(legacy_table):
     ]
 
 
-def respond(request, body):
-    request.send_response(200)
+def respond(request, body, status=200):
+    request.send_response(status)
     request.send_header("Content-Type", "application/json")
     request.end_headers()
     request.wfile.write(json.dumps(body).encode())
@@ -78,12 +78,20 @@ def test_remote_unsupported_properties(attribute):
 
 
 @pytest.mark.parametrize("tables", [[], ["first", "last"]])
-def test_remote_len_and_contains_follow_pages(tables):
+def test_remote_len_paginates_and_contains_looks_up_table(tables):
     requests = []
 
     def handler(request):
         requests.append(request.path)
-        query = parse_qs(urlparse(request.path).query)
+        url = urlparse(request.path)
+        if request.command == "POST":
+            name = url.path.split("/")[3]
+            assert url.path == f"/v1/table/{name}/describe/"
+            respond(request, {}, status=200 if name in tables else 404)
+            return
+        assert request.command == "GET"
+        assert url.path == "/v1/namespace/$/table/list"
+        query = parse_qs(url.query)
         if tables and "page_token" not in query:
             respond(request, {"tables": [tables[0]], "page_token": "opaque-next"})
         else:
@@ -93,11 +101,12 @@ def test_remote_len_and_contains_follow_pages(tables):
 
     with mock_lancedb_connection(handler) as db:
         assert len(db) == len(tables)
-        assert ("last" in db) == bool(tables)
-        assert "missing" not in db
+        assert len(requests) == (2 if tables else 1)
         requests.clear()
-        assert ("first" in db) == bool(tables)
-        assert len(requests) == 1
+        for name in ("last", "missing", "first"):
+            assert (name in db) is (name in tables)
+            assert requests == [f"/v1/table/{name}/describe/"]
+            requests.clear()
 
 
 @pytest.mark.parametrize("namespace", [[], ["parent", "child"]])