diff --git a/docs/src/python/python.md b/docs/src/python/python.md index eccaae6ce..8de5b5517 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -42,6 +42,18 @@ table names. `db.list_tables().tables` is a list of the first page; `limit` controls its page size and `page_token` resumes at a saved token. The same patterns apply to `list_namespaces()`. Other resource listings use `page_limit`. +For existence checks, use `db.table_exists(["events"])` or `"events" in db`. +These look up the table directly and work even when the database has more than +100 tables. For a table in a namespace, include the namespace segments: + +```python +exists = db.table_exists(["analytics", "events"]) +``` + +When migrating from `name in db.table_names()`, use `name in db` or +`db.table_exists([name])`. Checking `name in db.list_tables().tables` only +searches the first page and can miss an existing table. + Resource listings (`list_storage_integrations`, `list_secrets`, `list_views`, `list_jobs`, `list_functions`, and `list_materialized_views`) return lazy iterators. Use `list(db.list_views())` to collect synchronous results, or `[name async for name in db.list_views()]` with @@ -1055,6 +1067,11 @@ the page list does not consume the iterator. `page.page_token` is the token for the next request; drain cached results before using it to resume in a new listing. The same behavior applies to namespaces. +Use `await db.table_exists(["events"])` to check for a table in the root +namespace, or `await db.table_exists(["analytics", "events"])` for a table in +a namespace. Checking membership in an awaited listing's `.tables` attribute +only searches one page. + ::: lancedb.listing.AsyncTableListing ::: lancedb.listing.AsyncNamespaceListing diff --git a/docs/web/management/table.mdx b/docs/web/management/table.mdx index 2a05b4a0f..23fdbdb4a 100644 --- a/docs/web/management/table.mdx +++ b/docs/web/management/table.mdx @@ -201,7 +201,7 @@ setting. See [Limits](/basics/limits#schema). ```python Python icon="python" tbl = db.open_table("documents") -db.table_names() +names = list(db.list_tables()) ``` ```typescript TypeScript icon="square-js" @@ -230,6 +230,27 @@ listing. A missing table raises `ValueError` on a local directory and A listing returns one page of names at a time, so check every page. TypeScript has no `tableExists`. +{/* +Draft for the table-existence lookup release (#4974). Once released, replace +the two table-existence paragraphs above with this guidance: + +For Python local, remote, and namespace connections, use +`db.table_exists(["documents"])` or `"documents" in db` to check whether a root +table exists. For an async connection, use +`await db.table_exists(["documents"])`. Include namespace segments before the +table name, for example `["analytics", "documents"]`. These checks return +`False` when the table is missing and propagate other errors, including +permission errors. They look up the table directly, regardless of the number +of tables in the database. + +When migrating from `name in db.table_names()`, use `name in db` or +`db.table_exists([name])`. To collect every table name, use +`list(db.list_tables())`, or `[name async for name in db.list_tables()]` on an +async connection. The `.tables` attribute holds only one page (100 names by +default), so membership checks against it can miss existing tables. TypeScript +has no `tableExists`. +*/} + ## Rename and clone tables {#rename-clone} diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index 73c8305ff..84ebb41a1 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -322,6 +322,9 @@ class DBConnection(EnforceOverrides): def table_exists(self, table_id: List[str]) -> bool: """Check if a table exists. + Use this method instead of checking membership in + ``list_tables().tables``, which contains only one page of table names. + Parameters ---------- table_id: List[str] @@ -331,12 +334,20 @@ class DBConnection(EnforceOverrides): Returns ------- bool - True if the table exists, False otherwise. + True if the table exists, False if it is not found. + Other errors, including permission errors, are propagated. Raises ------ NotImplementedError - If the connection type does not support namespace operations. + If the connection type does not support table existence checks. + + Examples + -------- + >>> def check_table(conn): + ... root_exists = conn.table_exists(["events"]) + ... nested_exists = conn.table_exists(["analytics", "events"]) + ... return root_exists, nested_exists """ raise NotImplementedError( "Namespace operations are not supported for this connection type" @@ -348,7 +359,11 @@ class DBConnection(EnforceOverrides): page_token: Optional[str] = None, limit: Optional[int] = None, ) -> TableListing: - """List tables in this database, one page at a time. + """Iterate over all tables in this database, fetching pages as needed. + + ``list_tables().tables`` contains only the first page (100 names by + default). Use ``list(list_tables())`` to collect all names, or + ``table_exists([name])`` to check whether a particular table exists. Parameters ---------- @@ -934,7 +949,7 @@ class DBConnection(EnforceOverrides): def __contains__(self, name: str) -> bool: """Return whether the root namespace contains the named table.""" - return name in self._all_table_names() + return self.table_exists([name]) def namespace_client(self) -> LanceNamespace: """Get the equivalent namespace client for this connection. @@ -1849,6 +1864,10 @@ class LanceDBConnection(DBConnection): ) -> DescribeNamespaceResponse: return LOOP.run(self._conn.describe_namespace(namespace_path=namespace_path)) + @override + def table_exists(self, table_id: List[str]) -> bool: + return LOOP.run(self._conn.table_exists(table_id)) + @override def list_tables( self, @@ -1856,7 +1875,11 @@ class LanceDBConnection(DBConnection): page_token: Optional[str] = None, limit: Optional[int] = None, ) -> TableListing: - """List tables in this database, one page at a time. + """Iterate over all tables in this database, fetching pages as needed. + + The ``tables`` attribute contains only the current page (100 names by + default). Iterate over the listing to visit all pages, or use + ``table_exists([name])`` to check for a particular table. Parameters ---------- @@ -1894,7 +1917,9 @@ class LanceDBConnection(DBConnection): """Get the names of all tables in the database. The names are sorted. .. deprecated:: - Use :meth:`list_tables` instead, which provides proper pagination support. + Iterate over :meth:`list_tables` to get all names, or use + :meth:`table_exists` to check for a particular table. The listing's + ``tables`` attribute contains only one page. Parameters ---------- @@ -1914,7 +1939,8 @@ class LanceDBConnection(DBConnection): import warnings warnings.warn( - "table_names() is deprecated, use list_tables() instead", + "table_names() is deprecated, iterate list_tables() for all names " + "or use table_exists([name]) to check existence", DeprecationWarning, stacklevel=2, ) @@ -2878,6 +2904,9 @@ class AsyncConnection(object): async def table_exists(self, table_id: List[str]) -> bool: """Check if a table exists. + Use this method instead of checking membership in an awaited listing's + ``tables`` attribute, which contains only one page of table names. + Parameters ---------- table_id: List[str] @@ -2903,7 +2932,11 @@ class AsyncConnection(object): page_token: Optional[str] = None, limit: Optional[int] = None, ) -> AsyncTableListing: - """List tables in this database, one page at a time. + """Iterate over all tables in this database, fetching pages as needed. + + The ``tables`` attribute contains only the current page (100 names by + default). Iterate over the listing to visit all pages, or use + ``table_exists([name])`` to check for a particular table. Parameters ---------- @@ -2942,7 +2975,9 @@ class AsyncConnection(object): """List all tables in this database, in sorted order .. deprecated:: - Use :meth:`list_tables` instead, which provides proper pagination support. + Iterate over :meth:`list_tables` to get all names, or use + :meth:`table_exists` to check for a particular table. The listing's + ``tables`` attribute contains only one page. Parameters ---------- @@ -2966,7 +3001,8 @@ class AsyncConnection(object): import warnings warnings.warn( - "table_names() is deprecated, use list_tables() instead", + "table_names() is deprecated, iterate list_tables() for all names " + "or use table_exists([name]) to check existence", DeprecationWarning, stacklevel=2, ) diff --git a/python/python/lancedb/listing.py b/python/python/lancedb/listing.py index 8abc9d85b..59aba1431 100644 --- a/python/python/lancedb/listing.py +++ b/python/python/lancedb/listing.py @@ -175,6 +175,9 @@ class _AsyncPagedNameListing(AsyncIterator[str]): class AsyncTableListing(_AsyncPagedNameListing): """Awaitable table iterator; awaiting loads one page. + Iteration fetches all pages. The ``tables`` attribute contains only the + current page, so use ``await db.table_exists([name])`` for existence checks. + For example:: page = await db.list_tables(limit=20) @@ -229,6 +232,10 @@ class _PagedNameListing(Iterator[str]): class TableListing(_PagedNameListing): """Table iterator retaining the current page as a list. + Iteration fetches all pages. The ``tables`` attribute contains only the + current page, so use ``db.table_exists([name])`` or ``name in db`` for + existence checks. Use ``list(db.list_tables())`` to collect all names. + For example:: page = db.list_tables(limit=20) @@ -239,6 +246,7 @@ class TableListing(_PagedNameListing): @property def tables(self) -> list[str]: + """A copy of the table names in the most recently fetched page.""" return self._inner.tables diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py index 3356b363c..aa9025641 100644 --- a/python/python/lancedb/namespace.py +++ b/python/python/lancedb/namespace.py @@ -579,12 +579,15 @@ class LanceNamespaceDBConnection(DBConnection): If limit is None, return all table names; zero returns an empty list. .. deprecated:: - Use :meth:`list_tables` instead, which provides proper pagination support. + Iterate over :meth:`list_tables` to get all names, or use + :meth:`table_exists` to check for a particular table. The listing's + ``tables`` attribute contains only one page. """ import warnings warnings.warn( - "table_names() is deprecated, use list_tables() instead", + "table_names() is deprecated, iterate list_tables() for all names " + "or use table_exists([name]) to check existence", DeprecationWarning, stacklevel=2, ) @@ -1015,6 +1018,8 @@ class LanceNamespaceDBConnection(DBConnection): bool True if the table exists, False otherwise. """ + if self._route_pushdown_to_rust: + return LOOP.run(self._inner.table_exists(table_id)) request = TableExistsRequest(id=table_id) try: self._namespace_client.table_exists(request) @@ -1030,7 +1035,11 @@ class LanceNamespaceDBConnection(DBConnection): limit: Optional[int] = None, ) -> TableListing: """ - List all tables in this database with pagination support. + Iterate over all tables in this database, fetching pages as needed. + + The ``tables`` attribute contains only the current page (100 names by + default). Iterate over the listing to visit all pages, or use + ``table_exists([name])`` to check for a particular table. Parameters ---------- @@ -1227,12 +1236,15 @@ class AsyncLanceNamespaceDBConnection: If limit is None, return all table names; zero returns an empty list. .. deprecated:: - Use :meth:`list_tables` instead, which provides proper pagination support. + Iterate over :meth:`list_tables` to get all names, or use + :meth:`table_exists` to check for a particular table. The listing's + ``tables`` attribute contains only one page. """ import warnings warnings.warn( - "table_names() is deprecated, use list_tables() instead", + "table_names() is deprecated, iterate list_tables() for all names " + "or use table_exists([name]) to check existence", DeprecationWarning, stacklevel=2, ) @@ -1614,6 +1626,8 @@ class AsyncLanceNamespaceDBConnection: bool True if the table exists, False otherwise. """ + if self._route_pushdown_to_rust: + return await self._inner.table_exists(table_id) request = TableExistsRequest(id=table_id) try: self._namespace_client.table_exists(request) @@ -1628,7 +1642,11 @@ class AsyncLanceNamespaceDBConnection: limit: Optional[int] = None, ) -> AsyncTableListing: """ - List all tables in this database with pagination support. + Iterate over all tables in this database, fetching pages as needed. + + The ``tables`` attribute contains only the current page (100 names by + default). Iterate over the listing to visit all pages, or use + ``table_exists([name])`` to check for a particular table. Parameters ---------- diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index ebceba16b..21ad37b36 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -430,7 +430,11 @@ class RemoteDBConnection(DBConnection): page_token: Optional[str] = None, limit: Optional[int] = None, ) -> TableListing: - """List a page of tables in this database. + """Iterate over all tables in this database, fetching pages as needed. + + The ``tables`` attribute contains only the current page (100 names by + default). Iterate over the listing to visit all pages, or use + ``table_exists([name])`` to check for a particular table. Parameters ---------- @@ -468,7 +472,9 @@ class RemoteDBConnection(DBConnection): """List the names of all tables in the database. .. deprecated:: - Use :meth:`list_tables` instead, which provides proper pagination support. + Iterate over :meth:`list_tables` to get all names, or use + :meth:`table_exists` to check for a particular table. The listing's + ``tables`` attribute contains only one page. Parameters ---------- @@ -488,7 +494,8 @@ class RemoteDBConnection(DBConnection): import warnings warnings.warn( - "table_names() is deprecated, use list_tables() instead", + "table_names() is deprecated, iterate list_tables() for all names " + "or use table_exists([name]) to check existence", DeprecationWarning, stacklevel=2, ) diff --git a/python/python/tests/test_db.py b/python/python/tests/test_db.py index 64bf928c0..80a98bc18 100644 --- a/python/python/tests/test_db.py +++ b/python/python/tests/test_db.py @@ -282,9 +282,7 @@ async def test_table_listing_defaults_and_zero_limit(tmp_path): assert empty.page_token is None -def test_db_contains_stops_after_matching_table_page( - tmp_db: lancedb.DBConnection, monkeypatch -): +def test_db_len_includes_all_table_pages(tmp_db: lancedb.DBConnection, monkeypatch): calls = [] pages = { None: SimpleNamespace(tables=["table_0", "table_1"], page_token="next"), @@ -297,18 +295,44 @@ def test_db_contains_stops_after_matching_table_page( monkeypatch.setattr(tmp_db, "list_tables", list_tables) - assert "table_1" in tmp_db - assert calls == [None] - - calls.clear() - assert "table_2" in tmp_db - assert calls == [None, "next"] - - calls.clear() assert len(tmp_db) == 3 assert calls == [None, "next"] +@pytest.mark.asyncio +@pytest.mark.parametrize("namespace_path", [[], ["analytics", "nested"]]) +async def test_local_table_exists_without_listing( + tmp_path, monkeypatch, namespace_path +): + db = lancedb.connect(tmp_path) + for depth in range(1, len(namespace_path) + 1): + db.create_namespace(namespace_path[:depth]) + db.create_table("events", data=[{"id": 1}], namespace_path=namespace_path) + if not namespace_path: + for idx in range(100): + (tmp_path / f"a{idx:03}.lance").mkdir() + assert "events" not in db.list_tables().tables + adb = await lancedb.connect_async(tmp_path) + + def fail_listing(*args, **kwargs): + raise AssertionError("Existence checks must not list tables") + + monkeypatch.setattr(db, "list_tables", fail_listing) + monkeypatch.setattr(adb, "list_tables", fail_listing) + table_id = namespace_path + ["events"] + assert db.table_exists(table_id) is True + assert await adb.table_exists(table_id) is True + assert db.table_exists(namespace_path + ["missing"]) is False + assert await adb.table_exists(namespace_path + ["missing"]) is False + assert ("events" in db) is (not namespace_path) + assert "missing" not in db + assert table_id == namespace_path + ["events"] + + db.drop_table("events", namespace_path=namespace_path) + assert db.table_exists(table_id) is False + assert await adb.table_exists(table_id) is False + + @pytest.mark.asyncio async def test_table_names_async(tmp_path): db = lancedb.connect(tmp_path) diff --git a/python/python/tests/test_namespace.py b/python/python/tests/test_namespace.py index 5eb9c02bd..9bce1059d 100644 --- a/python/python/tests/test_namespace.py +++ b/python/python/tests/test_namespace.py @@ -34,6 +34,27 @@ PUSHDOWN_DATA = pa.table( ) +@pytest.mark.asyncio +@pytest.mark.parametrize("table_id", [["events"], ["analytics", "events"], ["missing"]]) +async def test_custom_namespace_table_exists_uses_namespace_api(table_id): + class ExistenceNamespace(LanceNamespace): + def namespace_id(self): + return "existence-test" + + def table_exists(self, request): + if request.id == ["missing"]: + raise TableNotFoundError("Table not found") + + def describe_table(self, request): + raise AssertionError("Custom existence checks must not open tables") + + namespace = ExistenceNamespace() + db = LanceNamespaceDBConnection(namespace) + adb = lancedb.AsyncLanceNamespaceDBConnection(namespace) + assert db.table_exists(table_id) is (table_id != ["missing"]) + assert await adb.table_exists(table_id) is (table_id != ["missing"]) + + class TestNamespaceCreateTableOptions: @pytest.mark.parametrize("version", ["legacy", "stable"]) @pytest.mark.parametrize("empty", [False, True]) @@ -411,8 +432,16 @@ class TestNamespaceConnection: assert "test_ns" in db.list_namespaces().namespaces schema = pa.schema([pa.field("id", pa.int64())]) + db.create_table("root_table", schema=schema) + assert db.table_exists(["root_table"]) is True + assert "root_table" in db + assert db.table_exists(["missing"]) is False + assert "missing" not in db table = db.create_table("test_table", schema=schema, namespace_path=["test_ns"]) assert table.namespace == ["test_ns"] + assert db.table_exists(["test_ns", "test_table"]) is True + assert db.table_exists(["test_ns", "missing"]) is False + assert "test_table" not in db assert "test_table" in db.table_names(namespace_path=["test_ns"]) assert "test_table" in db.list_tables(namespace_path=["test_ns"]).tables @@ -420,7 +449,10 @@ class TestNamespaceConnection: assert opened.namespace == ["test_ns"] db.drop_table("test_table", namespace_path=["test_ns"]) + assert db.table_exists(["test_ns", "test_table"]) is False assert db.list_tables(namespace_path=["test_ns"]).tables == [] + db.drop_table("root_table") + assert "root_table" not in db db.drop_namespace(["test_ns"]) assert "test_ns" not in db.list_namespaces().namespaces @@ -942,12 +974,18 @@ class TestAsyncNamespaceConnection: assert "test_ns" in (await db.list_namespaces()).namespaces schema = pa.schema([pa.field("id", pa.int64())]) + await db.create_table("root_table", schema=schema) + assert await db.table_exists(["root_table"]) is True + assert await db.table_exists(["missing"]) is False table = await db.create_table( "test_table", schema=schema, namespace_path=["test_ns"] ) assert table._namespace_path == ["test_ns"] assert table._namespace_client is None assert table._route_pushdown_to_rust is True + assert await db.table_exists(["test_ns", "test_table"]) is True + assert await db.table_exists(["test_ns", "missing"]) is False + assert await db.table_exists(["test_table"]) is False assert "test_table" in await db.table_names(namespace_path=["test_ns"]) assert "test_table" in (await db.list_tables(namespace_path=["test_ns"])).tables @@ -955,7 +993,10 @@ class TestAsyncNamespaceConnection: assert opened._namespace_path == ["test_ns"] await db.drop_table("test_table", namespace_path=["test_ns"]) + assert await db.table_exists(["test_ns", "test_table"]) is False assert (await db.list_tables(namespace_path=["test_ns"])).tables == [] + await db.drop_table("root_table") + assert await db.table_exists(["root_table"]) is False await db.drop_namespace(["test_ns"]) assert "test_ns" not in (await db.list_namespaces()).namespaces diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 407596d26..952437051 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -22,6 +22,7 @@ import time from urllib.parse import parse_qs, urlparse from unittest.mock import MagicMock, patch import uuid +import warnings from packaging.version import Version from lance_namespace.errors import TableNotFoundError @@ -461,6 +462,31 @@ def test_remote_table_exists_rechecks_cached_table(): assert requests == ["/v1/table/ns$table/describe/"] * 2 +@pytest.mark.parametrize("status", [200, 404, 403]) +def test_remote_membership_uses_direct_lookup(status): + requests = [] + + def handler(request): + requests.append(request.path) + assert request.command == "POST" + assert request.path == "/v1/table/t129/describe/" + request.send_response(status) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(b"{}" if status == 200 else b"lookup failed") + + with mock_lancedb_connection(handler) as db: + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + if status in (200, 404): + assert ("t129" in db) is (status == 200) + else: + with pytest.raises(HttpError) as exc_info: + "t129" in db + assert exc_info.value.status_code == status + assert requests == ["/v1/table/t129/describe/"] + + def paginated_table_listing_handler(request): url = urlparse(request.path) assert url.path in { diff --git a/python/python/tests/test_sync_api_parity.py b/python/python/tests/test_sync_api_parity.py index a12efd0c9..230fb45c6 100644 --- a/python/python/tests/test_sync_api_parity.py +++ b/python/python/tests/test_sync_api_parity.py @@ -58,8 +58,8 @@ def test_legacy_table_replace_field_metadata(legacy_table): ] -def respond(request, body): - request.send_response(200) +def respond(request, body, status=200): + request.send_response(status) request.send_header("Content-Type", "application/json") request.end_headers() request.wfile.write(json.dumps(body).encode()) @@ -78,12 +78,20 @@ def test_remote_unsupported_properties(attribute): @pytest.mark.parametrize("tables", [[], ["first", "last"]]) -def test_remote_len_and_contains_follow_pages(tables): +def test_remote_len_paginates_and_contains_looks_up_table(tables): requests = [] def handler(request): requests.append(request.path) - query = parse_qs(urlparse(request.path).query) + url = urlparse(request.path) + if request.command == "POST": + name = url.path.split("/")[3] + assert url.path == f"/v1/table/{name}/describe/" + respond(request, {}, status=200 if name in tables else 404) + return + assert request.command == "GET" + assert url.path == "/v1/namespace/$/table/list" + query = parse_qs(url.query) if tables and "page_token" not in query: respond(request, {"tables": [tables[0]], "page_token": "opaque-next"}) else: @@ -93,11 +101,12 @@ def test_remote_len_and_contains_follow_pages(tables): with mock_lancedb_connection(handler) as db: assert len(db) == len(tables) - assert ("last" in db) == bool(tables) - assert "missing" not in db + assert len(requests) == (2 if tables else 1) requests.clear() - assert ("first" in db) == bool(tables) - assert len(requests) == 1 + for name in ("last", "missing", "first"): + assert (name in db) is (name in tables) + assert requests == [f"/v1/table/{name}/describe/"] + requests.clear() @pytest.mark.parametrize("namespace", [[], ["parent", "child"]])