mirror of
https://github.com/lancedb/lancedb.git
synced 2026-10-12 23:02:28 +00:00
fix: align table listing pagination across backends (#4412)
Fixes #4379 Remote `table_names` discarded the continuation token after the first root-listing request, silently returning the server's default page of 10 names. Remote `list_tables` also inherited that server default, while local listings had no SDK page limit. Python's synchronous and namespace wrappers still imposed a default limit of 10 names. The namespace adapter also discarded continuation tokens, so an unlimited REST namespace listing could stop at the server's page cap. Follow server-provided root listing tokens until the requested name limit or the end of the listing, preserving `start_after`, cache population, and guards against empty or repeated tokens. The namespace adapter collects every server page, then sorts names and applies the name cursor and total limit: namespace tokens are opaque, and page order need not be name order. Empty tokens end the listing; token cycles return an error. Python sync and namespace `table_names` wrappers now default to `None`, matching the asynchronous and TypeScript APIs. Namespace overrides accept the base connection's optional limit, avoiding import-time signature errors on Python 3.10 and 3.11. Set the `list_tables` default to 100 at the shared connection boundary, including namespaces, and return empty results for a zero limit without sending an invalid server request. Update the SDK documentation to describe these defaults. `list_tables()` is now a paginated call in both modes: callers with more than 100 tables must follow the returned token or supply an explicit larger limit. `table_names()` without a limit returns every name. Namespace table-name calls enumerate the namespace before applying name cursors and limits; use `list_tables()` for paginated access. Validation: - Regression cases reproduce the original root-listing truncation and cover 130 tables, opaque tokens, short pages, explicit limits, `start_after`, and zero limits. - Relevant Rust remote database, connection, and local listing test groups: 175 passed. Namespace adapter tests: 23 passed, including opaque tokens, unordered and empty pages, name cursors, limits, zero requests, and token cycles. - Rebuilt the Python extension; namespace and local/mocked remote listing tests using `uv.lock`: 46 passed on Python 3.10 and 46 passed on Python 3.13. - REST namespace regressions reproduced the truncation before the fix and now return all 12 names from a server capped at 10 per response, for sync/async connections at root and child paths. They also cover unordered pages, name cursors, explicit limits, and zero requests. The exact Python 3.10 import error also reproduced before the optional-signature fix. - TypeScript connection test file: 38 passed; `pnpm build`, `pnpm lint`, and `pnpm run docs` passed. - `cargo check --quiet --features remote --tests --examples` and `cargo clippy --quiet --features remote --tests --examples` passed. - Rust formatting, Python Ruff formatting/lint, and diff whitespace checks passed. <!-- lance-gatekeeper-fix:v1 agent=43f92f1f05ac9ca74229af590945cb88 generation=1 --> --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com>
This commit is contained in:
lancedb-gatefixer[bot]
+20
-14
@@ -317,7 +317,7 @@ class DBConnection(EnforceOverrides):
|
||||
page_token: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> ListTablesResponse:
|
||||
"""List all tables in this database with pagination support.
|
||||
"""List a page of tables in this database.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -328,7 +328,8 @@ class DBConnection(EnforceOverrides):
|
||||
Token for pagination. Use the token from a previous response
|
||||
to get the next page of results.
|
||||
limit: int, optional
|
||||
The maximum number of results to return.
|
||||
The maximum number of results to return, default 100.
|
||||
Zero returns an empty page without a continuation token.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -343,7 +344,7 @@ class DBConnection(EnforceOverrides):
|
||||
def table_names(
|
||||
self,
|
||||
page_token: Optional[str] = None,
|
||||
limit: int = 10,
|
||||
limit: Optional[int] = None,
|
||||
*,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> Iterable[str]:
|
||||
@@ -357,8 +358,9 @@ class DBConnection(EnforceOverrides):
|
||||
page_token: str, optional
|
||||
The token to use for pagination. If not present, start from the beginning.
|
||||
Typically, this token is last table name from the previous page.
|
||||
limit: int, default 10
|
||||
The size of the page to return.
|
||||
limit: int, optional
|
||||
The maximum number of tables to return. None returns all tables;
|
||||
zero returns an empty list.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -1490,7 +1492,7 @@ class LanceDBConnection(DBConnection):
|
||||
page_token: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> ListTablesResponse:
|
||||
"""List all tables in this database with pagination support.
|
||||
"""List a page of tables in this database.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -1501,7 +1503,8 @@ class LanceDBConnection(DBConnection):
|
||||
Token for pagination. Use the token from a previous response
|
||||
to get the next page of results.
|
||||
limit: int, optional
|
||||
The maximum number of results to return.
|
||||
The maximum number of results to return, default 100.
|
||||
Zero returns an empty page without a continuation token.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -1520,7 +1523,7 @@ class LanceDBConnection(DBConnection):
|
||||
def table_names(
|
||||
self,
|
||||
page_token: Optional[str] = None,
|
||||
limit: int = 10,
|
||||
limit: Optional[int] = None,
|
||||
*,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> Iterable[str]:
|
||||
@@ -1535,8 +1538,9 @@ class LanceDBConnection(DBConnection):
|
||||
The namespace to list tables in.
|
||||
page_token: str, optional
|
||||
The token to use for pagination.
|
||||
limit: int, default 10
|
||||
The maximum number of tables to return.
|
||||
limit: int, optional
|
||||
The maximum number of tables to return. None returns all tables;
|
||||
zero returns an empty list.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -2366,7 +2370,7 @@ class AsyncConnection(object):
|
||||
page_token: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> ListTablesResponse:
|
||||
"""List all tables in this database with pagination support.
|
||||
"""List a page of tables in this database.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -2377,7 +2381,8 @@ class AsyncConnection(object):
|
||||
Token for pagination. Use the token from a previous response
|
||||
to get the next page of results.
|
||||
limit: int, optional
|
||||
The maximum number of results to return.
|
||||
The maximum number of results to return, default 100.
|
||||
Zero returns an empty page without a continuation token.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -2414,8 +2419,9 @@ class AsyncConnection(object):
|
||||
|
||||
This can be combined with limit to implement pagination by setting this to
|
||||
the last table name from the previous page.
|
||||
limit: int, default 10
|
||||
The number of results to return.
|
||||
limit: int, optional
|
||||
The maximum number of tables to return. None returns all tables;
|
||||
zero returns an empty list.
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
@@ -522,13 +522,15 @@ class LanceNamespaceDBConnection(DBConnection):
|
||||
def table_names(
|
||||
self,
|
||||
page_token: Optional[str] = None,
|
||||
limit: int = 10,
|
||||
limit: Optional[int] = None,
|
||||
*,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> Iterable[str]:
|
||||
"""
|
||||
List table names in the database.
|
||||
|
||||
If limit is None, return all table names; zero returns an empty list.
|
||||
|
||||
.. deprecated::
|
||||
Use :meth:`list_tables` instead, which provides proper pagination support.
|
||||
"""
|
||||
@@ -1157,13 +1159,15 @@ class AsyncLanceNamespaceDBConnection:
|
||||
async def table_names(
|
||||
self,
|
||||
page_token: Optional[str] = None,
|
||||
limit: int = 10,
|
||||
limit: Optional[int] = None,
|
||||
*,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> Iterable[str]:
|
||||
"""
|
||||
List table names in the namespace.
|
||||
|
||||
If limit is None, return all table names; zero returns an empty list.
|
||||
|
||||
.. deprecated::
|
||||
Use :meth:`list_tables` instead, which provides proper pagination support.
|
||||
"""
|
||||
|
||||
@@ -379,7 +379,7 @@ class RemoteDBConnection(DBConnection):
|
||||
page_token: Optional[str] = None,
|
||||
limit: Optional[int] = None,
|
||||
) -> ListTablesResponse:
|
||||
"""List all tables in this database with pagination support.
|
||||
"""List a page of tables in this database.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -390,7 +390,8 @@ class RemoteDBConnection(DBConnection):
|
||||
Token for pagination. Use the token from a previous response
|
||||
to get the next page of results.
|
||||
limit: int, optional
|
||||
The maximum number of results to return.
|
||||
The maximum number of results to return, default 100.
|
||||
Zero returns an empty page without a continuation token.
|
||||
|
||||
Returns
|
||||
-------
|
||||
@@ -409,7 +410,7 @@ class RemoteDBConnection(DBConnection):
|
||||
def table_names(
|
||||
self,
|
||||
page_token: Optional[str] = None,
|
||||
limit: int = 10,
|
||||
limit: Optional[int] = None,
|
||||
*,
|
||||
namespace_path: Optional[List[str]] = None,
|
||||
) -> Iterable[str]:
|
||||
@@ -425,8 +426,9 @@ class RemoteDBConnection(DBConnection):
|
||||
Empty list represents root namespace.
|
||||
page_token: str
|
||||
The last token to start the new page.
|
||||
limit: int, default 10
|
||||
The maximum number of tables to return for each page.
|
||||
limit: int, optional
|
||||
The maximum number of tables to return. None returns all tables;
|
||||
zero returns an empty list.
|
||||
|
||||
Returns
|
||||
-------
|
||||
|
||||
@@ -245,6 +245,40 @@ def test_db_contains_and_len_include_all_table_name_pages(tmp_db: lancedb.DBConn
|
||||
assert "does_not_exist" not in tmp_db
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_table_listing_defaults_and_zero_limit(tmp_path):
|
||||
names = [f"t{i:03}" for i in range(130)]
|
||||
for name in names:
|
||||
(tmp_path / f"{name}.lance").mkdir()
|
||||
|
||||
db = lancedb.connect(tmp_path)
|
||||
assert db.table_names() == names
|
||||
assert db.table_names(limit=None) == names
|
||||
assert db.table_names(limit=0) == []
|
||||
first = db.list_tables()
|
||||
assert first.tables == names[:100]
|
||||
assert first.page_token
|
||||
second = db.list_tables(page_token=first.page_token)
|
||||
assert second.tables == names[100:]
|
||||
assert second.page_token is None
|
||||
assert db.list_tables(limit=200).tables == names
|
||||
empty = db.list_tables(limit=0)
|
||||
assert empty.tables == []
|
||||
assert empty.page_token is None
|
||||
|
||||
adb = await lancedb.connect_async(tmp_path)
|
||||
assert await adb.table_names() == names
|
||||
assert await adb.table_names(limit=0) == []
|
||||
first = await adb.list_tables()
|
||||
assert first.tables == names[:100]
|
||||
second = await adb.list_tables(page_token=first.page_token)
|
||||
assert second.tables == names[100:]
|
||||
assert second.page_token is None
|
||||
empty = await adb.list_tables(limit=0)
|
||||
assert empty.tables == []
|
||||
assert empty.page_token is None
|
||||
|
||||
|
||||
def test_db_contains_stops_after_matching_table_page(
|
||||
tmp_db: lancedb.DBConnection, monkeypatch
|
||||
):
|
||||
|
||||
@@ -6,6 +6,10 @@
|
||||
import tempfile
|
||||
import shutil
|
||||
import importlib
|
||||
import http.server
|
||||
import json
|
||||
import threading
|
||||
from urllib.parse import parse_qs, unquote, urlparse
|
||||
import pytest
|
||||
import pyarrow as pa
|
||||
import lancedb
|
||||
@@ -73,6 +77,132 @@ def _namespace_lance_table(namespace_client: _NamespaceClient) -> LanceTable:
|
||||
return table
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("namespace_path", [[], ["test_ns"]])
|
||||
async def test_namespace_table_names_defaults_and_limits(tmp_path, namespace_path):
|
||||
db = lancedb.connect_namespace("dir", {"root": str(tmp_path)})
|
||||
if namespace_path:
|
||||
db.create_namespace(namespace_path)
|
||||
names = [f"t{idx:03}" for idx in range(12)]
|
||||
schema = pa.schema([pa.field("id", pa.int64())])
|
||||
for name in names:
|
||||
db.create_table(name, schema=schema, namespace_path=namespace_path)
|
||||
|
||||
assert list(db.table_names(namespace_path=namespace_path)) == names
|
||||
assert list(db.table_names(limit=None, namespace_path=namespace_path)) == names
|
||||
assert list(db.table_names(limit=0, namespace_path=namespace_path)) == []
|
||||
assert list(db.table_names(limit=2, namespace_path=namespace_path)) == names[:2]
|
||||
assert (
|
||||
list(
|
||||
db.table_names(page_token=names[1], limit=2, namespace_path=namespace_path)
|
||||
)
|
||||
== names[2:4]
|
||||
)
|
||||
|
||||
adb = lancedb.connect_namespace_async("dir", {"root": str(tmp_path)})
|
||||
assert list(await adb.table_names(namespace_path=namespace_path)) == names
|
||||
assert (
|
||||
list(await adb.table_names(limit=None, namespace_path=namespace_path)) == names
|
||||
)
|
||||
assert list(await adb.table_names(limit=0, namespace_path=namespace_path)) == []
|
||||
assert (
|
||||
list(await adb.table_names(limit=2, namespace_path=namespace_path)) == names[:2]
|
||||
)
|
||||
assert (
|
||||
list(
|
||||
await adb.table_names(
|
||||
page_token=names[1], limit=2, namespace_path=namespace_path
|
||||
)
|
||||
)
|
||||
== names[2:4]
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def paginated_rest_namespace():
|
||||
names = [f"t{idx:03}" for idx in reversed(range(12))]
|
||||
requests = []
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
url = urlparse(self.path)
|
||||
query = parse_qs(url.query)
|
||||
requests.append((unquote(url.path), query))
|
||||
token = query.get("page_token", [None])[0]
|
||||
if token not in {None, "opaque-next"}:
|
||||
self.send_error(400, "Unknown page token")
|
||||
return
|
||||
start = 0 if token is None else 10
|
||||
body = {"tables": names[start : start + 10]}
|
||||
if start == 0:
|
||||
body["page_token"] = "opaque-next"
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.end_headers()
|
||||
self.wfile.write(json.dumps(body).encode())
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
with http.server.HTTPServer(("127.0.0.1", 0), Handler) as server:
|
||||
thread = threading.Thread(target=server.serve_forever)
|
||||
thread.start()
|
||||
try:
|
||||
yield {"uri": f"http://127.0.0.1:{server.server_port}"}, requests
|
||||
finally:
|
||||
server.shutdown()
|
||||
thread.join()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("namespace_path", [[], ["n"]])
|
||||
async def test_rest_namespace_table_names_collects_pages(
|
||||
paginated_rest_namespace, namespace_path
|
||||
):
|
||||
properties, requests = paginated_rest_namespace
|
||||
names = [f"t{idx:03}" for idx in range(12)]
|
||||
db = lancedb.connect_namespace("rest", properties)
|
||||
first = db.list_tables(namespace_path=namespace_path)
|
||||
second = db.list_tables(namespace_path=namespace_path, page_token=first.page_token)
|
||||
assert len(first.tables) == 10
|
||||
assert first.page_token == "opaque-next"
|
||||
assert sorted(first.tables + second.tables) == names
|
||||
assert second.page_token is None
|
||||
|
||||
assert list(db.table_names(namespace_path=namespace_path)) == names
|
||||
assert list(db.table_names(limit=None, namespace_path=namespace_path)) == names
|
||||
assert list(db.table_names(limit=11, namespace_path=namespace_path)) == names[:11]
|
||||
assert (
|
||||
list(db.table_names(page_token="t005", limit=2, namespace_path=namespace_path))
|
||||
== names[6:8]
|
||||
)
|
||||
|
||||
adb = lancedb.connect_namespace_async("rest", properties)
|
||||
assert list(await adb.table_names(namespace_path=namespace_path)) == names
|
||||
assert (
|
||||
list(await adb.table_names(limit=None, namespace_path=namespace_path)) == names
|
||||
)
|
||||
assert (
|
||||
list(await adb.table_names(limit=11, namespace_path=namespace_path))
|
||||
== names[:11]
|
||||
)
|
||||
assert (
|
||||
list(
|
||||
await adb.table_names(
|
||||
page_token="t005", limit=2, namespace_path=namespace_path
|
||||
)
|
||||
)
|
||||
== names[6:8]
|
||||
)
|
||||
|
||||
request_count = len(requests)
|
||||
assert list(db.table_names(limit=0, namespace_path=namespace_path)) == []
|
||||
assert list(await adb.table_names(limit=0, namespace_path=namespace_path)) == []
|
||||
assert len(requests) == request_count
|
||||
identifier = "$" if not namespace_path else "n"
|
||||
assert all(path == f"/v1/namespace/{identifier}/table/list" for path, _ in requests)
|
||||
|
||||
|
||||
class TestNamespaceConnection:
|
||||
"""Test namespace-based LanceDB connection using DirectoryNamespace."""
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ import threading
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch
|
||||
import uuid
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
from packaging.version import Version
|
||||
|
||||
import lancedb
|
||||
@@ -105,6 +106,67 @@ async def mock_lancedb_connection_async(handler, **client_config):
|
||||
handle.join()
|
||||
|
||||
|
||||
def paginated_table_listing_handler(request):
|
||||
url = urlparse(request.path)
|
||||
assert url.path in {
|
||||
"/v1/table/",
|
||||
"/v1/namespace/$/table/list",
|
||||
"/v1/namespace/n/table/list",
|
||||
}
|
||||
query = parse_qs(url.query)
|
||||
limit = int(query.get("limit", [10])[0])
|
||||
assert limit > 0
|
||||
token = query.get("page_token", ["page-0"])[0]
|
||||
start = int(token.removeprefix("page-"))
|
||||
end = min(start + limit, 130)
|
||||
body = {"tables": [f"t{i:03}" for i in range(start, end)]}
|
||||
if end < 130:
|
||||
body["page_token"] = f"page-{end}"
|
||||
request.send_response(200)
|
||||
request.send_header("Content-Type", "application/json")
|
||||
request.end_headers()
|
||||
request.wfile.write(json.dumps(body).encode())
|
||||
|
||||
|
||||
@pytest.mark.parametrize("namespace", [[], ["n"]])
|
||||
def test_remote_table_listing_defaults_and_zero_limit(namespace):
|
||||
names = [f"t{i:03}" for i in range(130)]
|
||||
with mock_lancedb_connection(paginated_table_listing_handler) as db:
|
||||
assert db.table_names(namespace_path=namespace) == names
|
||||
assert db.table_names(limit=None, namespace_path=namespace) == names
|
||||
assert db.table_names(limit=0, namespace_path=namespace) == []
|
||||
first = db.list_tables(namespace_path=namespace)
|
||||
assert first.tables == names[:100]
|
||||
assert first.page_token == "page-100"
|
||||
second = db.list_tables(namespace_path=namespace, page_token=first.page_token)
|
||||
assert second.tables == names[100:]
|
||||
assert second.page_token is None
|
||||
assert db.list_tables(namespace_path=namespace, limit=200).tables == names
|
||||
empty = db.list_tables(namespace_path=namespace, limit=0)
|
||||
assert empty.tables == []
|
||||
assert empty.page_token is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("namespace", [[], ["n"]])
|
||||
async def test_async_remote_table_listing_defaults_and_zero_limit(namespace):
|
||||
names = [f"t{i:03}" for i in range(130)]
|
||||
async with mock_lancedb_connection_async(paginated_table_listing_handler) as db:
|
||||
assert await db.table_names(namespace_path=namespace) == names
|
||||
assert await db.table_names(limit=0, namespace_path=namespace) == []
|
||||
first = await db.list_tables(namespace_path=namespace)
|
||||
assert first.tables == names[:100]
|
||||
assert first.page_token == "page-100"
|
||||
second = await db.list_tables(
|
||||
namespace_path=namespace, page_token=first.page_token
|
||||
)
|
||||
assert second.tables == names[100:]
|
||||
assert second.page_token is None
|
||||
empty = await db.list_tables(namespace_path=namespace, limit=0)
|
||||
assert empty.tables == []
|
||||
assert empty.page_token is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_remote_db():
|
||||
def handler(request):
|
||||
|
||||
Reference in New Issue
Block a user