rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-10-29 23:06:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: use SQL for all remote materialized view APIs (#4673)

Browse Source 
Remote materialized-view APIs now use SQL for the full lifecycle. Create
and refresh already submit SQL; this change moves describe, paginated
listing, and drop onto `DESCRIBE MATERIALIZED VIEW`, `SHOW MATERIALIZED
VIEWS`, and `DROP MATERIALIZED VIEW`.

Async create, refresh, and drop return the existing Job handle with
`job_type = "query"`. The handle has one `id`: the connection-scoped SQL
query ID, also accepted by `describe_query`. Submission starts the
query, the job drains its result once, and repeated waits reuse the
completed result. Synchronous drop waits for catalog removal; physical
cleanup proceeds in the background.

Listing uses namespace-scoped keyset pagination with opaque resume
tokens. Streamed and deferred SQL responses share the same path. Adds
transport coverage and updates Python/TypeScript documentation for
query-backed drop jobs.

---------

Co-authored-by: Xuanwo <github@xuanwo.io>
This commit is contained in:
Jack Ye
2026-10-28 04:37:00 +08:00
committed by GitHub
co-authored by Xuanwo
parent f6a348ac5b
commit 45dbf5eff5
13 changed files with 420 additions and 375 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/python/lancedb/db.py
+12 -13
View File
@@ -721,9 +721,8 @@ class DBConnection(EnforceOverrides):
) -> None:
"""Drop a materialized view.
The view may become unavailable before physical cleanup finishes. Use
:meth:`drop_materialized_view_async` to retain and wait for the cleanup
job.
Remote drops wait for the SQL command to remove the catalog binding.
Physical cleanup continues in the background.
"""
raise NotImplementedError(
"materialized views are not supported on this connection type"
@@ -732,11 +731,11 @@ class DBConnection(EnforceOverrides):
def drop_materialized_view_async(
self, name: str, namespace_path: Optional[List[str]] = None
) -> Job[None]:
"""Start dropping a materialized view and return its cleanup job.
"""Start dropping a materialized view and return its job.
The job may already be complete for a local database. On LanceDB Cloud
and Enterprise, its ``id`` is the server job identifier from the
``202 Accepted`` drop response.
The job may already be complete for a local database. Remote jobs have
type ``query`` and use the SQL query ID as ``id``. Waiting confirms
catalog removal; physical cleanup continues in the background.
"""
raise NotImplementedError(
"materialized views are not supported on this connection type"
@@ -2946,9 +2945,8 @@ class AsyncConnection(object):
) -> None:
"""Drop a materialized view.
The view may become unavailable before physical cleanup finishes. Use
:meth:`drop_materialized_view_async` to retain and wait for the cleanup
job.
Remote drops wait for the SQL command to remove the catalog binding.
Physical cleanup continues in the background.
"""
if namespace_path is None:
namespace_path = []
@@ -2960,10 +2958,11 @@ class AsyncConnection(object):
*,
namespace_path: Optional[List[str]] = None,
) -> AsyncJob[None]:
"""Start dropping a materialized view and return its cleanup job.
"""Start dropping a materialized view and return its job.
Await :meth:`AsyncJob.wait` before assuming physical cleanup has
finished.
For remote connections, the job has type ``query`` and its ID identifies
the SQL query. Await :meth:`AsyncJob.wait` for catalog removal; physical
cleanup continues in the background.
"""
if namespace_path is None:
namespace_path = []
python/python/tests/test_listing.py
+4 -6
View File
@@ -66,7 +66,7 @@ def response_page(kind, names, token):
]
else:
items = names
key = "views" if kind == "materialized_views" else kind
key = kind
return {key: items, "page_token": token}
@@ -85,7 +85,7 @@ async def advance(items, asynchronous):
return await items.__anext__() if asynchronous else next(items)
KINDS = ["secrets", "views", "jobs", "functions", "materialized_views"]
KINDS = ["secrets", "views", "jobs", "functions"]
@pytest.mark.asyncio
@@ -236,9 +236,7 @@ async def test_listing_lazy_pages_and_resume(listing_server, kind, asynchronous)
)
db = await connection(endpoint, asynchronous)
method = getattr(db, "list_" + kind)
kwargs = (
{} if kind in ("jobs", "materialized_views") else {"namespace_path": ["team"]}
)
kwargs = {} if kind == "jobs" else {"namespace_path": ["team"]}
items = method(page_token="start/token", page_limit=2, **kwargs)
assert requests == []
assert items.page_token() == "start/token"
@@ -273,7 +271,7 @@ async def test_listing_lazy_pages_and_resume(listing_server, kind, asynchronous)
assert path.path == "/v1/jobs/list"
assert body == {"page_token": token, "limit": 2}
else:
namespace = "$" if kind == "materialized_views" else "team"
namespace = "team"
resource = kind.removesuffix("s")
assert path.path == f"/v1/namespace/{namespace}/{resource}/list"
expected = {"page_token": [token], "limit": ["2"]}
python/python/tests/test_materialized_views.py
+36 -100
View File
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright The LanceDB Authors
import contextlib
import http.server
import json
@@ -30,89 +31,6 @@ def make_db(tmp_path):
return db
@contextlib.contextmanager
def mock_remote_materialized_views():
requests = []
class Handler(http.server.BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def do_GET(self):
requests.append(self.path)
encoded = json.dumps({"views": ["daily_sales"]}).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(encoded)))
self.end_headers()
self.wfile.write(encoded)
with http.server.HTTPServer(("localhost", 0), Handler) as server:
thread = threading.Thread(target=server.serve_forever)
thread.start()
try:
yield f"http://localhost:{server.server_address[1]}", requests
finally:
server.shutdown()
thread.join()
@contextlib.contextmanager
def mock_remote_materialized_view_drop():
requests = []
class Handler(http.server.BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def do_POST(self):
length = int(self.headers.get("Content-Length", 0))
body = json.loads(self.rfile.read(length) or b"{}")
requests.append((self.path, body))
assert self.path.endswith("/drop")
job_id = "mv-drop-123"
encoded = json.dumps({"job_id": job_id}).encode()
self.send_response(202)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(encoded)))
self.end_headers()
self.wfile.write(encoded)
with http.server.HTTPServer(("localhost", 0), Handler) as server:
thread = threading.Thread(target=server.serve_forever)
thread.start()
try:
yield f"http://localhost:{server.server_address[1]}", requests
finally:
server.shutdown()
thread.join()
def test_remote_list_uses_namespace_route():
with mock_remote_materialized_views() as (host, requests):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
assert list(db.list_materialized_views()) == ["daily_sales"]
assert requests == ["/v1/namespace/$/materialized_view/list"]
def test_remote_drop_async_returns_server_job():
with mock_remote_materialized_view_drop() as (host, requests):
db = lancedb.connect(
"db://dev",
api_key="fake",
host_override=host,
client_config={"retry_config": {"retries": 0}},
)
job = db.drop_materialized_view_async("adults")
assert job.id == "mv-drop-123"
assert requests == [("/v1/materialized_view/adults/drop", {})]
@contextlib.contextmanager
def mock_remote_materialized_view_refresh():
requests = []
@@ -123,11 +41,8 @@ def mock_remote_materialized_view_refresh():
def do_POST(self):
requests.append(self.path)
if self.path == "/v1/materialized_view/view/describe":
response = {"query": "SELECT * FROM source"}
else:
assert self.path == "/v1/table/view/describe/"
response = {"version": 42, "schema": {"fields": []}}
assert self.path == "/v1/table/view/describe/"
response = {"version": 42, "schema": {"fields": []}}
encoded = json.dumps(response).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
@@ -159,6 +74,16 @@ async def test_remote_refresh_reports_unknown_metrics(sync, background, full):
def do_get(self, context, ticket):
self.commands.append(ticket.ticket)
if b"DESCRIBE MATERIALIZED VIEW" in ticket.ticket:
return flight.RecordBatchStream(
pa.table(
{
"name": ["view"],
"query": ["SELECT * FROM source"],
}
)
)
assert b"REFRESH MATERIALIZED VIEW" in ticket.ticket
# SQL completion carries no refresh statistics.
return flight.RecordBatchStream(pa.table({"status": ["ok"]}))
@@ -199,9 +124,24 @@ async def test_remote_refresh_reports_unknown_metrics(sync, background, full):
"RefreshMaterializedViewResult(mode=None, rows_written=None, "
"source_version=None, version=42)"
)
(command,) = server.commands
assert b'REFRESH MATERIALIZED VIEW "view"' in command
assert (b"WITH (full = true)" in command) is full
describes = [
command
for command in server.commands
if b"DESCRIBE MATERIALIZED VIEW" in command
]
refreshes = [
command
for command in server.commands
if b"REFRESH MATERIALIZED VIEW" in command
]
assert len(describes) == 2
assert len(refreshes) == 1
assert all(
b'DESCRIBE MATERIALIZED VIEW "view"' in command for command in describes
)
(refresh,) = refreshes
assert b'REFRESH MATERIALIZED VIEW "view"' in refresh
assert (b"WITH (full = true)" in refresh) is full
assert not any("count_rows" in path for path in requests)
@@ -639,8 +579,6 @@ async def test_async_namespace_connection_materialized_views(tmp_path):
def test_stored_queries_and_legacy_layouts_are_read():
import json
import pyarrow as pa
from lancedb.materialized_view import _definition_from_schema
@@ -795,13 +733,11 @@ def test_embedded_create_refuses_automatic_refresh_before_creating_view(
@pytest.mark.parametrize("seconds", [0, -1, float("nan"), float("inf")])
def test_remote_create_rejects_invalid_interval_without_request(seconds):
with mock_remote_materialized_view_drop() as (host, requests):
db = lancedb.connect("db://dev", api_key="fake", host_override=host)
with pytest.raises(ValueError, match="positive, finite"):
db.create_materialized_view_async(
"adults", "people", auto_refresh_interval_seconds=seconds
)
assert requests == []
db = lancedb.connect("db://dev", api_key="fake", host_override="http://127.0.0.1:1")
with pytest.raises(ValueError, match="positive, finite"):
db.create_materialized_view_async(
"mv", "source", auto_refresh_interval_seconds=seconds
)
@pytest.mark.experimental_functions