rust/lancedb
1
0
Fork 0
mirror of https://github.com/lancedb/lancedb.git synced 2026-08-25 15:38:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into gatekeeper/fix-2325-1

Browse Source
This commit is contained in:
Gatefixer
2026-08-20 11:09:16 +00:00
parent 82f5355b71 27cea03b7d
commit d33b05328c
58 changed files with 3972 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/Cargo.toml
+1 -1
View File
@@ -1,6 +1,6 @@
[package]
name = "lancedb-python"
version = "0.38.0-beta.0"
version = "0.38.0-beta.2"
publish = false
edition.workspace = true
description = "Python bindings for LanceDB"
python/python/lancedb/__init__.py
+2
View File
@@ -12,6 +12,7 @@ __version__ = importlib.metadata.version("lancedb")
from ._lancedb import connect as lancedb_connect
from ._lancedb import FtsToken
from ._lancedb import LsmWriteSpec
from ._lancedb import tokenize as _tokenize
from .common import URI, sanitize_uri
from urllib.parse import urlparse
@@ -519,6 +520,7 @@ __all__ = [
"Job",
"LanceDBConnection",
"LanceNamespaceDBConnection",
"LsmWriteSpec",
"RemoteDBConnection",
"Session",
"Table",
python/python/lancedb/_lancedb.pyi
+1
View File
@@ -342,6 +342,7 @@ class Table:
self, columns: list[tuple[str, str]]
) -> AddColumnsResult: ...
async def refresh_column(self, column: str) -> RefreshColumnResult: ...
async def refresh_column_async(self, column: str) -> Job: ...
async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ...
async def alter_columns(
self, columns: list[dict[str, Any]]
python/python/lancedb/query.py
+9 -3
View File
@@ -2235,6 +2235,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
reranker=self._reranker,
limit=self._limit,
with_row_ids=True,
offset=self._offset,
)
return self._finish_hybrid_results(results)
@@ -2256,6 +2257,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
reranker,
limit: int,
with_row_ids: bool,
offset: Optional[int] = None,
) -> pa.Table:
if norm == "rank":
vector_results = LanceHybridQueryBuilder._rank(vector_results, "_distance")
@@ -2332,7 +2334,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
score_i = results.column_names.index("_score")
results = results.set_column(score_i, "_score", original_scores)
results = results.slice(length=limit)
results = results.slice(offset=offset or 0, length=limit)
if not with_row_ids:
results = results.drop(["_rowid"])
@@ -2679,8 +2681,12 @@ class LanceHybridQueryBuilder(LanceQueryBuilder):
# Apply common configurations
if self._limit:
self._vector_query.limit(self._limit)
self._fts_query.limit(self._limit)
# The final offset/limit window is sliced out of the combined,
# reranked results, so each sub-query must fetch enough rows to
# cover the skipped prefix as well as the window itself.
sub_query_limit = self._limit + (self._offset or 0)
self._vector_query.limit(sub_query_limit)
self._fts_query.limit(sub_query_limit)
if self._columns:
self._vector_query.select(self._columns)
self._fts_query.select(self._columns)
python/python/lancedb/remote/table.py
+29 -8
View File
@@ -974,14 +974,13 @@ class RemoteTable(Table):
*,
computed: Dict[str, str] | None = None,
) -> AddColumnsResult:
if computed:
raise NotImplementedError(
"computed columns are supported only on local tables"
)
return LOOP.run(self._table.add_columns(transforms))
return LOOP.run(self._table.add_columns(transforms, computed=computed))
def refresh_column(self, column: str):
raise NotImplementedError("computed columns are supported only on local tables")
return LOOP.run(self._table.refresh_column(column))
def refresh_column_async(self, column: str) -> Job:
return Job(LOOP.run(self._table.refresh_column_async(column)))
def alter_columns(
self, *alterations: Iterable[Dict[str, str]]
@@ -1001,17 +1000,39 @@ class RemoteTable(Table):
return LOOP.run(self._table.set_unenforced_primary_key(columns))
def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None:
"""Not supported on LanceDB Cloud."""
"""Install an LsmWriteSpec."""
return LOOP.run(self._table.set_lsm_write_spec(spec))
def unset_lsm_write_spec(self) -> None:
"""Not supported on LanceDB Cloud."""
"""Remove the LsmWriteSpec."""
return LOOP.run(self._table.unset_lsm_write_spec())
def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]:
"""Read the installed LsmWriteSpec, or ``None``."""
return LOOP.run(self._table.get_lsm_write_spec())
def checkpoint_lsm(self) -> None:
"""Synchronous version of
[`AsyncTable.checkpoint_lsm`][lancedb.AsyncTable.checkpoint_lsm]."""
return LOOP.run(self._table.checkpoint_lsm())
def flush_lsm(self) -> None:
"""Synchronous version of
[`AsyncTable.flush_lsm`][lancedb.AsyncTable.flush_lsm]."""
return LOOP.run(self._table.flush_lsm())
def compact_lsm(self) -> None:
"""Synchronous version of
[`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm]."""
return LOOP.run(self._table.compact_lsm())
def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]:
"""Synchronous version of
[`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats]."""
return LOOP.run(
self._table.get_lsm_stats(include_generation_rows=include_generation_rows)
)
def close_lsm_writers(self) -> None:
"""No-op on LanceDB Cloud (no local shard writers)."""
return LOOP.run(self._table.close_lsm_writers())
python/python/lancedb/table.py
+74 -11
View File
@@ -2029,8 +2029,10 @@ class Table(ABC):
dropping the column and declaring it again. While a declaration
reads a column, that column cannot be renamed, retyped or dropped.
Local tables only; LanceDB Cloud and Enterprise raise
``NotImplementedError``. Cannot be combined with ``transforms``.
On LanceDB Cloud and Enterprise the expression is planned by the
server, and the refresh runs as a server job -- see
[`refresh_column_async`][lancedb.table.Table.refresh_column_async].
Cannot be combined with ``transforms``.
Returns
-------
@@ -2062,8 +2064,8 @@ class Table(ABC):
by the next one; rows already filled are left as they are, so the call
is idempotent and does not observe a mutated input.
Local tables only; LanceDB Cloud and Enterprise raise
``NotImplementedError``.
Local tables only: a remote refresh runs as a server job, through
[`refresh_column_async`][lancedb.table.Table.refresh_column_async].
Parameters
----------
@@ -2077,6 +2079,31 @@ class Table(ABC):
version: the new version number of the table.
"""
@abstractmethod
def refresh_column_async(self, column: str) -> Job:
"""
Like :meth:`refresh_column`, but returns a handle to the refresh job
instead of blocking until it completes.
The job may already be complete when returned; callers must not assume
the column is filled until :meth:`Job.wait` returns. Invalid input --
an unknown column, or one that is not computed -- raises here rather
than failing the job. On local tables the job runs in-process; on
LanceDB Cloud and Enterprise it is the server's backfill job.
Examples
--------
>>> import lancedb
>>> db = lancedb.connect("./.lancedb")
>>> table = db.create_table("computed_job_demo", [{"x": 1}, {"x": 2}])
>>> table.add_columns(computed={"doubled": "x * 2"})
AddColumnsResult(version=2)
>>> job = table.refresh_column_async("doubled")
>>> job.wait()
>>> job.status()
'finished'
"""
@abstractmethod
def alter_columns(self, *alterations: Iterable[Dict[str, str]]):
"""
@@ -4101,6 +4128,13 @@ class LanceTable(Table):
[`AsyncTable.refresh_column`][lancedb.AsyncTable.refresh_column]."""
return LOOP.run(self._table.refresh_column(column))
def refresh_column_async(self, column: str) -> Job:
"""Fill a computed column's unfilled rows, returning a handle to the
refresh job. See
[`Table.refresh_column_async`][lancedb.table.Table.refresh_column_async].
"""
return Job(LOOP.run(self._table.refresh_column_async(column)))
def alter_columns(
self, *alterations: Iterable[Dict[str, str]]
) -> AlterColumnsResult:
@@ -4848,7 +4882,7 @@ class AsyncTable:
Examples
--------
>>> from lancedb._lancedb import LsmWriteSpec
>>> from lancedb import LsmWriteSpec
>>> # table.set_unenforced_primary_key("id")
>>> # table.set_lsm_write_spec(LsmWriteSpec.bucket("id", 16))
"""
@@ -4893,7 +4927,7 @@ class AsyncTable:
``asyncio.wait_for`` for a wall-clock bound; abandoning it partway
costs nothing.
"""
return await self._inner.checkpoint_lsm()
await self._inner.checkpoint_lsm()
async def flush_lsm(self) -> None:
"""Seal every bucket's active memtable into L0.
@@ -4902,7 +4936,7 @@ class AsyncTable:
`compact_lsm`. On a node that has not claimed this table, this claims
it and replays its WAL log first.
"""
return await self._inner.flush_lsm()
await self._inner.flush_lsm()
async def compact_lsm(self) -> None:
"""Trigger a background L0 to base compaction pass per bucket.
@@ -4911,7 +4945,7 @@ class AsyncTable:
``get_lsm_stats`` for progress, or use ``checkpoint_lsm`` to loop
until the current L0 has reached base.
"""
return await self._inner.compact_lsm()
await self._inner.compact_lsm()
async def get_lsm_stats(
self, *, include_generation_rows: bool = False
@@ -6048,7 +6082,8 @@ class AsyncTable:
declaration reads a column, that column cannot be renamed, retyped
or dropped.
Local tables only. Cannot be combined with ``transforms``.
On LanceDB Cloud and Enterprise the expression is planned by
the server. Cannot be combined with ``transforms``.
Returns
-------
@@ -6084,8 +6119,8 @@ class AsyncTable:
by the next one; rows already filled are left as they are, so the call
is idempotent and does not observe a mutated input.
Local tables only; LanceDB Cloud and Enterprise raise
``NotImplementedError``.
Local tables only: a remote refresh runs as a server job, through
[`refresh_column_async`][lancedb.table.Table.refresh_column_async].
Parameters
----------
@@ -6099,6 +6134,34 @@ class AsyncTable:
"""
return await self._inner.refresh_column(column)
async def refresh_column_async(self, column: str) -> AsyncJob:
"""
Like :meth:`refresh_column`, but returns a handle to the refresh job
instead of blocking until it completes.
The job may already be complete when returned; callers must not assume
the column is filled until :meth:`AsyncJob.wait` resolves. Invalid
input -- an unknown column, or one that is not computed -- raises here
rather than failing the job. On local tables the job runs
in-process; on LanceDB Cloud and Enterprise it is the server's
backfill job.
Examples
--------
>>> import asyncio
>>> import lancedb
>>> async def refresh_in_background():
... db = await lancedb.connect_async("./.lancedb")
... table = await db.create_table("computed_job_async_demo", [{"x": 1}])
... await table.add_columns(computed={"doubled": "x * 2"})
... job = await table.refresh_column_async("doubled")
... await job.wait()
... return await job.status()
>>> asyncio.run(refresh_in_background())
'finished'
"""
return AsyncJob(await self._inner.refresh_column_async(column))
async def alter_columns(
self, *alterations: Iterable[dict[str, Any]]
) -> AlterColumnsResult:
python/python/tests/test_expr.py
+98
View File
@@ -632,3 +632,101 @@ class TestExprBytesIntegration:
.to_arrow()
)
assert result.num_rows == 2
# ── datetime / timezone integration for lit() (issue #3262) ──────────────────
class TestExprDatetimeTimezoneIntegration:
"""Integration coverage for lit(datetime) against table timestamp columns.
PyArrow stores naive timestamps as UTC wall-clock microseconds. Python's
datetime.timestamp() treats naive values as *local* time, which used to
shift lit(naive) by the host UTC offset and break equality filters on
non-UTC machines. These cases lock the expected semantics.
"""
def test_both_naive_match(self, tmp_path):
"""Table naive + lit naive with the same wall clock must match."""
db = lancedb.connect(str(tmp_path / "naive"))
ts = datetime(2024, 7, 1, 10, 0, 0)
table = db.create_table(
"t", [{"id": 1, "ts": ts}, {"id": 2, "ts": datetime(2024, 7, 2, 10, 0, 0)}]
)
result = table.search().where(col("ts") == lit(ts)).to_list()
assert len(result) == 1
assert result[0]["id"] == 1
def test_both_same_timezone_match(self, tmp_path):
"""Table UTC + lit UTC for the same instant must match."""
db = lancedb.connect(str(tmp_path / "utc"))
ts = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc)
table = db.create_table(
"t",
pa.table(
{
"id": [1, 2],
"ts": pa.array(
[ts, datetime(2024, 7, 2, 10, 0, 0, tzinfo=timezone.utc)],
type=pa.timestamp("us", tz="UTC"),
),
}
),
)
result = table.search().where(col("ts") == lit(ts)).to_list()
assert len(result) == 1
assert result[0]["id"] == 1
def test_different_timezones_same_instant(self, tmp_path):
"""UTC table row equals lit of the same instant in a different zone."""
db = lancedb.connect(str(tmp_path / "diff_tz"))
ts_utc = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc)
# Same instant as 06:00 in UTC-4
ts_est = datetime(2024, 7, 1, 6, 0, 0, tzinfo=timezone(timedelta(hours=-4)))
table = db.create_table(
"t",
pa.table(
{
"id": [1],
"ts": pa.array([ts_utc], type=pa.timestamp("us", tz="UTC")),
}
),
)
result = table.search().where(col("ts") == lit(ts_est)).to_list()
assert len(result) == 1
assert result[0]["id"] == 1
def test_table_tz_literal_naive(self, tmp_path):
"""UTC table + naive lit uses wall-clock equality (10:00 == 10:00 UTC)."""
db = lancedb.connect(str(tmp_path / "tz_naive"))
ts_utc = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc)
ts_naive = datetime(2024, 7, 1, 10, 0, 0)
table = db.create_table(
"t",
pa.table(
{
"id": [1],
"ts": pa.array([ts_utc], type=pa.timestamp("us", tz="UTC")),
}
),
)
result = table.search().where(col("ts") == lit(ts_naive)).to_list()
assert len(result) == 1
assert result[0]["id"] == 1
def test_table_naive_literal_aware(self, tmp_path):
"""Naive table + UTC lit with the same wall clock must match."""
db = lancedb.connect(str(tmp_path / "naive_aware"))
ts_naive = datetime(2024, 7, 1, 10, 0, 0)
ts_utc = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc)
table = db.create_table("t", [{"id": 1, "ts": ts_naive}])
result = table.search().where(col("ts") == lit(ts_utc)).to_list()
assert len(result) == 1
assert result[0]["id"] == 1
def test_naive_lit_sql_is_wall_clock_not_local_shifted(self):
"""Regression: naive lit must not apply the host local UTC offset."""
ts = datetime(2024, 7, 1, 10, 0, 0)
sql = lit(ts).to_sql()
# Must encode 10:00 wall clock, not 10:00+local_offset.
assert "2024-07-01 10:00:00" in sql
python/python/tests/test_hybrid_query.py
+25
View File
@@ -203,6 +203,31 @@ async def test_async_hybrid_query_default_limit(table: AsyncTable):
assert texts.count("a") == 1
def test_hybrid_query_offset(sync_table: Table):
# The offset window of a hybrid query must be a suffix of the same query
# run without an offset -- it must not be silently ignored.
full = (
sync_table.search(query_type="hybrid")
.vector([0.0, 0.4])
.text("dog")
.limit(4)
.with_row_id(True)
.to_arrow()
)
assert len(full) == 4
offset_result = (
sync_table.search(query_type="hybrid")
.vector([0.0, 0.4])
.text("dog")
.offset(2)
.limit(2)
.with_row_id(True)
.to_arrow()
)
assert offset_result["_rowid"].to_pylist() == full["_rowid"].to_pylist()[2:]
def test_hybrid_query_minimum_nprobes_zero_raises(sync_table: Table):
# minimum_nprobes(0) must raise the same validation error a plain vector
# query raises, not silently no-op because 0 is falsy.
python/python/tests/test_remote_db.py
+125
View File
@@ -1133,6 +1133,131 @@ def test_stats():
assert res == stats
@contextlib.contextmanager
def lsm_test_table(lsm_handler):
"""A remote table whose LSM routes are served by ``lsm_handler``.
``lsm_handler(request, route)`` is called for ``/v1/table/test/<route>/``
where route is one of flush_lsm, compact_lsm, get_lsm_stats, and is
responsible for writing the response.
"""
routes = ("flush_lsm", "compact_lsm", "get_lsm_stats")
def handler(request):
match = re.fullmatch(r"/v1/table/test/(\w+)/", request.path)
route = match.group(1) if match else None
if route in routes:
lsm_handler(request, route)
elif route == "describe":
request.send_response(200)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(b'{"version": 1, "schema": {"fields": []}}')
else:
request.send_response(404)
request.end_headers()
with mock_lancedb_connection(handler) as db:
yield db.open_table("test")
def read_json_body(request):
content_len = int(request.headers.get("Content-Length"))
return json.loads(request.rfile.read(content_len))
def send_json(request, payload, status=200):
request.send_response(status)
request.send_header("Content-Type", "application/json")
request.end_headers()
request.wfile.write(json.dumps(payload).encode())
def test_get_lsm_stats_sync():
"""The sync wrapper round-trips the server payload into a dict."""
bucket = {
"shard_id": "b0",
"status": "Active",
"writer_epoch": 3,
"manifest_version": 12,
"current_generation": 6,
"replay_after_wal_entry_position": 40,
"wal_entry_position_last_seen": 42,
"generations": [{"generation": 5, "bytes": 1024, "rows": 7}],
"compacting": False,
"memtables": [
{
"generation": 6,
"rows": 2,
"bytes": 64,
"batches": 1,
"indexes": ["vec_idx"],
}
],
}
seen_bodies = []
def lsm_handler(request, route):
assert route == "get_lsm_stats"
seen_bodies.append(read_json_body(request))
send_json(request, {"lsm_stats": {"buckets": [bucket]}})
with lsm_test_table(lsm_handler) as table:
assert table.get_lsm_stats() == {"buckets": [bucket]}
# Off by default, and forwarded when asked for.
assert seen_bodies == [{"include_generation_rows": False}]
table.get_lsm_stats(include_generation_rows=True)
assert seen_bodies[-1] == {"include_generation_rows": True}
def test_get_lsm_stats_sync_returns_none_when_lsm_disabled():
"""A null envelope means the LSM write path is not enabled, not an error."""
def lsm_handler(request, route):
send_json(request, {"lsm_stats": None})
with lsm_test_table(lsm_handler) as table:
assert table.get_lsm_stats() is None
def test_flush_and_compact_lsm_sync():
"""Both are one-shot POSTs answered 202 with no body."""
called = []
def lsm_handler(request, route):
called.append(route)
request.send_response(202)
request.end_headers()
with lsm_test_table(lsm_handler) as table:
assert table.flush_lsm() is None
assert table.compact_lsm() is None
assert called == ["flush_lsm", "compact_lsm"]
def test_checkpoint_lsm_sync():
"""Seal, read the watermark, and return once L0 holds nothing.
The convergence loop itself is covered in Rust; this pins the sync
binding to the endpoints it drives.
"""
called = []
def lsm_handler(request, route):
called.append(route)
if route == "get_lsm_stats":
# An empty L0 yields no target watermark, so the loop is done
# after the seal without ever polling compaction.
send_json(request, {"lsm_stats": {"buckets": []}})
else:
request.send_response(202)
request.end_headers()
with lsm_test_table(lsm_handler) as table:
assert table.checkpoint_lsm() is None
assert called == ["flush_lsm", "get_lsm_stats"]
@contextlib.contextmanager
def query_test_table(query_handler, *, server_version=Version("0.1.0")):
def handler(request):
python/python/tests/test_table.py
+28
View File
@@ -3953,3 +3953,31 @@ async def test_computed_column_async(tmp_path):
await table.refresh_column("tripled")
assert (await table.to_arrow())["tripled"].to_pylist() == [9]
def test_refresh_column_async_returns_job(tmp_path):
db = lancedb.connect(tmp_path)
table = db.create_table("computed_job", [{"x": 1}, {"x": 2}])
table.add_columns(computed={"doubled": "x * 2"})
job = table.refresh_column_async("doubled")
assert job.id is None # in-process jobs have no server id
job.wait()
assert job.status() == "finished"
assert sorted(table.to_arrow()["doubled"].to_pylist()) == [2, 4]
# Bad input raises at the call, not through the job.
with pytest.raises(Exception, match="not a computed column"):
table.refresh_column_async("x")
@pytest.mark.asyncio
async def test_refresh_column_async_job_async_table(tmp_path):
db = await lancedb.connect_async(tmp_path)
table = await db.create_table("computed_job_async", [{"x": 3}])
await table.add_columns(computed={"tripled": "x * 3"})
job = await table.refresh_column_async("tripled")
await job.wait()
assert await job.status() == "finished"
assert (await table.to_arrow())["tripled"].to_pylist() == [9]
python/src/expr.rs
+20 -1
View File
@@ -191,8 +191,27 @@ pub fn expr_lit(value: Bound<'_, PyAny>) -> PyResult<PyExpr> {
}
// datetime.datetime is a subclass of datetime.date, so it must be checked first.
//
// Python's datetime.timestamp() treats *naive* datetimes as local wall time.
// PyArrow (and therefore Lance table storage) encodes naive timestamps as
// UTC wall-clock microseconds. Using .timestamp() for naive values therefore
// shifts the literal by the local UTC offset on non-UTC machines, so
// `col("ts") == lit(naive_dt)` fails against a table that holds the same
// naive value. Fix: treat naive datetimes as UTC wall clock (match Arrow);
// keep aware datetimes on the real .timestamp() path (correct epoch).
if let Ok(dt) = value.cast::<PyDateTime>() {
let ts: f64 = dt.call_method0("timestamp")?.extract()?;
let ts: f64 = if dt.getattr("tzinfo")?.is_none() {
// Force UTC interpretation of the naive wall clock.
let utc = pyo3::types::PyModule::import(value.py(), "datetime")?
.getattr("timezone")?
.getattr("utc")?;
let kwargs = pyo3::types::PyDict::new(value.py());
kwargs.set_item("tzinfo", utc)?;
let aware = dt.call_method("replace", (), Some(&kwargs))?;
aware.call_method0("timestamp")?.extract()?
} else {
dt.call_method0("timestamp")?.extract()?
};
let micros = (ts * 1_000_000.0).round() as i64;
return Ok(PyExpr(df_lit(ScalarValue::TimestampMicrosecond(
Some(micros),
python/src/table.rs
+11
View File
@@ -1668,6 +1668,17 @@ impl Table {
})
}
pub fn refresh_column_async(
self_: PyRef<'_, Self>,
column: String,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let job = inner.refresh_column_async(column).await.infer_error()?;
Ok(crate::job::Job::new(job))
})
}
pub fn add_columns_with_schema(
self_: PyRef<'_, Self>,
schema: PyArrowType<Schema>,