From 040a4120c876dc105df18afc34c075a36fc64cb6 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Mon, 17 Aug 2026 16:56:20 +0000 Subject: [PATCH 1/5] =?UTF-8?q?Bump=20version:=200.38.0-beta.0=20=E2=86=92?= =?UTF-8?q?=200.38.0-beta.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package-lock.json | 4 ++-- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 17 files changed, 20 insertions(+), 20 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index cab6bb104..e2e693549 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.38.0-beta.0" +current_version = "0.38.0-beta.1" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index f0a213459..2f7499e91 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5398,7 +5398,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.38.0-beta.0" +version = "0.38.0-beta.1" dependencies = [ "ahash", "anyhow", @@ -5486,7 +5486,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.38.0-beta.0" +version = "0.38.0-beta.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -5511,7 +5511,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.38.0-beta.0" +version = "0.38.0-beta.1" dependencies = [ "arrow", "async-trait", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index f9a0ea053..42bc06b9d 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.38.0-beta.0 + 0.38.0-beta.1 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 09b088e46..94c72b326 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.0 + 0.38.0-beta.1 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 3d0682c46..90ad2f7f8 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.38.0-beta.0 + 0.38.0-beta.1 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 2e9373b9b..3496e2839 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.38.0-beta.0" +version = "0.38.0-beta.1" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index e0fd7426d..ad1503090 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.38.0-beta.0", + "version": "0.38.0-beta.1", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index ef281de3d..e7455832d 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.38.0-beta.0", + "version": "0.38.0-beta.1", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index d535820fa..8269bc4ce 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.38.0-beta.0", + "version": "0.38.0-beta.1", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 7aa21301e..7a7d0a097 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.38.0-beta.0", + "version": "0.38.0-beta.1", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index d220991f7..ce95c0174 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.38.0-beta.0", + "version": "0.38.0-beta.1", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 519d7376a..2ae43e763 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.38.0-beta.0", + "version": "0.38.0-beta.1", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 9f608d6d0..1030609fa 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.38.0-beta.0", + "version": "0.38.0-beta.1", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package-lock.json b/nodejs/package-lock.json index 9222bf582..26091b118 100644 --- a/nodejs/package-lock.json +++ b/nodejs/package-lock.json @@ -1,12 +1,12 @@ { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.0", + "version": "0.38.0-beta.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@lancedb/lancedb", - "version": "0.38.0-beta.0", + "version": "0.38.0-beta.1", "cpu": [ "x64", "arm64" diff --git a/nodejs/package.json b/nodejs/package.json index c87af926b..cfdc851ef 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.38.0-beta.0", + "version": "0.38.0-beta.1", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index bede2bc37..745ca4ea2 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.38.0-beta.0" +version = "0.38.0-beta.1" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 23c0dcfd0..92f020956 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.38.0-beta.0" +version = "0.38.0-beta.1" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From a075aa62f8cdd87ef666eea2f3d7555507e8d773 Mon Sep 17 00:00:00 2001 From: Igor Ganapolsky Date: Mon, 17 Aug 2026 10:48:02 -0700 Subject: [PATCH 2/5] fix(python): treat naive lit(datetime) as UTC wall clock (#3262) (#3775) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Fixes naive `lit(datetime)` equality filters against table timestamp columns on non-UTC hosts, and adds the integration matrix from #3262. ## Failure (before) On a machine in US Eastern (UTC−4 / EDT), with PyPI `lancedb==0.36.0`: ```python from datetime import datetime import lancedb from lancedb.expr import col, lit db = lancedb.connect("memory://") ts = datetime(2024, 7, 1, 10, 0, 0) # naive table = db.create_table("t", [{"id": 1, "ts": ts}]) rows = table.search().where(col("ts") == lit(ts)).to_list() # actual: [] (0 rows) # expected: 1 row ``` ### Root cause In `python/src/expr.rs`, `expr_lit` converted every `datetime` via Python's `.timestamp()`: - **naive** `.timestamp()` = local wall → UTC epoch (shifted by host offset) - **PyArrow naive** storage = UTC wall-clock microseconds (no local shift) So `lit(naive)` became `CAST('2024-07-01 14:00:00' AS TIMESTAMP)` on EDT while the table held `10:00:00`. ## After Naive datetimes are interpreted as UTC wall clock (`replace(tzinfo=timezone.utc).timestamp()`), matching Arrow storage. Aware datetimes still use `.timestamp()` (correct epoch). Same repro on this branch: **1 matching row**. ## Tests Added `TestExprDatetimeTimezoneIntegration` covering: | Case | Result | |------|--------| | both naive | match | | both same TZ (UTC) | match | | different TZs, same instant | match | | table TZ + naive lit | match (wall clock) | | table naive + aware lit | match | | naive lit SQL is wall clock, not local-shifted | asserts `10:00:00` in SQL | ### Verification ```bash cd python maturin develop pytest python/tests/test_expr.py -v ``` **102 passed** (full `test_expr.py`, including the 6 new cases). Closes #3262 --------- Co-authored-by: Will Jones Co-authored-by: Claude Opus 5 (1M context) --- python/python/tests/test_expr.py | 98 ++++++++++++++++++++++++++++++++ python/src/expr.rs | 21 ++++++- 2 files changed, 118 insertions(+), 1 deletion(-) diff --git a/python/python/tests/test_expr.py b/python/python/tests/test_expr.py index 6aa78943e..0eb6f8929 100644 --- a/python/python/tests/test_expr.py +++ b/python/python/tests/test_expr.py @@ -632,3 +632,101 @@ class TestExprBytesIntegration: .to_arrow() ) assert result.num_rows == 2 + + +# ── datetime / timezone integration for lit() (issue #3262) ────────────────── + + +class TestExprDatetimeTimezoneIntegration: + """Integration coverage for lit(datetime) against table timestamp columns. + + PyArrow stores naive timestamps as UTC wall-clock microseconds. Python's + datetime.timestamp() treats naive values as *local* time, which used to + shift lit(naive) by the host UTC offset and break equality filters on + non-UTC machines. These cases lock the expected semantics. + """ + + def test_both_naive_match(self, tmp_path): + """Table naive + lit naive with the same wall clock must match.""" + db = lancedb.connect(str(tmp_path / "naive")) + ts = datetime(2024, 7, 1, 10, 0, 0) + table = db.create_table( + "t", [{"id": 1, "ts": ts}, {"id": 2, "ts": datetime(2024, 7, 2, 10, 0, 0)}] + ) + result = table.search().where(col("ts") == lit(ts)).to_list() + assert len(result) == 1 + assert result[0]["id"] == 1 + + def test_both_same_timezone_match(self, tmp_path): + """Table UTC + lit UTC for the same instant must match.""" + db = lancedb.connect(str(tmp_path / "utc")) + ts = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc) + table = db.create_table( + "t", + pa.table( + { + "id": [1, 2], + "ts": pa.array( + [ts, datetime(2024, 7, 2, 10, 0, 0, tzinfo=timezone.utc)], + type=pa.timestamp("us", tz="UTC"), + ), + } + ), + ) + result = table.search().where(col("ts") == lit(ts)).to_list() + assert len(result) == 1 + assert result[0]["id"] == 1 + + def test_different_timezones_same_instant(self, tmp_path): + """UTC table row equals lit of the same instant in a different zone.""" + db = lancedb.connect(str(tmp_path / "diff_tz")) + ts_utc = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc) + # Same instant as 06:00 in UTC-4 + ts_est = datetime(2024, 7, 1, 6, 0, 0, tzinfo=timezone(timedelta(hours=-4))) + table = db.create_table( + "t", + pa.table( + { + "id": [1], + "ts": pa.array([ts_utc], type=pa.timestamp("us", tz="UTC")), + } + ), + ) + result = table.search().where(col("ts") == lit(ts_est)).to_list() + assert len(result) == 1 + assert result[0]["id"] == 1 + + def test_table_tz_literal_naive(self, tmp_path): + """UTC table + naive lit uses wall-clock equality (10:00 == 10:00 UTC).""" + db = lancedb.connect(str(tmp_path / "tz_naive")) + ts_utc = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc) + ts_naive = datetime(2024, 7, 1, 10, 0, 0) + table = db.create_table( + "t", + pa.table( + { + "id": [1], + "ts": pa.array([ts_utc], type=pa.timestamp("us", tz="UTC")), + } + ), + ) + result = table.search().where(col("ts") == lit(ts_naive)).to_list() + assert len(result) == 1 + assert result[0]["id"] == 1 + + def test_table_naive_literal_aware(self, tmp_path): + """Naive table + UTC lit with the same wall clock must match.""" + db = lancedb.connect(str(tmp_path / "naive_aware")) + ts_naive = datetime(2024, 7, 1, 10, 0, 0) + ts_utc = datetime(2024, 7, 1, 10, 0, 0, tzinfo=timezone.utc) + table = db.create_table("t", [{"id": 1, "ts": ts_naive}]) + result = table.search().where(col("ts") == lit(ts_utc)).to_list() + assert len(result) == 1 + assert result[0]["id"] == 1 + + def test_naive_lit_sql_is_wall_clock_not_local_shifted(self): + """Regression: naive lit must not apply the host local UTC offset.""" + ts = datetime(2024, 7, 1, 10, 0, 0) + sql = lit(ts).to_sql() + # Must encode 10:00 wall clock, not 10:00+local_offset. + assert "2024-07-01 10:00:00" in sql diff --git a/python/src/expr.rs b/python/src/expr.rs index 242e88b05..eae1d96ec 100644 --- a/python/src/expr.rs +++ b/python/src/expr.rs @@ -191,8 +191,27 @@ pub fn expr_lit(value: Bound<'_, PyAny>) -> PyResult { } // datetime.datetime is a subclass of datetime.date, so it must be checked first. + // + // Python's datetime.timestamp() treats *naive* datetimes as local wall time. + // PyArrow (and therefore Lance table storage) encodes naive timestamps as + // UTC wall-clock microseconds. Using .timestamp() for naive values therefore + // shifts the literal by the local UTC offset on non-UTC machines, so + // `col("ts") == lit(naive_dt)` fails against a table that holds the same + // naive value. Fix: treat naive datetimes as UTC wall clock (match Arrow); + // keep aware datetimes on the real .timestamp() path (correct epoch). if let Ok(dt) = value.cast::() { - let ts: f64 = dt.call_method0("timestamp")?.extract()?; + let ts: f64 = if dt.getattr("tzinfo")?.is_none() { + // Force UTC interpretation of the naive wall clock. + let utc = pyo3::types::PyModule::import(value.py(), "datetime")? + .getattr("timezone")? + .getattr("utc")?; + let kwargs = pyo3::types::PyDict::new(value.py()); + kwargs.set_item("tzinfo", utc)?; + let aware = dt.call_method("replace", (), Some(&kwargs))?; + aware.call_method0("timestamp")?.extract()? + } else { + dt.call_method0("timestamp")?.extract()? + }; let micros = (ts * 1_000_000.0).round() as i64; return Ok(PyExpr(df_lit(ScalarValue::TimestampMicrosecond( Some(micros), From d742b174c4d5c10086694213e17f26bbee4c2dd2 Mon Sep 17 00:00:00 2001 From: Adityaj0 <93090622+Adityaj0@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:38:38 -0700 Subject: [PATCH 3/5] fix: hybrid search silently ignores .offset() (#3769) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary `LanceHybridQueryBuilder` (sync hybrid search, `table.search(query_type="hybrid")`) silently ignored `.offset()`. `self._offset` was never forwarded to the vector/FTS sub-queries and never applied when slicing the final combined/reranked result, so `.offset(N)` behaved identically to `.offset(0)` — no error, just wrong pagination. Fixes #3765 ## Changes - `_create_query_builders()`: each sub-query now fetches `limit + offset` rows so there's enough data to slice the correct window out of after combining/reranking. - `_combine_hybrid_results()` / `to_arrow()`: the final table is sliced with `offset=self._offset` instead of always starting at 0. ## Test plan - [x] New regression test `test_hybrid_query_offset` in `python/python/tests/test_hybrid_query.py` - [x] `uv run --extra tests pytest python/tests/test_hybrid_query.py -vv` — 13 passed - [x] `uv run --extra dev ruff format` / `ruff check` — clean Co-authored-by: Claude Sonnet 5 Co-authored-by: Will Jones --- python/python/lancedb/query.py | 12 +++++++++--- python/python/tests/test_hybrid_query.py | 25 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/python/python/lancedb/query.py b/python/python/lancedb/query.py index 095a7b5ff..e2bb491ea 100644 --- a/python/python/lancedb/query.py +++ b/python/python/lancedb/query.py @@ -2235,6 +2235,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): reranker=self._reranker, limit=self._limit, with_row_ids=True, + offset=self._offset, ) return self._finish_hybrid_results(results) @@ -2256,6 +2257,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): reranker, limit: int, with_row_ids: bool, + offset: Optional[int] = None, ) -> pa.Table: if norm == "rank": vector_results = LanceHybridQueryBuilder._rank(vector_results, "_distance") @@ -2332,7 +2334,7 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): score_i = results.column_names.index("_score") results = results.set_column(score_i, "_score", original_scores) - results = results.slice(length=limit) + results = results.slice(offset=offset or 0, length=limit) if not with_row_ids: results = results.drop(["_rowid"]) @@ -2679,8 +2681,12 @@ class LanceHybridQueryBuilder(LanceQueryBuilder): # Apply common configurations if self._limit: - self._vector_query.limit(self._limit) - self._fts_query.limit(self._limit) + # The final offset/limit window is sliced out of the combined, + # reranked results, so each sub-query must fetch enough rows to + # cover the skipped prefix as well as the window itself. + sub_query_limit = self._limit + (self._offset or 0) + self._vector_query.limit(sub_query_limit) + self._fts_query.limit(sub_query_limit) if self._columns: self._vector_query.select(self._columns) self._fts_query.select(self._columns) diff --git a/python/python/tests/test_hybrid_query.py b/python/python/tests/test_hybrid_query.py index 72dcaaa49..5e9b45ecb 100644 --- a/python/python/tests/test_hybrid_query.py +++ b/python/python/tests/test_hybrid_query.py @@ -203,6 +203,31 @@ async def test_async_hybrid_query_default_limit(table: AsyncTable): assert texts.count("a") == 1 +def test_hybrid_query_offset(sync_table: Table): + # The offset window of a hybrid query must be a suffix of the same query + # run without an offset -- it must not be silently ignored. + full = ( + sync_table.search(query_type="hybrid") + .vector([0.0, 0.4]) + .text("dog") + .limit(4) + .with_row_id(True) + .to_arrow() + ) + assert len(full) == 4 + + offset_result = ( + sync_table.search(query_type="hybrid") + .vector([0.0, 0.4]) + .text("dog") + .offset(2) + .limit(2) + .with_row_id(True) + .to_arrow() + ) + assert offset_result["_rowid"].to_pylist() == full["_rowid"].to_pylist()[2:] + + def test_hybrid_query_minimum_nprobes_zero_raises(sync_table: Table): # minimum_nprobes(0) must raise the same validation error a plain vector # query raises, not silently no-op because 0 is falsy. From 76942306b796b329f67a162beffc3e04c28acd1f Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 18 Aug 2026 20:03:20 +0800 Subject: [PATCH 4/5] docs(java): add vended credentials example (#3958) ## Context Java users opening catalog-backed tables with vended credentials currently lack a documented workflow. Opening the catalog-returned URI directly drops the namespace-provided storage options and automatic credential refresh. Document the namespace-backed `Dataset.open()` path so temporary object store credentials are applied and refreshed transparently. --- docs/src/java/java.md | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 42bc06b9d..df8f4a119 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -55,6 +55,38 @@ LanceNamespace namespaceClient = LanceDbNamespaceClientBuilder.newBuilder() | `region(String)` | AWS region (default: "us-east-1") | No | | `config(String, String)` | Additional configuration parameters | No | +### Opening a Table with Vended Credentials + +When the catalog vends temporary object store credentials, open the table through the +namespace client. The Lance dataset builder fetches the table location and storage options +from the catalog and refreshes the credentials when they expire. + +```java +import com.lancedb.LanceDbNamespaceClientBuilder; +import org.lance.Dataset; +import org.lance.namespace.LanceNamespace; + +import java.util.Arrays; + +LanceNamespace namespaceClient = LanceDbNamespaceClientBuilder.newBuilder() + .apiKey(System.getenv("LANCEDB_API_KEY")) + .database(System.getenv("LANCEDB_DATABASE")) + // Set the endpoint for a LanceDB Enterprise deployment. + // .endpoint("https://your-enterprise-endpoint") + .build(); + +try (Dataset dataset = Dataset.open() + .namespaceClient(namespaceClient) + .tableId(Arrays.asList("my_namespace", "my_table")) + .build()) { + System.out.println("Rows: " + dataset.countRows()); +} +``` + +Do not call `describeTable()` and then open the returned location with `Dataset.open(uri)`. +Opening through `namespaceClient()` is what applies the vended storage options and enables +automatic credential refresh. No object store credentials need to be passed by the application. + ## Metadata Operations ### Creating a Namespace Path From cdebea118d43e0bcc7ef3a31a959bda3c9956acf Mon Sep 17 00:00:00 2001 From: Dan Rammer Date: Tue, 18 Aug 2026 17:25:08 -0500 Subject: [PATCH 5/5] feat(python): expose LSM checkpoint and stats on sync RemoteTable (#3961) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary The sync `RemoteTable` carried `set_lsm_write_spec`, `unset_lsm_write_spec`, `get_lsm_write_spec`, and `close_lsm_writers`, but not `checkpoint_lsm`, `flush_lsm`, `compact_lsm`, or `get_lsm_stats`. That left the four LSM control methods reachable from `AsyncTable` only. They are also the four that *only* work against a remote table — `NativeTable` does not override the `BaseTable` defaults, so on a local table they return `NotSupported` (`rust/lancedb/src/table.rs:679-701`). The net effect for sync users: | | `checkpoint_lsm` / `get_lsm_stats` | |---|---| | `LanceTable` (sync, local) | present, but always `NotSupported` | | `RemoteTable` (sync, remote) | `AttributeError` — method absent | | `AsyncTable` (remote) | works | So there was no working sync path at all, despite the Rust `RemoteTable` implementing every one of these against real endpoints. ## Changes * Add `checkpoint_lsm`, `flush_lsm`, `compact_lsm`, and `get_lsm_stats` to `lancedb.remote.table.RemoteTable`, mirroring the delegation style of their neighbours. * Correct the docstrings on `set_lsm_write_spec` / `unset_lsm_write_spec`, which read `"""Not supported on LanceDB Cloud."""` although `rust/lancedb/src/remote/table.rs:2549-2601` implements both against `/v1/table/{}/set_lsm_write_spec/` and `/unset_lsm_write_spec/`. They appear to have been copy-pasted from `set_unenforced_primary_key` directly above. No Rust or PyO3 changes — the bindings and the `AsyncTable` methods already existed. The `Table` ABC is left alone, matching how the existing `*_lsm_write_spec` methods are declared on the concrete classes only. ## Tests Four new tests in `python/python/tests/test_remote_db.py`, against the existing mock HTTP server: * `test_get_lsm_stats_sync` — the server payload round-trips into the dict, and `include_generation_rows` defaults to `False` and is forwarded when set. * `test_get_lsm_stats_sync_returns_none_when_lsm_disabled` — a `{"lsm_stats": null}` envelope yields `None` rather than an error. * `test_flush_and_compact_lsm_sync` — both are one-shot POSTs answered `202` with no body. * `test_checkpoint_lsm_sync` — pins the binding to the endpoints it drives (`flush_lsm` then `get_lsm_stats`); the convergence loop itself is already covered in Rust. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5 (1M context) --- python/python/lancedb/remote/table.py | 26 +++++- python/python/lancedb/table.py | 6 +- python/python/tests/test_remote_db.py | 125 ++++++++++++++++++++++++++ 3 files changed, 152 insertions(+), 5 deletions(-) diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index aa822b913..25363cf8f 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -990,17 +990,39 @@ class RemoteTable(Table): return LOOP.run(self._table.set_unenforced_primary_key(columns)) def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None: - """Not supported on LanceDB Cloud.""" + """Install an LsmWriteSpec.""" return LOOP.run(self._table.set_lsm_write_spec(spec)) def unset_lsm_write_spec(self) -> None: - """Not supported on LanceDB Cloud.""" + """Remove the LsmWriteSpec.""" return LOOP.run(self._table.unset_lsm_write_spec()) def get_lsm_write_spec(self) -> Optional["LsmWriteSpec"]: """Read the installed LsmWriteSpec, or ``None``.""" return LOOP.run(self._table.get_lsm_write_spec()) + def checkpoint_lsm(self) -> None: + """Synchronous version of + [`AsyncTable.checkpoint_lsm`][lancedb.AsyncTable.checkpoint_lsm].""" + return LOOP.run(self._table.checkpoint_lsm()) + + def flush_lsm(self) -> None: + """Synchronous version of + [`AsyncTable.flush_lsm`][lancedb.AsyncTable.flush_lsm].""" + return LOOP.run(self._table.flush_lsm()) + + def compact_lsm(self) -> None: + """Synchronous version of + [`AsyncTable.compact_lsm`][lancedb.AsyncTable.compact_lsm].""" + return LOOP.run(self._table.compact_lsm()) + + def get_lsm_stats(self, *, include_generation_rows: bool = False) -> Optional[dict]: + """Synchronous version of + [`AsyncTable.get_lsm_stats`][lancedb.AsyncTable.get_lsm_stats].""" + return LOOP.run( + self._table.get_lsm_stats(include_generation_rows=include_generation_rows) + ) + def close_lsm_writers(self) -> None: """No-op on LanceDB Cloud (no local shard writers).""" return LOOP.run(self._table.close_lsm_writers()) diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 4ecf6e836..f97d0331c 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -4846,7 +4846,7 @@ class AsyncTable: ``asyncio.wait_for`` for a wall-clock bound; abandoning it partway costs nothing. """ - return await self._inner.checkpoint_lsm() + await self._inner.checkpoint_lsm() async def flush_lsm(self) -> None: """Seal every bucket's active memtable into L0. @@ -4855,7 +4855,7 @@ class AsyncTable: `compact_lsm`. On a node that has not claimed this table, this claims it and replays its WAL log first. """ - return await self._inner.flush_lsm() + await self._inner.flush_lsm() async def compact_lsm(self) -> None: """Trigger a background L0 to base compaction pass per bucket. @@ -4864,7 +4864,7 @@ class AsyncTable: ``get_lsm_stats`` for progress, or use ``checkpoint_lsm`` to loop until the current L0 has reached base. """ - return await self._inner.compact_lsm() + await self._inner.compact_lsm() async def get_lsm_stats( self, *, include_generation_rows: bool = False diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index ce8d5bd6e..13ffc4415 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -1133,6 +1133,131 @@ def test_stats(): assert res == stats +@contextlib.contextmanager +def lsm_test_table(lsm_handler): + """A remote table whose LSM routes are served by ``lsm_handler``. + + ``lsm_handler(request, route)`` is called for ``/v1/table/test//`` + where route is one of flush_lsm, compact_lsm, get_lsm_stats, and is + responsible for writing the response. + """ + routes = ("flush_lsm", "compact_lsm", "get_lsm_stats") + + def handler(request): + match = re.fullmatch(r"/v1/table/test/(\w+)/", request.path) + route = match.group(1) if match else None + if route in routes: + lsm_handler(request, route) + elif route == "describe": + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(b'{"version": 1, "schema": {"fields": []}}') + else: + request.send_response(404) + request.end_headers() + + with mock_lancedb_connection(handler) as db: + yield db.open_table("test") + + +def read_json_body(request): + content_len = int(request.headers.get("Content-Length")) + return json.loads(request.rfile.read(content_len)) + + +def send_json(request, payload, status=200): + request.send_response(status) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(json.dumps(payload).encode()) + + +def test_get_lsm_stats_sync(): + """The sync wrapper round-trips the server payload into a dict.""" + bucket = { + "shard_id": "b0", + "status": "Active", + "writer_epoch": 3, + "manifest_version": 12, + "current_generation": 6, + "replay_after_wal_entry_position": 40, + "wal_entry_position_last_seen": 42, + "generations": [{"generation": 5, "bytes": 1024, "rows": 7}], + "compacting": False, + "memtables": [ + { + "generation": 6, + "rows": 2, + "bytes": 64, + "batches": 1, + "indexes": ["vec_idx"], + } + ], + } + seen_bodies = [] + + def lsm_handler(request, route): + assert route == "get_lsm_stats" + seen_bodies.append(read_json_body(request)) + send_json(request, {"lsm_stats": {"buckets": [bucket]}}) + + with lsm_test_table(lsm_handler) as table: + assert table.get_lsm_stats() == {"buckets": [bucket]} + # Off by default, and forwarded when asked for. + assert seen_bodies == [{"include_generation_rows": False}] + table.get_lsm_stats(include_generation_rows=True) + assert seen_bodies[-1] == {"include_generation_rows": True} + + +def test_get_lsm_stats_sync_returns_none_when_lsm_disabled(): + """A null envelope means the LSM write path is not enabled, not an error.""" + + def lsm_handler(request, route): + send_json(request, {"lsm_stats": None}) + + with lsm_test_table(lsm_handler) as table: + assert table.get_lsm_stats() is None + + +def test_flush_and_compact_lsm_sync(): + """Both are one-shot POSTs answered 202 with no body.""" + called = [] + + def lsm_handler(request, route): + called.append(route) + request.send_response(202) + request.end_headers() + + with lsm_test_table(lsm_handler) as table: + assert table.flush_lsm() is None + assert table.compact_lsm() is None + assert called == ["flush_lsm", "compact_lsm"] + + +def test_checkpoint_lsm_sync(): + """Seal, read the watermark, and return once L0 holds nothing. + + The convergence loop itself is covered in Rust; this pins the sync + binding to the endpoints it drives. + """ + called = [] + + def lsm_handler(request, route): + called.append(route) + if route == "get_lsm_stats": + # An empty L0 yields no target watermark, so the loop is done + # after the seal without ever polling compaction. + send_json(request, {"lsm_stats": {"buckets": []}}) + else: + request.send_response(202) + request.end_headers() + + with lsm_test_table(lsm_handler) as table: + assert table.checkpoint_lsm() is None + assert called == ["flush_lsm", "get_lsm_stats"] + + @contextlib.contextmanager def query_test_table(query_handler, *, server_version=Version("0.1.0")): def handler(request):