From 6bb64c3edb50f372088d8ed34af9509daf807d0d Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 20:50:54 +0800 Subject: [PATCH 01/43] fix: reject repeated job pagination tokens (#4139) Fixes #4138 `RemoteDatabase::list_jobs` followed every returned pagination token without remembering previously seen values. A server-side token cycle therefore caused repeated requests and duplicate accumulation until the 100-page safeguard returned partial results as a success. This change tracks non-empty job-list page tokens and returns an HTTP-context error as soon as a token repeats, matching the existing `list_functions` behavior. A mock-handler regression test verifies a repeated `loop` token is rejected after two requests. Validation: - `cargo test --quiet --features remote -p lancedb test_list_jobs` - `cargo fmt --all` - `cargo check --quiet --features remote --tests --examples` Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo --- rust/lancedb/src/remote/db.rs | 46 ++++++++++++++++++++++++++++++++--- 1 file changed, 43 insertions(+), 3 deletions(-) diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 917bf909b..1d6f9abe8 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -700,6 +700,7 @@ impl Database for RemoteDatabase { async fn list_jobs(&self) -> Result> { let mut out = Vec::new(); let mut page_token: Option = None; + let mut seen_page_tokens = HashSet::new(); for page in 0..MAX_LIST_JOBS_PAGES { let mut body = serde_json::json!({}); if let Some(token) = &page_token { @@ -708,7 +709,8 @@ impl Database for RemoteDatabase { let req = self.client.post("/v1/jobs/list").json(&body); let (request_id, rsp) = self.client.send(req).await?; let rsp = self.client.check_response(&request_id, rsp).await?; - let body: RemoteListJobsResponse = rsp.json().await.err_to_http(request_id)?; + let status = rsp.status(); + let body: RemoteListJobsResponse = rsp.json().await.err_to_http(request_id.clone())?; out.extend(body.jobs.into_iter().map(|row| JobInfo { job_id: row.job_id, table: row.table, @@ -716,10 +718,17 @@ impl Database for RemoteDatabase { state: job_state_to_client(&row.state), created_at_millis: row.created_at_millis, })); - page_token = body.page_token; - if page_token.is_none() { + let Some(next_page_token) = body.page_token.filter(|token| !token.is_empty()) else { break; + }; + if !seen_page_tokens.insert(next_page_token.clone()) { + return Err(Error::Http { + source: "Job listing response repeated a page_token".into(), + request_id, + status_code: Some(status), + }); } + page_token = Some(next_page_token); if page + 1 == MAX_LIST_JOBS_PAGES { log::warn!( "list_jobs truncated after {} pages ({} jobs)", @@ -2634,6 +2643,37 @@ mod tests { assert_eq!(jobs[2].state, "failed"); } + #[tokio::test] + async fn test_list_jobs_rejects_a_page_token_cycle() { + let requests = Arc::new(AtomicUsize::new(0)); + let seen = requests.clone(); + let conn = Connection::new_with_handler(move |request| { + let body: serde_json::Value = + serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); + match seen.fetch_add(1, Ordering::SeqCst) { + 0 => assert!(body.get("page_token").is_none()), + _ => assert_eq!(body["page_token"], "loop"), + } + http::Response::builder() + .status(200) + .body(r#"{"jobs": [], "page_token": "loop"}"#) + .unwrap() + }); + + let error = conn.list_jobs().await.unwrap_err(); + assert!( + matches!( + &error, + Error::Http { + status_code: Some(http::StatusCode::OK), + .. + } + ), + "got {error:?}" + ); + assert_eq!(requests.load(Ordering::SeqCst), 2); + } + #[tokio::test] async fn test_open_job() { let conn = Connection::new_with_handler(|request| { From 0113cee48976c5c4632ff251ff42a46f913ba4e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E5=BF=97=E8=B0=A6?= <89645338+simpleqt@users.noreply.github.com> Date: Mon, 14 Sep 2026 22:40:50 +0800 Subject: [PATCH 02/43] docs(embeddings): correct the documented max_retries default (#4145) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `lancedb/embeddings/utils.py` documents `max_retries` with "(default is 10)" — the signature default is `7`. Docs-only; conventional title per the contribution guide. Co-authored-by: Xuanwo --- python/python/lancedb/embeddings/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/python/lancedb/embeddings/utils.py b/python/python/lancedb/embeddings/utils.py index 189bbe53c..98806fa52 100644 --- a/python/python/lancedb/embeddings/utils.py +++ b/python/python/lancedb/embeddings/utils.py @@ -249,7 +249,7 @@ def retry_with_exponential_backoff( initial_delay (float): Initial delay in seconds (default is 1). exponential_base (float): The base for exponential backoff (default is 2). jitter (bool): Whether to add jitter to the delay (default is True). - max_retries (int): Maximum number of retries (default is 10). + max_retries (int): Maximum number of retries (default is 7). Returns: function: The decorated function. From 0665575a76454f6e23c28d5a599310e544081e08 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Mon, 14 Sep 2026 09:39:55 -0700 Subject: [PATCH 03/43] feat: recompute computed column rows whose inputs changed (#4161) refresh_column fills nulls, so once a row has a value nothing revisits it: an update to one of its inputs, or a definition change, leaves the computed value stale for good. This stamps the column's field metadata with the definition it was computed under and a per-fragment signature of the input storage it was read from (input data files and overlays; not the deletion file, since a delete changes no surviving value). A refresh recomputes every live row of a fragment whose stamp disagrees with the manifest, then records what it computed from in a second commit after the fill. A compacted fragment inherits freshness through the Rewrite lineage when every fragment it was built from was signed, or was appended since the stamp, never had an input moved, and left its rows of the product unfilled (a raw append may supply a value; the product's data is the evidence, and the null fill covers those rows); otherwise it recomputes. A column declared before the stamps existed keeps the null-fill contract on its first refresh, which enrolls it as it stood. The map is one entry per fragment per column, so it is kept out of the manifest: each stamp writes an immutable sidecar under `_computed/`, named by its content digest, and the field metadata holds the digest. Pruning old versions also drops the sidecars no remaining version references, keeping any younger than seven days as lance keeps unverified files, since a sidecar is put before the commit that references it. The stamp commit is metadata-only, so a materialized view's drift check treats it like the fill. The core lives in `table::freshness` so a remote refresh can share the contract. --- Cargo.lock | 1 + docs/src/js/classes/Table.md | 16 +- nodejs/lancedb/table.ts | 16 +- python/python/lancedb/table.py | 33 +- python/python/tests/test_table.py | 7 +- rust/lancedb/Cargo.toml | 3 +- rust/lancedb/src/materialized_view/refresh.rs | 64 +- rust/lancedb/src/table.rs | 15 +- rust/lancedb/src/table/add_columns.rs | 9 +- rust/lancedb/src/table/computed_columns.rs | 25 +- rust/lancedb/src/table/freshness.rs | 1536 +++++++++++++++++ rust/lancedb/src/table/optimize.rs | 12 +- rust/lancedb/src/table/refresh.rs | 424 ++++- 13 files changed, 2052 insertions(+), 109 deletions(-) create mode 100644 rust/lancedb/src/table/freshness.rs diff --git a/Cargo.lock b/Cargo.lock index be5040d37..efa59c4e3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5529,6 +5529,7 @@ dependencies = [ "serde_json", "serde_with", "serial_test", + "sha2 0.10.9", "snafu 0.8.9", "tempfile", "test-log", diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index ef6e9535a..dfa0a9819 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -74,10 +74,10 @@ now: the column is committed with no values, and rows get them from [Table#refreshColumn](Table.md#refreshcolumn). Declaring one therefore costs the same on a large table as on an empty one. -A refresh does not revisit rows it has already filled, so mutating an -input leaves the value computed at fill time; recomputing means dropping -the column and declaring it again. While a declaration reads a column, -that column cannot be renamed, retyped or dropped. +A refresh also recomputes the rows whose inputs changed since they were +computed, so a mutated input is reflected by the next refresh. While a +declaration reads a column, that column cannot be renamed, retyped or +dropped. On LanceDB Cloud and Enterprise the expression is planned by the server, and the refresh runs as a server job -- see @@ -916,10 +916,10 @@ abstract refreshColumn(column): Promise Fill the rows of a computed column that hold no value yet. -Rows appended since the last refresh are filled by the next one; rows -already filled are left as they are, so the call is idempotent and does -not observe a mutated input. Local tables only: a remote refresh runs -as a server job, through [Table#refreshColumnAsync](Table.md#refreshcolumnasync). +Rows appended since the last refresh are filled by the next one, and +rows whose inputs changed since they were computed are recomputed; +everything else is left as it is. Local tables only: a remote refresh +runs as a server job, through [Table#refreshColumnAsync](Table.md#refreshcolumnasync). #### Parameters diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index eac9f490d..8d8b0d675 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -572,10 +572,10 @@ export abstract class Table { * {@link Table#refreshColumn}. Declaring one therefore costs the same on a * large table as on an empty one. * - * A refresh does not revisit rows it has already filled, so mutating an - * input leaves the value computed at fill time; recomputing means dropping - * the column and declaring it again. While a declaration reads a column, - * that column cannot be renamed, retyped or dropped. + * A refresh also recomputes the rows whose inputs changed since they were + * computed, so a mutated input is reflected by the next refresh. While a + * declaration reads a column, that column cannot be renamed, retyped or + * dropped. * * On LanceDB Cloud and Enterprise the expression is planned by the * server, and the refresh runs as a server job -- see @@ -606,10 +606,10 @@ export abstract class Table { /** * Fill the rows of a computed column that hold no value yet. * - * Rows appended since the last refresh are filled by the next one; rows - * already filled are left as they are, so the call is idempotent and does - * not observe a mutated input. Local tables only: a remote refresh runs - * as a server job, through {@link Table#refreshColumnAsync}. + * Rows appended since the last refresh are filled by the next one, and + * rows whose inputs changed since they were computed are recomputed; + * everything else is left as it is. Local tables only: a remote refresh + * runs as a server job, through {@link Table#refreshColumnAsync}. * @param {string} column The name of the computed column to fill. * @returns {Promise} A promise that resolves to the * number of rows filled and the new version number of the table. diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 72362acad..84ae4e836 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -2188,10 +2188,10 @@ class Table(ABC): Declaring one therefore costs the same on a large table as on an empty one. - A refresh does not revisit rows it has already filled, so mutating - an input leaves the value computed at fill time; recomputing means - dropping the column and declaring it again. While a declaration - reads a column, that column cannot be renamed, retyped or dropped. + A refresh also recomputes the rows whose inputs changed since they + were computed, so a mutated input is reflected by the next refresh. + While a declaration reads a column, that column cannot be renamed, + retyped or dropped. On LanceDB Cloud and Enterprise the expression is planned by the server, and the refresh runs as a server job -- see @@ -2211,7 +2211,7 @@ class Table(ABC): >>> table.add_columns(computed={"doubled": "x * 2"}) AddColumnsResult(version=2) >>> table.refresh_column("doubled") - RefreshColumnResult(rows_filled=2, version=3) + RefreshColumnResult(rows_filled=2, version=4) >>> table.to_arrow().sort_by("x").to_pandas() x doubled 0 1 2 @@ -2225,8 +2225,8 @@ class Table(ABC): Declared with ``add_columns(computed=...)``, a column starts empty and gets its values here. Rows appended since the last refresh are filled - by the next one; rows already filled are left as they are, so the call - is idempotent and does not observe a mutated input. + by the next one, and rows whose inputs changed since they were computed + are recomputed; everything else is left as it is. Local tables only: a remote refresh runs as a server job, through [`refresh_column_async`][lancedb.table.Table.refresh_column_async]. @@ -4318,13 +4318,14 @@ class LanceTable(Table): return LOOP.run(self._table.add_columns(transforms, computed=computed)) def refresh_column(self, column: str) -> "RefreshColumnResult": - """Fill a computed column's unfilled rows. See + """Fill a computed column's unfilled rows and recompute those whose + inputs changed. See [`AsyncTable.refresh_column`][lancedb.AsyncTable.refresh_column].""" return LOOP.run(self._table.refresh_column(column)) def refresh_column_async(self, column: str) -> Job[RefreshColumnJobResult]: - """Fill a computed column's unfilled rows, returning a handle to the - refresh job. See + """Fill a computed column's unfilled rows and recompute those whose + inputs changed, returning a handle to the refresh job. See [`Table.refresh_column_async`][lancedb.table.Table.refresh_column_async]. """ return Job(LOOP.run(self._table.refresh_column_async(column))) @@ -6312,10 +6313,10 @@ class AsyncTable: them from [`refresh_column`][lancedb.table.AsyncTable.refresh_column]. - A refresh does not revisit rows it has already filled, so mutating - an input leaves the value computed at fill time. While a - declaration reads a column, that column cannot be renamed, retyped - or dropped. + A refresh also recomputes the rows whose inputs changed since they + were computed, so a mutated input is reflected by the next refresh. + While a declaration reads a column, that column cannot be renamed, + retyped or dropped. On LanceDB Cloud and Enterprise the expression is planned by the server. Cannot be combined with ``transforms``. @@ -6377,8 +6378,8 @@ class AsyncTable: Declared with ``add_columns(computed=...)``, a column starts empty and gets its values here. Rows appended since the last refresh are filled - by the next one; rows already filled are left as they are, so the call - is idempotent and does not observe a mutated input. + by the next one, and rows whose inputs changed since they were computed + are recomputed; everything else is left as it is. Local tables only: a remote refresh runs as a server job, through [`refresh_column_async`][lancedb.table.Table.refresh_column_async]. diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index b85486412..ac7dc660d 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -4183,13 +4183,14 @@ def test_refresh_column_async_returns_job(tmp_path): assert result.rows_failed == 0 assert result.rows_remaining == 0 assert result.source_version == 2 - assert result.published_version == 3 + # The fill lands at 3; the stamp recording its inputs is published at 4. + assert result.published_version == 4 assert job.status() == "finished" assert sorted(table.to_arrow()["doubled"].to_pylist()) == [2, 4] no_op = table.refresh_column_async("doubled").wait() assert no_op.rows_assigned == 0 - assert no_op.source_version == 3 + assert no_op.source_version == 4 assert no_op.published_version is None # Bad input raises at the call, not through the job. @@ -4208,6 +4209,6 @@ async def test_refresh_column_async_job_async_table(tmp_path): assert isinstance(result, lancedb.RefreshColumnResult) assert result.rows_assigned == 1 assert result.source_version == 2 - assert result.published_version == 3 + assert result.published_version == 4 assert await job.status() == "finished" assert (await table.to_arrow())["tripled"].to_pylist() == [9] diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 404753d7a..998d041e2 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -95,13 +95,14 @@ candle-transformers = { version = "0.9.1", optional = true } candle-nn = { version = "0.9.1", optional = true } tokenizers = { version = "0.19.1", optional = true } semver = { workspace = true } +roaring = "0.11.4" +sha2 = "0.10" [dev-dependencies] anyhow = "1" lance-testing = { workspace = true } tempfile = { workspace = true } random_word = { version = "0.4.3", features = ["en"] } -roaring = "0.11.4" tokio = { workspace = true, features = ["io-util", "macros", "net", "test-util"] } uuid = { workspace = true } walkdir = "2" diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs index 23bb51566..433b185f7 100644 --- a/rust/lancedb/src/materialized_view/refresh.rs +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -1124,8 +1124,9 @@ struct RowScope { /// Whether every commit on the view after `recorded` is a fill of its /// computed columns: a column rewrite or data replacement touching only -/// those fields and neither adding nor removing rows. A version whose -/// transaction cannot be read is not proven, so it counts as drift. +/// those fields and neither adding nor removing rows, or the freshness +/// stamp a fill leaves on them. A version whose transaction cannot be read +/// is not proven, so it counts as drift. async fn only_computed_rewrites_since(view_ds: &Dataset, recorded: u64) -> Result { // A fill may write any field under a computed column, so the whole // subtree counts, not only the root. @@ -1176,6 +1177,19 @@ async fn only_computed_rewrites_since(view_ds: &Dataset, recorded: u64) -> Resul .all(|field| computed_fields.contains(&(*field as u32))) }) } + // The stamp `refresh_column` writes after its fill (see + // `table::freshness`): field metadata on computed columns, no data. + Operation::UpdateConfig { + config_updates: None, + table_metadata_updates: None, + schema_metadata_updates: None, + field_metadata_updates, + } => { + !field_metadata_updates.is_empty() + && field_metadata_updates + .keys() + .all(|field| computed_fields.contains(&(*field as u32))) + } _ => false, }; if !fill { @@ -3359,6 +3373,46 @@ mod tests { ); } + /// Field metadata on `field` only, the commit shape of the freshness + /// stamp `refresh_column` leaves after its fill. + async fn commit_field_metadata(view: &MaterializedView, field: &str, key: &str) { + let native = view.table().as_native().unwrap(); + native.dataset.reload().await.unwrap(); + let mut dataset = native.dataset.get().await.unwrap().as_ref().clone(); + dataset + .update_field_metadata() + .update(field, [(key.to_string(), "{}".to_string())]) + .unwrap() + .await + .unwrap(); + } + + /// The stamp is metadata on the computed column and rewrites nothing + /// refresh certifies, so it is not drift; the same commit shape on a + /// projected column is, like any other write to it. + #[tokio::test] + async fn test_a_freshness_stamp_is_not_drift() { + let conn = connect("memory://").execute().await.unwrap(); + let view = refreshed_computed_view(&conn).await; + + commit_field_metadata( + &view, + "emb", + crate::table::computed_columns::SOURCE_SIGNATURE_META_KEY, + ) + .await; + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::NoOp + ); + + commit_field_metadata(&view, "id", "probe").await; + assert_eq!( + view.refresh().execute().await.unwrap().mode, + RefreshMode::Rebuild + ); + } + /// The fill job's commit rewrites only computed columns. It is the one /// commit on a view that is not drift: the next refresh carries on from /// its watermark instead of rebuilding, which would null what the fill @@ -3524,9 +3578,9 @@ mod tests { } /// A SQL declaration is filled by `refresh_column` on the view, which - /// commits a data replacement; the next refresh continues from its - /// watermark and keeps what the fill wrote, and only rows the view added - /// since come back unfilled. + /// commits a data replacement and then its freshness stamp; the next + /// refresh continues from its watermark and keeps what the fill wrote, + /// and only rows the view added since come back unfilled. #[tokio::test] async fn test_a_sql_fill_is_not_drift() { use crate::materialized_view::tests::{people, sql_field}; diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 7f139c4cb..508c2ca49 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -75,6 +75,7 @@ mod create_index; pub mod datafusion; pub(crate) mod dataset; pub mod delete; +pub mod freshness; pub mod lsm_stats; pub mod merge; pub mod optimize; @@ -778,7 +779,8 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { message: "Function columns are supported only on LanceDB Cloud and Enterprise".into(), }) } - /// Fill a computed column's unfilled rows. + /// Fill a computed column's unfilled rows and recompute those whose + /// inputs changed. /// /// The default returns `NotSupported`; Lance-backed tables override it. async fn refresh_column(&self, _column: &str) -> Result { @@ -786,8 +788,8 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { message: "computed columns are supported only on local tables".into(), }) } - /// Fill a computed column's unfilled rows, returning a [`Job`] tracking - /// the operation. + /// Fill a computed column's unfilled rows and recompute those whose + /// inputs changed, returning a [`Job`] tracking the operation. async fn refresh_column_async( &self, _column: &str, @@ -1749,9 +1751,10 @@ impl Table { /// Declared with /// [`AddColumnsBuilder::computed`](add_columns::AddColumnsBuilder::computed), /// a column starts empty and gets its values here. Fragments appended - /// since the last refresh are filled by the next one; fragments already - /// filled are left as they are, so the call is idempotent and does not - /// observe a mutated input. + /// since the last refresh are filled by the next one, and fragments whose + /// inputs changed since they were computed are recomputed (see + /// [`freshness`](crate::table::freshness)); everything else is left as + /// it is. /// /// Local tables only: a remote refresh runs as a server job, through /// [`Table::refresh_column_async`]. diff --git a/rust/lancedb/src/table/add_columns.rs b/rust/lancedb/src/table/add_columns.rs index 1ac0c6b4f..635a99bea 100644 --- a/rust/lancedb/src/table/add_columns.rs +++ b/rust/lancedb/src/table/add_columns.rs @@ -60,10 +60,11 @@ impl AddColumnsBuilder { /// every fragment that has none -- including fragments appended since the /// last refresh. /// - /// Refresh does not revisit a fragment it has filled, so mutating an input - /// leaves the value computed at fill time; recomputing means dropping the - /// column and declaring it again. An input cannot be renamed, retyped or - /// dropped while a declaration reads it, since the expression names it. + /// A refresh also recomputes the rows of a fragment whose inputs changed + /// since it was computed (see [`freshness`](super::freshness)), so a + /// mutated input is reflected by the next refresh. An input cannot be + /// renamed, retyped or dropped while a declaration reads it, since the + /// expression names it. /// /// On LanceDB Cloud and Enterprise the expression is planned by the /// server, and the refresh runs as a server job -- see diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index f1ec75213..a70ac31ae 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -71,6 +71,22 @@ pub const FUNCTION_BINDINGS_META_KEY: &str = "lancedb::function_bindings"; /// Version of the schema-level Function binding envelope. pub const FUNCTION_BINDINGS_VERSION: u32 = 1; +/// Field metadata key holding `{fragment id -> input signature}` as JSON, +/// recorded by the refresh that last computed each fragment. Outside the +/// declaration namespace on purpose: a declaration is immutable through +/// metadata edits, this is rewritten by every refresh. Seeded empty at +/// declaration, so a column is tracked from birth; a column without it was +/// declared before signatures existed. +pub const SOURCE_SIGNATURE_META_KEY: &str = "computed_refresh.source_signature"; + +/// Field metadata key holding the definition digest a column was last +/// computed under. A change to it makes every row stale. +pub const DEFINITION_VERSION_META_KEY: &str = "computed_refresh.definition_version"; + +/// Field metadata key holding the table version the signature map describes: +/// where a refresh starts following compactions to carry freshness forward. +pub const RECORDED_AT_VERSION_META_KEY: &str = "computed_refresh.recorded_at_version"; + /// Value of [`KIND_META_KEY`] for a column defined by a SQL expression. pub const SQL_KIND: &str = "sql"; @@ -139,6 +155,7 @@ fn computed_column_metadata(expression: &str, inputs: &[String]) -> HashMap Result< } /// Reject a write that supplies values for a computed column directly: -/// only refresh materializes one, and refresh never revisits a filled row. +/// only refresh materializes one, and only refresh decides what it +/// recomputes. pub(crate) fn ensure_not_written<'a>( schema: &ArrowSchema, written: impl IntoIterator, @@ -1470,7 +1489,9 @@ fn ensure_no_foreign_declaration(field: &ArrowField) -> Result<()> { /// kind, the expression, the inputs -- would bypass that validation or move /// a binding out from under a refresh. Drop the column and declare it again. pub(crate) fn is_declaration_key(key: &str) -> bool { - key == COMPUTED_COLUMN_META_KEY || key.starts_with("computed_column.") + key == COMPUTED_COLUMN_META_KEY + || key.starts_with("computed_column.") + || key.starts_with("computed_refresh.") } /// Reject retyping a computed column itself. diff --git a/rust/lancedb/src/table/freshness.rs b/rust/lancedb/src/table/freshness.rs new file mode 100644 index 000000000..e7a1841e6 --- /dev/null +++ b/rust/lancedb/src/table/freshness.rs @@ -0,0 +1,1536 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Source-change detection for computed columns. +//! +//! A refresh fills nulls, so once a value is durable nothing recomputes it and +//! a later write to one of its inputs leaves it stale forever. Two stamps in +//! the column's field metadata close that: the definition it was computed +//! under, and a per-fragment signature of the input storage it was computed +//! from. A refresh compares both with the manifest and recomputes what +//! disagrees. Signatures are read from manifests, never from data. +//! +//! The map is not stored in the manifest: it is one entry per fragment per +//! column, which would dominate the manifest of a large table with many +//! computed columns. It lives in an immutable sidecar object under +//! `_computed/`, named by its content digest, and the field metadata holds +//! only the reference. Sidecars no retained version references are removed +//! by [`prune_sidecars`]. + +use std::collections::{BTreeMap, HashSet}; +use std::ops::Range; + +use arrow_array::{Array, UInt64Array}; +use futures::TryStreamExt; +use lance::Dataset; +use lance::dataset::transaction::Operation; +use lance_core::ROW_ADDR; +use lance_core::datatypes::{Field as LanceField, Schema as LanceSchema}; +use lance_io::object_store::ObjectStore; +use lance_table::format::{DataFile, Fragment}; +use object_store::path::Path; +use roaring::RoaringBitmap; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::table::computed_columns::{ + DEFINITION_VERSION_META_KEY, RECORDED_AT_VERSION_META_KEY, SOURCE_SIGNATURE_META_KEY, +}; +use crate::{Error, Result}; + +/// A map recorded further back than this carries nothing through the +/// compactions since: the fragments they produced are recomputed instead. +const MAX_CARRY_FORWARD_VERSIONS: u64 = 1024; + +/// `{fragment id -> input signature}`. +pub type SignatureMap = BTreeMap; + +/// Every field an input covers, keyed by column path: the field's own id +/// first, then its ancestors', because a packed file records the physical +/// column under an ancestor's id. +pub type InputFields = BTreeMap, Vec>; + +fn invalid(message: String) -> Error { + Error::InvalidInput { message } +} + +/// FNV-1a over the text, as hex. Stable across processes and versions, which +/// a signature compared against a stored one has to be. +fn short_hash(value: &str) -> String { + let digest = Sha256::digest(value.as_bytes()); + digest.iter().take(8).map(|b| format!("{b:02x}")).collect() +} + +/// Digest of the definition a column is computed under. +pub fn definition_version(definition: &str) -> String { + short_hash(definition) +} + +/// The fields the named input column paths cover, children included. Paths, +/// not ids, pair one schema with another: a rewrite renumbers fields. The key +/// is the path's components, so a field named `a.b` and a nested `a` -> `b` +/// are different columns. +pub fn fields_for_paths(schema: &LanceSchema, paths: &[String]) -> Result { + fn collect(field: &LanceField, path: Vec, ancestors: &[i32], out: &mut InputFields) { + let mut ids = vec![field.id]; + ids.extend_from_slice(ancestors); + for child in &field.children { + let mut child_path = path.clone(); + child_path.push(child.name.clone()); + collect(child, child_path, &ids, out); + } + out.insert(path, ids); + } + let mut out = InputFields::new(); + for field_path in paths { + let parts = lance_core::datatypes::parse_field_path(field_path)?; + let (root, rest) = parts + .split_first() + .ok_or_else(|| invalid("computed column input path is empty".to_string()))?; + let mut field = schema + .field(root) + .ok_or_else(|| invalid(format!("unknown computed column input '{field_path}'")))?; + let mut ancestors = Vec::new(); + for name in rest { + ancestors.insert(0, field.id); + field = field + .children + .iter() + .find(|child| child.name == *name) + .ok_or_else(|| invalid(format!("unknown computed column input '{field_path}'")))?; + } + collect(field, parts, &ancestors, &mut out); + } + Ok(out) +} + +/// Where one field's values come from in a fragment: the files and physical +/// columns storing it, and the overlays overriding cells of it, newest last +/// with the physical column and the cells each covers. A file stores the +/// field under its own id or, packed, under an ancestor's; `ids` is the +/// field's id followed by its ancestors'. Object identity is by base and +/// path; field ids are left out, since a sibling column's rewrite re-labels +/// them without touching a value. +#[derive(Debug, PartialEq, Eq)] +pub struct InputBasis { + files: Vec<(Option, String, i32)>, + overlays: Vec<(Option, String, i32, RoaringBitmap, u64)>, +} + +pub fn input_basis(metadata: &Fragment, ids: &[i32]) -> Result { + let column_of = |file: &DataFile| { + file.fields + .iter() + .position(|id| ids.contains(id)) + .map(|pos| (pos, file.column_indices.get(pos).copied().unwrap_or(-1))) + }; + let files = metadata + .files + .iter() + .filter_map(|file| { + column_of(file).map(|(_, column)| (file.base_id, file.path.clone(), column)) + }) + .collect(); + let mut overlays = Vec::new(); + for overlay in &metadata.overlays { + let Some((pos, column)) = column_of(&overlay.data_file) else { + continue; + }; + overlays.push(( + overlay.data_file.base_id, + overlay.data_file.path.clone(), + column, + overlay.coverage_for_field(pos)?.as_ref().clone(), + overlay.committed_version, + )); + } + Ok(InputBasis { files, overlays }) +} + +/// Identity of the input data a fragment currently holds: per input field, +/// its storage basis. Deletions are left out: a deleted row is never +/// computed, and the rows that stay keep their values. Physical identity, +/// not content, so a rewrite that preserves values still reads as a change; +/// compaction is followed separately. +pub fn fragment_input_signature(fragment: &Fragment, inputs: &InputFields) -> Result { + let mut parts = Vec::new(); + for (path, ids) in inputs { + let basis = input_basis(fragment, ids)?; + parts.push(format!("{}={basis:?}", path.join("."))); + } + Ok(short_hash(&parts.join("|"))) +} + +fn signature_of( + dataset: &Dataset, + fragment_id: u32, + inputs: &InputFields, +) -> Result> { + dataset + .get_fragment(fragment_id as usize) + .map(|fragment| fragment_input_signature(fragment.metadata(), inputs)) + .transpose() +} + +/// Signatures for `fragment_ids` as `dataset` currently holds them. +pub fn signatures_for( + dataset: &Dataset, + fragment_ids: &[u32], + inputs: &InputFields, +) -> Result { + let wanted: HashSet = fragment_ids.iter().copied().collect(); + dataset + .get_fragments() + .iter() + .filter(|fragment| wanted.contains(&(fragment.id() as u32))) + .map(|fragment| { + Ok(( + fragment.id() as u32, + fragment_input_signature(fragment.metadata(), inputs)?, + )) + }) + .collect() +} + +fn field_meta(dataset: &Dataset, column: &str, key: &str) -> Option { + dataset + .schema() + .field(column) + .and_then(|field| field.metadata.get(key)) + .cloned() +} + +/// What the column's stored map says. The three cases are distinct and the +/// callers act differently on each: see [`staleness_against`]. +#[derive(Debug)] +pub enum StoredSignatures { + /// No map has ever been written for this column. + Absent, + Present(SignatureMap), + /// A map exists but cannot be read. + Unreadable, +} + +/// Directory of signature sidecars, under the dataset root. +const SIDECAR_DIR: &str = "_computed"; +/// Metadata value prefix referencing a sidecar by its content digest. +const SIDECAR_REF: &str = "sidecar:"; +const SIDECAR_MAGIC: &[u8; 4] = b"CSIG"; +const SIDECAR_FORMAT: u8 = 1; +/// How long an unreferenced sidecar is presumed to be a stamp in flight: +/// lance's own threshold for unverified files, independent of how much +/// version history a cleanup keeps. +const SIDECAR_UNVERIFIED_THRESHOLD_DAYS: i64 = 7; + +/// The dataset's root directory: the parent of its versions directory. +/// Rebuilt from the raw parts, since re-encoding them would escape a +/// Windows drive letter's colon. +fn dataset_root(dataset: &Dataset) -> Path { + let versions = dataset.versions_dir(); + let count = versions.parts().count(); + Path::from_iter(versions.parts().take(count.saturating_sub(1))) +} + +fn sidecar_path(dataset: &Dataset, digest: &str) -> Path { + dataset_root(dataset) + .join(SIDECAR_DIR) + .join(format!("{digest}.sig")) +} + +/// `CSIG`, format byte, entry count, then one fragment id and 8-byte +/// signature per entry, all little-endian; 12 bytes per fragment. +fn encode_sidecar(map: &SignatureMap) -> Result> { + let mut bytes = Vec::with_capacity(9 + map.len() * 12); + bytes.extend_from_slice(SIDECAR_MAGIC); + bytes.push(SIDECAR_FORMAT); + bytes.extend_from_slice( + &u32::try_from(map.len()) + .map_err(|_| invalid("too many fragments for a signature sidecar".to_string()))? + .to_le_bytes(), + ); + for (fragment_id, signature) in map { + let hash = u64::from_str_radix(signature, 16).map_err(|_| { + invalid(format!( + "signature '{signature}' is not a 64-bit hex digest" + )) + })?; + bytes.extend_from_slice(&fragment_id.to_le_bytes()); + bytes.extend_from_slice(&hash.to_le_bytes()); + } + Ok(bytes) +} + +fn decode_sidecar(bytes: &[u8]) -> Result { + let malformed = || invalid("signature sidecar is malformed".to_string()); + if bytes.len() < 9 || &bytes[..4] != SIDECAR_MAGIC || bytes[4] != SIDECAR_FORMAT { + return Err(malformed()); + } + let count = u32::from_le_bytes(bytes[5..9].try_into().map_err(|_| malformed())?) as usize; + let body = &bytes[9..]; + if body.len() != count * 12 { + return Err(malformed()); + } + Ok(body + .chunks_exact(12) + .map(|entry| { + let fragment_id = u32::from_le_bytes(entry[..4].try_into().unwrap()); + let hash = u64::from_le_bytes(entry[4..].try_into().unwrap()); + (fragment_id, format!("{hash:016x}")) + }) + .collect()) +} + +fn digest_of(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|b| format!("{b:02x}")) + .collect() +} + +async fn store(dataset: &Dataset) -> Result> { + Ok(dataset.object_store(None).await?) +} + +/// Write `map` as a sidecar and return the metadata value referencing it. +/// The object is named by its digest, so two columns with the same map +/// share one object and a rewrite is idempotent. +async fn write_sidecar(dataset: &Dataset, map: &SignatureMap) -> Result { + let bytes = encode_sidecar(map)?; + let digest = digest_of(&bytes); + store(dataset) + .await? + .put(&sidecar_path(dataset, &digest), &bytes) + .await?; + Ok(format!("{SIDECAR_REF}{digest}")) +} + +async fn read_sidecar(dataset: &Dataset, digest: &str) -> Result { + let bytes = store(dataset) + .await? + .read_one_all(&sidecar_path(dataset, digest)) + .await?; + if digest_of(&bytes) != digest { + return Err(invalid(format!( + "signature sidecar {digest} does not match its digest" + ))); + } + decode_sidecar(&bytes) +} + +/// Remove signature sidecars that no version still present references, +/// the counterpart of lance's version cleanup for `_computed/`, with the +/// same protection for objects still being published: a sidecar is put +/// before the commit that references it, so one younger than +/// [`SIDECAR_UNVERIFIED_THRESHOLD_DAYS`] is left alone unless +/// `delete_unverified`. Returns how many were removed. +pub async fn prune_sidecars(dataset: &Dataset, delete_unverified: bool) -> Result { + let store = store(dataset).await?; + let dir = dataset_root(dataset).join(SIDECAR_DIR); + let unmodified_since = (!delete_unverified) + .then(|| chrono::Utc::now() - chrono::Duration::days(SIDECAR_UNVERIFIED_THRESHOLD_DAYS)); + let present: Vec = match store + .read_dir_all(&dir, unmodified_since) + .try_collect::>() + .await + { + Ok(objects) => objects + .into_iter() + .filter_map(|object| { + object + .location + .filename() + .and_then(|name| name.strip_suffix(".sig")) + .map(str::to_string) + }) + .collect(), + Err(_) => return Ok(0), + }; + if present.is_empty() { + return Ok(0); + } + let mut referenced = HashSet::new(); + for version in dataset.versions().await? { + let at = dataset.checkout_version(version.version).await?; + for field in at.schema().fields_pre_order() { + if let Some(digest) = field + .metadata + .get(SOURCE_SIGNATURE_META_KEY) + .and_then(|value| value.strip_prefix(SIDECAR_REF)) + { + referenced.insert(digest.to_string()); + } + } + } + let mut removed = 0; + for digest in present { + if !referenced.contains(&digest) { + store.delete(&sidecar_path(dataset, &digest)).await?; + removed += 1; + } + } + Ok(removed) +} + +/// Read the column's stored map. An unreadable map is a state, not an error: +/// failing here would make the column permanently unrefreshable, and the +/// unknown recomputes like every other unknown here. +pub async fn stored_signatures(dataset: &Dataset, column: &str) -> StoredSignatures { + let Some(encoded) = field_meta(dataset, column, SOURCE_SIGNATURE_META_KEY) else { + return StoredSignatures::Absent; + }; + // Inline JSON is the declaration's empty seed and the pre-sidecar form. + let read = match encoded.strip_prefix(SIDECAR_REF) { + Some(digest) => read_sidecar(dataset, digest).await, + None => serde_json::from_str(&encoded).map_err(|e| invalid(e.to_string())), + }; + match read { + Ok(map) => StoredSignatures::Present(map), + Err(error) => { + log::warn!( + "computed column '{column}' source signature map is unreadable ({error}); every fragment will be recomputed" + ); + StoredSignatures::Unreadable + } + } +} + +/// What a refresh must recompute beyond the null rows. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct StalenessPlan { + /// The definition changed, so every row is stale whatever its signature. + pub recompute_all: bool, + /// Fragments whose inputs moved since they were computed, or that were + /// never recorded. + pub dirty: HashSet, + /// Fragments a compaction produced from fresh ones, at their current + /// signature: not dirty, and for the seal to record. + pub inherited: SignatureMap, +} + +impl StalenessPlan { + pub fn is_dirty(&self, fragment_id: u32) -> bool { + self.recompute_all || self.dirty.contains(&fragment_id) + } +} + +/// A version of the log since the stamp that bears on carrying freshness +/// forward: an append, whose fragments hold no computed value yet, or a +/// compaction's rewrite groups, consumed and produced fragment ids. +enum Step { + Append, + Compaction(Vec<(Vec, Vec)>), +} + +/// The step committed at `version`. A compaction's transaction records a +/// new fragment before its id is assigned, so produced ids come from the +/// version's manifest, matched by data file. Every other operation is +/// skipped: a row-moving update rewrites the rows it moves, so it does not +/// carry their inputs unchanged. +async fn step_at(dataset: &Dataset, version: u64) -> Result> { + let Some(transaction) = dataset.read_transaction_by_version(version).await? else { + return Ok(None); + }; + let groups = match &transaction.operation { + Operation::Append { .. } => return Ok(Some(Step::Append)), + Operation::Rewrite { groups, .. } => groups, + _ => return Ok(None), + }; + let at = dataset.checkout_version(version).await?; + let by_file: BTreeMap<(Option, String), u32> = at + .get_fragments() + .iter() + .flat_map(|fragment| { + let id = fragment.id() as u32; + fragment + .metadata() + .files + .iter() + .map(move |file| ((file.base_id, file.path.clone()), id)) + }) + .collect(); + let compactions = groups + .iter() + .map(|group| { + let consumed = group.old_fragments.iter().map(|f| f.id as u32).collect(); + let produced = group + .new_fragments + .iter() + .map(|fragment| { + fragment + .files + .iter() + .find_map(|file| by_file.get(&(file.base_id, file.path.clone())).copied()) + .ok_or_else(|| { + invalid(format!( + "a fragment added in version {version} is not in that version's manifest" + )) + }) + }) + .collect::>>()?; + Ok((consumed, produced)) + }) + .collect::>>()?; + Ok(Some(Step::Compaction(compactions))) +} + +/// Manifests since the stamp, each loaded once. +struct Manifests<'a> { + dataset: &'a Dataset, + loaded: BTreeMap, +} + +impl Manifests<'_> { + async fn at(&mut self, version: u64) -> Result<&Dataset> { + if !self.loaded.contains_key(&version) { + let manifest = self.dataset.checkout_version(version).await?; + self.loaded.insert(version, manifest); + } + Ok(&self.loaded[&version]) + } +} + +/// Whether a fragment consumed by a compaction was created by an append +/// since the stamp and its inputs never moved after: `current` is its +/// signature just before the compaction. LanceDB's writes leave a computed +/// column null (see `ensure_not_written`), but a raw append need not, so +/// the rows it contributed to the product are checked to hold no value +/// (`holds_values_in`) before the product inherits freshness. +async fn appended_untouched( + manifests: &mut Manifests<'_>, + appends: &[u64], + fragment_id: u32, + current: Option<&String>, + inputs: &InputFields, +) -> Result { + let Some(current) = current else { + return Ok(false); + }; + for &version in appends.iter().rev() { + let Some(born) = signature_of(manifests.at(version).await?, fragment_id, inputs)? else { + continue; + }; + if signature_of(manifests.at(version - 1).await?, fragment_id, inputs)?.is_some() { + // Live before this append: born earlier. + continue; + } + return Ok(&born == current); + } + Ok(false) +} + +/// Whether `column` holds a value in any of `ranges`, offsets within the +/// fragment. Compaction scans its sources in order, so the rows an appended +/// source contributed sit at known offsets of the product. +async fn holds_values_in( + dataset: &Dataset, + fragment_id: u32, + column: &str, + ranges: &[Range], +) -> Result { + let Some(fragment) = dataset.get_fragment(fragment_id as usize) else { + return Ok(false); + }; + let mut scanner = dataset.scan(); + scanner + .with_fragments(vec![fragment.metadata().clone()]) + .with_row_address() + .project(&[column])? + .filter(&format!( + "{} IS NOT NULL", + super::refresh::quote_identifier(column) + ))?; + let mut batches = scanner.try_into_stream().await?; + while let Some(batch) = batches.try_next().await? { + let addresses = batch + .column_by_name(ROW_ADDR) + .and_then(|column| column.as_any().downcast_ref::()) + .ok_or_else(|| invalid("row addresses missing from a freshness scan".to_string()))?; + if addresses + .iter() + .flatten() + .map(|address| address & 0xFFFF_FFFF) + .any(|offset| ranges.iter().any(|range| range.contains(&offset))) + { + return Ok(true); + } + } + Ok(false) +} + +/// Compaction copies inputs verbatim, so a fragment it produced from +/// recorded fragments whose inputs had not moved is as fresh as they were, +/// and a fragment an append created since the stamp, untouched after and +/// unfilled in the product, changes nothing (`appended_untouched`). +/// Followed from the version the map was recorded at through every +/// compaction since, so a chain of them carries too. Returns the produced +/// fragments' signatures at production; the caller compares each with the +/// current manifest, which catches anything written to them afterwards. +async fn carried_forward( + dataset: &Dataset, + column: &str, + stored: &SignatureMap, + inputs: &InputFields, +) -> Result { + let Some(recorded_at) = field_meta(dataset, column, RECORDED_AT_VERSION_META_KEY) + .and_then(|version| version.parse::().ok()) + else { + return Ok(SignatureMap::new()); + }; + let to = dataset.version().version; + if to <= recorded_at || to - recorded_at > MAX_CARRY_FORWARD_VERSIONS { + return Ok(SignatureMap::new()); + } + let mut fresh = stored.clone(); + let mut inherited = SignatureMap::new(); + let mut appends = Vec::new(); + let mut manifests = Manifests { + dataset, + loaded: BTreeMap::new(), + }; + for version in (recorded_at + 1)..=to { + let compactions = match step_at(dataset, version).await? { + None => continue, + Some(Step::Append) => { + appends.push(version); + continue; + } + Some(Step::Compaction(compactions)) => compactions, + }; + for (consumed, produced) in compactions { + // Each source's signature and live rows just before the + // compaction; live rows place its contribution in the product. + let mut sources = Vec::with_capacity(consumed.len()); + { + let before = manifests.at(version - 1).await?; + for id in &consumed { + let live = match before.get_fragment(*id as usize) { + Some(fragment) => fragment.count_rows(None).await? as u64, + None => 0, + }; + sources.push((*id, signature_of(before, *id, inputs)?, live)); + } + } + let mut all_fresh = !consumed.is_empty(); + let mut appended = Vec::new(); + let mut offset = 0u64; + for (id, current, live) in sources { + let recorded = matches!( + (fresh.get(&id), current.as_ref()), + (Some(recorded), Some(current)) if recorded == current + ); + if !recorded { + if appended_untouched(&mut manifests, &appends, id, current.as_ref(), inputs) + .await? + { + appended.push(offset..offset + live); + } else { + all_fresh = false; + break; + } + } + offset += live; + } + if !all_fresh { + continue; + } + let after = manifests.at(version).await?; + // The products in order, each with its share of the appended + // rows; a value there was supplied by the append, not computed. + let mut base = 0u64; + let mut unfilled = true; + for id in &produced { + let rows = match after.get_fragment(*id as usize) { + Some(fragment) => fragment.count_rows(None).await? as u64, + None => 0, + }; + let local: Vec> = appended + .iter() + .filter(|range| range.start < base + rows && range.end > base) + .map(|range| range.start.max(base) - base..range.end.min(base + rows) - base) + .collect(); + if !local.is_empty() && holds_values_in(after, *id, column, &local).await? { + unfilled = false; + break; + } + base += rows; + } + if !unfilled { + continue; + } + for id in &produced { + if let Some(signature) = signature_of(after, *id, inputs)? { + fresh.insert(*id, signature.clone()); + inherited.insert(*id, signature); + } + } + } + } + Ok(inherited) +} + +/// Decide what is stale, from the manifest alone. +/// +/// A column with no map at all was declared before signatures existed. It +/// keeps its null-fill behavior until its first stamp enrolls it (see +/// [`record_freshness`]). A map that omits a fragment is authoritative: the +/// fragment's input state is unknown, and unknown recomputes -- unless a +/// compaction of fresh fragments produced it, which is followed. Lineage is +/// read only when a live fragment has no entry, so a plan over a recorded +/// table costs no transaction reads. +pub async fn staleness_against( + dataset: &Dataset, + column: &str, + definition_version: &str, + inputs: &InputFields, +) -> Result { + let stored = match stored_signatures(dataset, column).await { + StoredSignatures::Absent => return Ok(StalenessPlan::default()), + StoredSignatures::Unreadable => { + return Ok(StalenessPlan { + recompute_all: true, + ..Default::default() + }); + } + StoredSignatures::Present(stored) => stored, + }; + let stored_version = field_meta(dataset, column, DEFINITION_VERSION_META_KEY); + if stored_version.is_some_and(|version| version != definition_version) { + return Ok(StalenessPlan { + recompute_all: true, + ..Default::default() + }); + } + let unrecorded = dataset + .get_fragments() + .iter() + .any(|fragment| !stored.contains_key(&(fragment.id() as u32))); + let mut inherited = if unrecorded { + carried_forward(dataset, column, &stored, inputs).await? + } else { + SignatureMap::new() + }; + let mut dirty = HashSet::new(); + let mut live = HashSet::new(); + for fragment in dataset.get_fragments() { + let id = fragment.id() as u32; + live.insert(id); + let current = fragment_input_signature(fragment.metadata(), inputs)?; + if stored.get(&id).or_else(|| inherited.get(&id)) != Some(¤t) { + dirty.insert(id); + } + } + inherited.retain(|id, _| live.contains(id) && !dirty.contains(id)); + Ok(StalenessPlan { + recompute_all: false, + dirty, + inherited, + }) +} + +/// What [`record_freshness`] wrote: the table version the stamp landed at, +/// if it wrote one, and the entries recorded and dropped for having moved. +#[derive(Debug, Default, PartialEq, Eq)] +pub struct FreshnessRecord { + pub version: Option, + pub recorded: usize, + pub moved: usize, +} + +/// Record, on `column`, the input state its fragments were computed from. +/// +/// `computed` is what this refresh computed in full, signed at the version +/// the values were read from. A column with no map yet was declared before +/// signatures existed; `pinned`, the version the refresh planned against, +/// is then the baseline: every fragment live there is trusted as it stood, +/// the null-fill contract its values were written under. Otherwise the +/// staleness decided on `pinned` supplies what compactions since the last +/// stamp carried forward. Either +/// way an entry is recorded only if `latest` still holds that input state -- +/// an input write can rebase under the output commit -- so a fragment whose +/// inputs moved stays unrecorded and is recomputed by the next refresh. +/// +/// Written once per refresh, after its data commit. +pub async fn record_freshness( + latest: &mut Dataset, + pinned: Option<(&Dataset, &StalenessPlan)>, + column: &str, + definition_version: &str, + inputs: &InputFields, + computed: SignatureMap, +) -> Result { + let absent = matches!( + stored_signatures(latest, column).await, + StoredSignatures::Absent + ); + let mut entries = SignatureMap::new(); + if let Some((pinned, staleness)) = pinned { + if absent { + let all: Vec = pinned + .get_fragments() + .iter() + .map(|fragment| fragment.id() as u32) + .collect(); + entries = signatures_for(pinned, &all, inputs)?; + } else { + entries = staleness.inherited.clone(); + } + } + entries.extend(computed); + if entries.is_empty() && !absent { + return Ok(FreshnessRecord::default()); + } + let fragments: Vec = entries.keys().copied().collect(); + let current = signatures_for(latest, &fragments, inputs)?; + let verified: SignatureMap = entries + .into_iter() + .filter(|(fragment_id, signature)| current.get(fragment_id) == Some(signature)) + .collect(); + let recorded = verified.len(); + let version = write_signatures(latest, column, definition_version, verified).await?; + Ok(FreshnessRecord { + version: Some(version), + recorded, + moved: fragments.len() - recorded, + }) +} + +/// Merge `entries` into the column's stored map and stamp the definition and +/// the version the map now describes. Merges rather than replaces: the +/// entries cover only the fragments this refresh wrote, and every fragment it +/// skipped keeps the entry an earlier one left. Entries for fragments no +/// longer in the manifest are dropped, so compaction cannot grow the map +/// without bound. Returns the version the stamp landed at. +pub async fn write_signatures( + dataset: &mut Dataset, + column: &str, + definition_version: &str, + entries: SignatureMap, +) -> Result { + let live: HashSet = dataset + .get_fragments() + .iter() + .map(|fragment| fragment.id() as u32) + .collect(); + // An unreadable map is discarded rather than merged: nothing in it can be + // trusted, and its fragments recompute until a later refresh records them. + let mut merged = match stored_signatures(dataset, column).await { + StoredSignatures::Present(stored) => stored, + StoredSignatures::Absent | StoredSignatures::Unreadable => SignatureMap::new(), + }; + merged.extend(entries); + merged.retain(|fragment_id, _| live.contains(fragment_id)); + // The sidecar is durable before the commit references it; a failure in + // between leaves an unreferenced object for `prune_sidecars`. + let encoded = if merged.is_empty() { + "{}".to_string() + } else { + write_sidecar(dataset, &merged).await? + }; + let describes = dataset.version().version; + dataset + .update_field_metadata() + .update( + column, + [ + (SOURCE_SIGNATURE_META_KEY.to_string(), encoded), + ( + DEFINITION_VERSION_META_KEY.to_string(), + definition_version.to_string(), + ), + ( + RECORDED_AT_VERSION_META_KEY.to_string(), + describes.to_string(), + ), + ], + )? + .await?; + Ok(dataset.version().version) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + use arrow_array::RecordBatchIterator; + use arrow_schema::{DataType, Field as ArrowField, Schema as ArrowSchema}; + use lance::dataset::{ + MergeInsertBuilder, MergeInsertWriteMode, NewColumnTransform, WhenMatched, WhenNotMatched, + WriteMode, WriteParams, + }; + use lance_file::version::ConcreteFileVersion; + use lance_table::format::overlay::{DataOverlayFile, OverlayCoverage}; + + const COLUMN: &str = "doubled"; + const EXPRESSION: &str = "value * 2"; + + /// Two fragments of 50 rows, `id` and `value`, with `doubled` declared + /// all-null against `value`, tracked from birth. + async fn table(uri: &str) -> Dataset { + let batch = arrow_array::record_batch!( + ("id", Int32, (0..100).collect::>()), + ("value", Int32, (0..100).collect::>()) + ) + .unwrap(); + let schema = batch.schema(); + let mut dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + uri, + Some(WriteParams { + mode: WriteMode::Create, + max_rows_per_file: 50, + ..Default::default() + }), + ) + .await + .unwrap(); + let mut metadata = std::collections::HashMap::from([ + ( + crate::table::computed_columns::COMPUTED_COLUMN_META_KEY.to_string(), + "true".to_string(), + ), + ( + crate::table::computed_columns::EXPRESSION_META_KEY.to_string(), + EXPRESSION.to_string(), + ), + ]); + metadata.insert(SOURCE_SIGNATURE_META_KEY.to_string(), "{}".to_string()); + dataset + .add_columns( + NewColumnTransform::AllNulls(Arc::new(ArrowSchema::new(vec![ + ArrowField::new(COLUMN, DataType::Int64, true).with_metadata(metadata), + ]))), + None, + None, + ) + .await + .unwrap(); + dataset + } + + fn inputs(dataset: &Dataset) -> InputFields { + fields_for_paths(dataset.schema(), &["value".to_string()]).unwrap() + } + + async fn plan(dataset: &Dataset) -> StalenessPlan { + staleness_against( + dataset, + COLUMN, + &definition_version(EXPRESSION), + &inputs(dataset), + ) + .await + .unwrap() + } + + async fn stamp_all(dataset: &mut Dataset) { + let ids = inputs(dataset); + let frags: Vec = dataset + .get_fragments() + .iter() + .map(|f| f.id() as u32) + .collect(); + let entries = signatures_for(dataset, &frags, &ids).unwrap(); + write_signatures(dataset, COLUMN, &definition_version(EXPRESSION), entries) + .await + .unwrap(); + } + + /// Strip the refresh's own keys: a column from before signatures existed. + async fn make_legacy(dataset: &mut Dataset) { + let declaration = dataset + .schema() + .field(COLUMN) + .unwrap() + .metadata + .iter() + .filter(|(key, _)| !key.starts_with("computed_refresh.")) + .map(|(key, value)| (key.clone(), value.clone())) + .collect::>(); + dataset + .update_field_metadata() + .replace(COLUMN, declaration) + .unwrap() + .await + .unwrap(); + assert!(matches!( + stored_signatures(dataset, COLUMN).await, + StoredSignatures::Absent + )); + } + + /// Rewrite `value` of the row with `id` in place: a partial merge-insert + /// attaches a new column file to the row's fragment, keeping its id. + async fn rewrite_value(dataset: &mut Dataset, id: i32) { + let batch = + arrow_array::record_batch!(("id", Int32, [id]), ("value", Int32, [1000])).unwrap(); + let schema = batch.schema(); + let mut builder = + MergeInsertBuilder::try_new(Arc::new(dataset.clone()), vec!["id".to_string()]).unwrap(); + builder + .when_matched(WhenMatched::UpdateAll) + .when_not_matched(WhenNotMatched::DoNothing) + .write_mode(MergeInsertWriteMode::RewriteColumns); + let (updated, _) = builder + .try_build() + .unwrap() + .execute_reader(RecordBatchIterator::new([Ok(batch)], schema)) + .await + .unwrap(); + *dataset = (*updated).clone(); + } + + async fn compact(dataset: &mut Dataset) -> u32 { + lance::dataset::optimize::compact_files( + dataset, + lance::dataset::optimize::CompactionOptions { + target_rows_per_fragment: 1000, + ..Default::default() + }, + None, + ) + .await + .unwrap(); + dataset.checkout_latest().await.unwrap(); + dataset.get_fragments()[0].id() as u32 + } + + /// `a.b` under `root` and a nested `a` -> `b` are different columns with + /// two bases; a dotted-string key would fold them into one. + #[test] + fn a_dotted_field_name_is_not_a_nested_path() { + let leaf = |name: &str| ArrowField::new(name, DataType::Int32, true); + let root = ArrowField::new( + "root", + DataType::Struct( + vec![ + ArrowField::new("a", DataType::Struct(vec![leaf("b")].into()), true), + leaf("a.b"), + ] + .into(), + ), + true, + ); + let schema = LanceSchema::try_from(&ArrowSchema::new(vec![root])).unwrap(); + let ids = fields_for_paths(&schema, &["root".to_string()]).unwrap(); + let path = |parts: &[&str]| parts.iter().map(|p| p.to_string()).collect::>(); + let nested = &ids[&path(&["root", "a", "b"])]; + let dotted = &ids[&path(&["root", "a.b"])]; + assert_ne!(nested[0], dotted[0], "{ids:?}"); + assert_eq!(ids.len(), 4, "{ids:?}"); + } + + /// A packed file records the physical column under an ancestor's id, so a + /// nested input's basis is found through its ancestors. + #[test] + fn a_packed_nested_input_has_a_file_basis() { + let word_count = ArrowField::new("word_count", DataType::Int32, true); + let metrics = ArrowField::new("metrics", DataType::Struct(vec![word_count].into()), true); + let analysis = ArrowField::new("analysis", DataType::Struct(vec![metrics].into()), true); + let schema = LanceSchema::try_from(&ArrowSchema::new(vec![analysis])).unwrap(); + let ids = fields_for_paths(&schema, &["analysis.metrics.word_count".to_string()]).unwrap(); + let word_count = &ids[&["analysis", "metrics", "word_count"] + .map(String::from) + .to_vec()]; + let analysis = schema.field("analysis").unwrap().id; + assert_eq!(word_count.last(), Some(&analysis), "{word_count:?}"); + let mut fragment = Fragment::new(0); + fragment.files.push(DataFile::new( + "packed.lance", + vec![analysis], + vec![3], + ConcreteFileVersion::V2_2, + None, + None, + )); + let basis = input_basis(&fragment, word_count).unwrap(); + assert_eq!(basis.files, vec![(None, "packed.lance".to_string(), 3)]); + } + + /// An overlay that stores the input in another physical column of the + /// same object is a different basis. + #[test] + fn an_overlay_column_remap_changes_the_basis() { + let overlay = |column: i32| { + let mut fragment = Fragment::new(0); + fragment.overlays.push(DataOverlayFile { + data_file: DataFile::new( + "overlay.lance", + vec![7], + vec![column], + ConcreteFileVersion::V2_2, + None, + None, + ), + coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), + committed_version: 2, + }); + input_basis(&fragment, &[7]).unwrap() + }; + assert_ne!(overlay(0), overlay(1)); + assert_eq!(overlay(0), overlay(0)); + } + + async fn sidecar_names(dataset: &Dataset) -> Vec { + let mut names = store(dataset) + .await + .unwrap() + .read_dir(dataset_root(dataset).join(SIDECAR_DIR)) + .await + .unwrap_or_default(); + names.sort(); + names + } + + fn signature_ref(dataset: &Dataset, column: &str) -> String { + field_meta(dataset, column, SOURCE_SIGNATURE_META_KEY).unwrap() + } + + /// The manifest carries only a digest; the map itself is a sidecar the + /// reader fetches and verifies. The declaration's empty seed stays inline. + #[tokio::test] + async fn a_stamp_is_a_sidecar_the_manifest_only_references() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + assert_eq!(signature_ref(&dataset, COLUMN), "{}"); + stamp_all(&mut dataset).await; + let reference = signature_ref(&dataset, COLUMN); + let digest = reference.strip_prefix(SIDECAR_REF).unwrap(); + assert_eq!(reference.len(), SIDECAR_REF.len() + 64, "{reference}"); + assert_eq!(sidecar_names(&dataset).await, vec![format!("{digest}.sig")]); + let StoredSignatures::Present(stored) = stored_signatures(&dataset, COLUMN).await else { + panic!("sidecar unreadable"); + }; + assert_eq!( + stored, + signatures_for(&dataset, &[0, 1], &inputs(&dataset)).unwrap() + ); + } + + /// Two columns with the same inputs share one sidecar, and a rewrite of + /// the same map is idempotent. + #[tokio::test] + async fn identical_maps_share_one_sidecar() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + stamp_all(&mut dataset).await; + let ids = inputs(&dataset); + let entries = signatures_for(&dataset, &[0, 1], &ids).unwrap(); + write_signatures( + &mut dataset, + "value", + &definition_version(EXPRESSION), + entries, + ) + .await + .unwrap(); + assert_eq!(sidecar_names(&dataset).await.len(), 1); + assert_eq!( + signature_ref(&dataset, COLUMN), + signature_ref(&dataset, "value") + ); + } + + /// A sidecar that is missing or whose bytes do not match the digest is + /// unreadable: everything recomputes and the next stamp replaces it. + #[tokio::test] + async fn a_missing_or_corrupt_sidecar_recomputes_everything() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let digest = signature_ref(&dataset, COLUMN) + .strip_prefix(SIDECAR_REF) + .unwrap() + .to_string(); + let path = sidecar_path(&dataset, &digest); + let object_store = store(&dataset).await.unwrap(); + object_store.put(&path, b"CSIG garbage").await.unwrap(); + assert!(plan(&dataset).await.recompute_all); + object_store.delete(&path).await.unwrap(); + assert!(plan(&dataset).await.recompute_all); + stamp_all(&mut dataset).await; + assert_eq!(plan(&dataset).await, StalenessPlan::default()); + } + + /// A map written inline, the pre-sidecar form, is still read. + #[tokio::test] + async fn an_inline_map_is_still_read() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + let ids = inputs(&dataset); + let entries = signatures_for(&dataset, &[0, 1], &ids).unwrap(); + dataset + .update_field_metadata() + .update( + COLUMN, + [ + ( + SOURCE_SIGNATURE_META_KEY.to_string(), + serde_json::to_string(&entries).unwrap(), + ), + ( + DEFINITION_VERSION_META_KEY.to_string(), + definition_version(EXPRESSION), + ), + ], + ) + .unwrap() + .await + .unwrap(); + assert_eq!(plan(&dataset).await, StalenessPlan::default()); + } + + /// Pruning removes only sidecars no version still present references: + /// an older stamp survives while its version does, and goes with it. + #[tokio::test] + async fn pruning_follows_version_retention() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + let ids = inputs(&dataset); + let first = signatures_for(&dataset, &[0], &ids).unwrap(); + write_signatures(&mut dataset, COLUMN, &definition_version(EXPRESSION), first) + .await + .unwrap(); + stamp_all(&mut dataset).await; + assert_eq!(sidecar_names(&dataset).await.len(), 2); + // Orphan from a stamp that never committed. + store(&dataset) + .await + .unwrap() + .put(&sidecar_path(&dataset, "orphan"), b"CSIG") + .await + .unwrap(); + assert_eq!(prune_sidecars(&dataset, true).await.unwrap(), 1); + assert_eq!(sidecar_names(&dataset).await.len(), 2); + + dataset + .cleanup_old_versions(chrono::Duration::zero(), Some(true), None) + .await + .unwrap(); + assert_eq!(prune_sidecars(&dataset, true).await.unwrap(), 1); + let remaining = sidecar_names(&dataset).await; + let current = signature_ref(&dataset, COLUMN); + assert_eq!( + remaining, + vec![format!( + "{}.sig", + current.strip_prefix(SIDECAR_REF).unwrap() + )] + ); + } + + /// The gate's reproducer: a sidecar is put before the commit that + /// references it, so a prune that interleaves must leave a recent, + /// as yet unreferenced object alone whatever version retention the + /// caller chose; only an unverified prune takes it. + #[tokio::test] + async fn pruning_does_not_collect_an_in_flight_sidecar() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + let ids = inputs(&dataset); + let entries = signatures_for(&dataset, &[0, 1], &ids).unwrap(); + let reference = write_sidecar(&dataset, &entries).await.unwrap(); + + let removed = prune_sidecars(&dataset, false).await.unwrap(); + assert_eq!(removed, 0); + dataset + .update_field_metadata() + .update(COLUMN, [(SOURCE_SIGNATURE_META_KEY.to_string(), reference)]) + .unwrap() + .await + .unwrap(); + assert!(matches!( + stored_signatures(&dataset, COLUMN).await, + StoredSignatures::Present(_) + )); + + let orphan = write_sidecar(&dataset, &SignatureMap::from([(9, "0".repeat(16))])) + .await + .unwrap(); + assert_eq!( + prune_sidecars(&dataset, false).await.unwrap(), + 0, + "a recent orphan waits for its window" + ); + assert_eq!(prune_sidecars(&dataset, true).await.unwrap(), 1); + assert!(!sidecar_names(&dataset).await.contains(&format!( + "{}.sig", + orphan.strip_prefix(SIDECAR_REF).unwrap() + ))); + } + + /// A freshly declared column is tracked from birth: every fragment is + /// unrecorded, so every fragment is stale until a refresh records it. + #[tokio::test] + async fn a_declared_column_is_stale_until_recorded() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + assert_eq!(plan(&dataset).await.dirty, HashSet::from([0, 1])); + stamp_all(&mut dataset).await; + assert_eq!(plan(&dataset).await, StalenessPlan::default()); + } + + /// The signature answers "did my inputs move": a write to any other + /// column, the computed column included, leaves it alone. + #[tokio::test] + async fn an_unrelated_column_rewrite_leaves_the_signature_alone() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + let ids = inputs(&dataset); + let before = signatures_for(&dataset, &[0, 1], &ids).unwrap(); + dataset + .add_columns( + NewColumnTransform::SqlExpressions(vec![("extra".into(), "id * 3".into())]), + None, + None, + ) + .await + .unwrap(); + assert_eq!(before, signatures_for(&dataset, &[0, 1], &ids).unwrap()); + } + + /// An in-place write to one fragment's input keeps every fragment id, so + /// only the signature can notice -- and on that fragment alone. + #[tokio::test] + async fn an_in_place_input_change_dirties_only_its_own_fragment() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + rewrite_value(&mut dataset, 60).await; + let stale = plan(&dataset).await; + assert_eq!(stale.dirty, HashSet::from([1]), "{stale:?}"); + } + + /// A deleted row is never computed and the rows that stay keep their + /// values, so a delete dirties nothing. + #[tokio::test] + async fn a_delete_dirties_nothing() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + dataset.delete("id >= 60 AND id < 70").await.unwrap(); + assert_eq!(plan(&dataset).await, StalenessPlan::default()); + } + + /// A definition change makes every row stale whatever the signatures say. + #[tokio::test] + async fn a_definition_change_recomputes_every_row() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let rebound = staleness_against(&dataset, COLUMN, "other", &inputs(&dataset)) + .await + .unwrap(); + assert!(rebound.recompute_all); + } + + /// A column declared before signatures existed carries no map, and keeps + /// null-fill behavior until its first stamp enrolls it -- at the pinned + /// state, not the latest: an input that moved in between is left out. + #[tokio::test] + async fn a_first_stamp_enrolls_an_older_column_as_it_stood() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + make_legacy(&mut dataset).await; + assert_eq!(plan(&dataset).await, StalenessPlan::default()); + let ids = inputs(&dataset); + let pinned = dataset.clone(); + let staleness = plan(&pinned).await; + rewrite_value(&mut dataset, 60).await; + let record = record_freshness( + &mut dataset, + Some((&pinned, &staleness)), + COLUMN, + &definition_version(EXPRESSION), + &ids, + SignatureMap::new(), + ) + .await + .unwrap(); + assert_eq!((record.recorded, record.moved), (1, 1)); + assert_eq!(record.version, Some(dataset.version().version)); + assert_eq!(plan(&dataset).await.dirty, HashSet::from([1])); + } + + /// Append `count` rows after `first_id`, the computed column null as a + /// write must leave it; returns the new fragment's id. + async fn append_rows(dataset: &mut Dataset, first_id: i32, count: i32) -> u32 { + let batch = arrow_array::record_batch!( + ( + "id", + Int32, + (first_id..first_id + count).collect::>() + ), + ( + "value", + Int32, + (first_id..first_id + count).collect::>() + ), + ("doubled", Int64, vec![None::; count as usize]) + ) + .unwrap(); + let schema = batch.schema(); + *dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + dataset.uri(), + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + dataset.get_fragments().last().unwrap().id() as u32 + } + + /// A fragment an append created since the stamp holds no computed value, + /// so a compaction folding it into recorded fragments produces a fresh + /// fragment: nothing recomputes, and the null fill covers the new rows. + #[tokio::test] + async fn a_compaction_folding_an_untouched_appended_fragment_carries_freshness_forward() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let appended = append_rows(&mut dataset, 100, 10).await; + assert_eq!(plan(&dataset).await.dirty, HashSet::from([appended])); + let compacted = compact(&mut dataset).await; + let stale = plan(&dataset).await; + assert!(stale.dirty.is_empty(), "{stale:?}"); + assert_eq!( + stale.inherited.keys().copied().collect::>(), + vec![compacted] + ); + } + + /// The same fragment with an input rewritten after the append is not + /// neutral: what it holds may have been computed from the older input. + #[tokio::test] + async fn a_compaction_folding_an_appended_fragment_whose_input_moved_recomputes() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + append_rows(&mut dataset, 100, 10).await; + rewrite_value(&mut dataset, 105).await; + let compacted = compact(&mut dataset).await; + let stale = plan(&dataset).await; + assert_eq!(stale.dirty, HashSet::from([compacted]), "{stale:?}"); + assert!(stale.inherited.is_empty()); + } + + /// A raw append can supply a computed value LanceDB's own writes never + /// do. The product holds it where the appended rows landed, so the + /// compaction is not carried: the value is recomputed, not certified. + #[tokio::test] + async fn a_compaction_folding_an_appended_fragment_with_values_recomputes() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let batch = arrow_array::record_batch!( + ("id", Int32, [100, 101]), + ("value", Int32, [100, 101]), + ("doubled", Int64, [None, Some(999_i64)]) + ) + .unwrap(); + let schema = batch.schema(); + dataset = Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + dataset.uri(), + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + let compacted = compact(&mut dataset).await; + let stale = plan(&dataset).await; + assert_eq!(stale.dirty, HashSet::from([compacted]), "{stale:?}"); + assert!(stale.inherited.is_empty()); + } + + /// Compaction copies inputs unchanged: a fragment it produced from + /// recorded, unmoved fragments is fresh, and the seal records it. One + /// produced from a fragment whose inputs had moved is not. + #[tokio::test] + async fn a_compaction_of_fresh_fragments_carries_freshness_forward() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let compacted = compact(&mut dataset).await; + let stale = plan(&dataset).await; + assert!(stale.dirty.is_empty(), "{stale:?}"); + assert_eq!( + stale.inherited.keys().copied().collect::>(), + vec![compacted] + ); + let ids = inputs(&dataset); + let pinned = dataset.clone(); + let record = record_freshness( + &mut dataset, + Some((&pinned, &stale)), + COLUMN, + &definition_version(EXPRESSION), + &ids, + SignatureMap::new(), + ) + .await + .unwrap(); + assert_eq!(record.recorded, 1); + let StoredSignatures::Present(stored) = stored_signatures(&dataset, COLUMN).await else { + panic!("stamped"); + }; + assert_eq!( + stored.keys().copied().collect::>(), + vec![compacted] + ); + + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + rewrite_value(&mut dataset, 60).await; + let compacted = compact(&mut dataset).await; + let stale = plan(&dataset).await; + assert_eq!(stale.dirty, HashSet::from([compacted]), "{stale:?}"); + assert!(stale.inherited.is_empty()); + } + + /// A recorded fragment whose signature no longer matches, or a fragment + /// the map omits without a compaction to explain it, is stale. + #[tokio::test] + async fn a_fragment_missing_from_the_stored_map_is_stale() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + let ids = inputs(&dataset); + let partial = signatures_for(&dataset, &[0], &ids).unwrap(); + write_signatures( + &mut dataset, + COLUMN, + &definition_version(EXPRESSION), + partial, + ) + .await + .unwrap(); + assert_eq!(plan(&dataset).await.dirty, HashSet::from([1])); + } + + /// An unreadable map recomputes everything rather than failing the + /// refresh, and the next stamp replaces it. + #[tokio::test] + async fn an_unreadable_stored_map_recomputes_rather_than_failing() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + dataset + .update_field_metadata() + .update( + COLUMN, + [(SOURCE_SIGNATURE_META_KEY.to_string(), "{".to_string())], + ) + .unwrap() + .await + .unwrap(); + assert!(plan(&dataset).await.recompute_all); + stamp_all(&mut dataset).await; + assert_eq!(plan(&dataset).await, StalenessPlan::default()); + } +} diff --git a/rust/lancedb/src/table/optimize.rs b/rust/lancedb/src/table/optimize.rs index 4ad58cffb..6e3fc0048 100644 --- a/rust/lancedb/src/table/optimize.rs +++ b/rust/lancedb/src/table/optimize.rs @@ -134,9 +134,17 @@ pub(crate) async fn cleanup_old_versions( ) -> Result { table.dataset.ensure_mutable()?; let dataset = table.dataset.get().await?; - Ok(dataset + let stats = dataset .cleanup_old_versions(older_than, delete_unverified, error_if_tagged_old_versions) - .await?) + .await?; + // Computed-column signature sidecars live outside lance's directories; + // drop the ones the surviving versions no longer reference. + let removed = + super::freshness::prune_sidecars(&dataset, delete_unverified.unwrap_or(false)).await?; + if removed > 0 { + log::debug!("removed {removed} unreferenced computed-column signature sidecars"); + } + Ok(stats) } /// Compact files in the dataset. diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index 511fce8ff..2ed479256 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -3,9 +3,9 @@ //! Filling computed columns. //! -//! A row without a value gets one; a row that has one keeps it. Refresh is -//! therefore idempotent and does not observe input mutation -- once a row is -//! filled, changing what the expression reads leaves the stored result alone. +//! A row without a value gets one; a row that has one keeps it unless its +//! fragment's inputs moved since it was computed, which `freshness` decides +//! from the manifest and stamps after every fill. //! //! A column's computed inputs are filled first -- the dependency graph is //! walked once, each reachable column filled once in dependency order, each @@ -31,6 +31,7 @@ use std::collections::HashSet; use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; use arrow_array::{ Array, ArrayRef, BooleanArray, LargeBinaryArray, RecordBatch, RecordBatchOptions, StructArray, @@ -48,6 +49,7 @@ use lance_core::datatypes::{BlobHandling, Schema as LanceSchema}; use serde::{Deserialize, Serialize}; use super::computed_columns::{BoundExpression, ComputedColumnKind, computed_column_from_field}; +use super::freshness::{self, SignatureMap, StalenessPlan}; use super::{BaseTable, NativeTable}; use crate::job::Job; use crate::{Error, Result}; @@ -110,28 +112,81 @@ async fn execute_refresh_column_with_source( }; let output_is_blob = field.is_blob_v2(); + // Which fragments the null filter cannot speak for: their inputs moved + // since they were computed, or the definition did. Decided once, from + // the manifest the values are read from. + let inputs = freshness::fields_for_paths(dataset.schema(), &bound.inputs)?; + let definition = freshness::definition_version(&expression); + let staleness = freshness::staleness_against(&dataset, column, &definition, &inputs).await?; + let mut rows_filled = 0u64; let mut replacements = Vec::new(); + // Fragments this refresh computed in full, signed at the version read. + let mut computed = SignatureMap::new(); for fragment in dataset.get_fragments() { - let gained = count_fragment_gains(&dataset, &fragment, &bound, column).await?; - if gained == 0 { - continue; + let fragment_id = u32::try_from(fragment.id()).map_err(|_| Error::Runtime { + message: format!("fragment id {} does not fit a signature map", fragment.id()), + })?; + // A recompute rewrites every live row, so it is staged without the + // probe and counted as it fills; a null fill probes first, since a + // fragment with nothing to gain is not worth a write. + let recompute = staleness.is_dirty(fragment_id); + let whole = recompute || { + let (gained, unfilled) = + count_fragment_gains(&dataset, &fragment, &bound, column).await?; + if gained == 0 { + continue; + } + rows_filled += gained; + unfilled == u64::try_from(fragment.count_rows(None).await?).unwrap_or(u64::MAX) + }; + if whole { + computed.insert( + fragment_id, + freshness::fragment_input_signature(fragment.metadata(), &inputs)?, + ); } - rows_filled += gained; - let values = - fill_stream(&dataset, &fragment, bound.clone(), column, output_is_blob).await?; + let gained = Arc::new(AtomicU64::new(0)); + let values = fill_stream( + &dataset, + &fragment, + bound.clone(), + column, + output_is_blob, + recompute, + gained.clone(), + ) + .await?; replacements.push(fragment.write_columns(values, &column_schema).await?); + if recompute { + rows_filled += gained.load(Ordering::Relaxed); + } } let source_version = dataset.version().version; if replacements.is_empty() { + // Nothing to fill; the stamp may still have something to record -- a + // column not yet enrolled, or fragments a compaction carried. + let mut latest = (*dataset).clone(); + let stamped = record( + &mut latest, + (&dataset, &staleness), + column, + &definition, + &inputs, + computed, + ) + .await; + if stamped.is_some() { + table.dataset.update(latest); + } return Ok(RefreshExecution { result: RefreshColumnResult { rows_filled: 0, - version: source_version, + version: stamped.unwrap_or(source_version), }, source_version, - published_version: None, + published_version: stamped, }); } @@ -149,7 +204,17 @@ async fn execute_refresh_column_with_source( ) .await?; - let version = new_dataset.version().version; + let mut new_dataset = new_dataset; + let version = record( + &mut new_dataset, + (&dataset, &staleness), + column, + &definition, + &inputs, + computed, + ) + .await + .unwrap_or(new_dataset.version().version); table.dataset.update(new_dataset); Ok(RefreshExecution { result: RefreshColumnResult { @@ -161,6 +226,31 @@ async fn execute_refresh_column_with_source( }) } +/// Stamp the input state the refresh computed from (see +/// [`freshness::record_freshness`]); the version the stamp landed at, which +/// is the last one the refresh wrote. Never fails the refresh: the values +/// are committed, and a missing stamp only costs a recompute next time. +async fn record( + latest: &mut Dataset, + pinned: (&Dataset, &StalenessPlan), + column: &str, + definition: &str, + inputs: &freshness::InputFields, + computed: SignatureMap, +) -> Option { + match freshness::record_freshness(latest, Some(pinned), column, definition, inputs, computed) + .await + { + Ok(record) => record.version, + Err(error) => { + log::warn!( + "could not record the input state computed column '{column}' was refreshed from ({error}); its fragments will recompute on the next refresh" + ); + None + } + } +} + /// Refuse while a computed input still has rows a refresh of it would fill: /// read now, its placeholder null would be evaluated as a value and kept. async fn ensure_inputs_filled( @@ -188,7 +278,9 @@ async fn ensure_inputs_filled( let input_bound = super::computed_columns::bind(schema.clone(), input, expression)?; let mut unfilled = 0u64; for fragment in dataset.get_fragments() { - unfilled += count_fragment_gains(dataset, &fragment, &input_bound, input).await?; + unfilled += count_fragment_gains(dataset, &fragment, &input_bound, input) + .await? + .0; } if unfilled > 0 { return Err(Error::InvalidInput { @@ -436,32 +528,35 @@ fn blob_array_from_binary( /// Scans only the unfilled live rows -- deleted rows never reach the /// expression here, the filter having already excluded them -- and counts the /// non-null results. Exact, so it is both the staging decision and the -/// fragment's contribution to `rows_filled`. +/// fragment's contribution to `rows_filled`. Returns the gains and the rows +/// scanned. async fn count_fragment_gains( dataset: &Dataset, fragment: &FileFragment, bound: &BoundExpression, column: &str, -) -> Result { +) -> Result<(u64, u64)> { let mut scanner = dataset.scan(); scanner .with_fragments(vec![fragment.metadata().clone()]) .with_row_id() - .filter(&format!("{} IS NULL", quote_identifier(column)))? - .project(&bound.roots)?; + .project(&bound.roots)? + .filter(&format!("{} IS NULL", quote_identifier(column)))?; configure_blob_inputs(&mut scanner, dataset.schema(), bound, None)?; let mut gained = 0u64; + let mut considered = 0u64; let mut batches = scanner.try_into_stream().await?; while let Some(batch) = batches.try_next().await? { let evaluated = evaluate(bound, &evaluation_batch(&batch, bound, None)?)?; gained += (batch.num_rows() - evaluated.null_count()) as u64; + considered += batch.num_rows() as u64; } - Ok(gained) + Ok((gained, considered)) } /// Stream one fragment's column in physical order, filling the unfilled live -/// rows and keeping every other value. +/// rows -- every live row, for a recompute -- and keeping every other value. /// /// Deleted rows are carried through so the values line up positionally with /// the fragment's data files; they are never read back, but the column file @@ -472,6 +567,8 @@ async fn fill_stream( bound: Arc, column: &str, output_is_blob: bool, + recompute: bool, + gained: Arc, ) -> Result> + Send + use<>> { let mut projection: Vec = bound.roots.clone(); projection.push(column.to_string()); @@ -521,14 +618,20 @@ async fn fill_stream( .column_by_name(ROW_ID) .ok_or_else(|| missing(ROW_ID))?; - // Only an unfilled live row gains a value; a deleted row has a null - // row id and keeps its (null) slot. - let unfilled = arrow::compute::is_null(existing.as_ref())?; + // Only an unfilled live row gains a value, or every live row under a + // recompute; a deleted row has a null row id and keeps its (null) slot. let live = arrow::compute::is_not_null(row_ids.as_ref())?; - let fill = arrow::compute::and(&unfilled, &live)?; + let fill = if recompute { + live + } else { + let unfilled = arrow::compute::is_null(existing.as_ref())?; + arrow::compute::and(&unfilled, &live)? + }; let keep = arrow::compute::not(&fill)?; let computed = evaluate(&bound, &evaluation_batch(&batch, &bound, Some(&keep))?)?; + let values = arrow::compute::and(&fill, &arrow::compute::is_not_null(&computed)?)?; + gained.fetch_add(values.true_count() as u64, Ordering::Relaxed); let merged = arrow_select::zip::zip(&fill, &computed, existing)?; let merged = if output_is_blob { blob_array_from_binary(&merged, projected.field(0))? @@ -737,7 +840,8 @@ mod tests { .await .unwrap(); assert_eq!(no_op.rows_assigned, 0); - assert_eq!(no_op.source_version, 3); + // The fill, then the stamp recording what it computed from. + assert_eq!(no_op.source_version, 4); assert_eq!(no_op.published_version, None); } @@ -777,8 +881,8 @@ mod tests { } /// A row is filled only by gaining a value, so an expression yielding null - /// settles at once instead of re-selecting the same rows forever. Nothing - /// is staged, so the version does not move either. + /// settles at once instead of re-selecting the same rows forever: the + /// second refresh finds the fragment signed and moves nothing. #[tokio::test] async fn test_refresh_converges_on_a_null_result() { let table = table_with("refresh_null_result", vec![1, 2, 3]).await; @@ -792,28 +896,186 @@ mod tests { let first = table.refresh_column("maybe").await.unwrap(); assert_eq!(first.rows_filled, 0); - assert_eq!(first.version, declared); + assert!(first.version > declared); assert_eq!(read(&table, "maybe").await, vec![None, None, None]); let again = table.refresh_column("maybe").await.unwrap(); assert_eq!(again.rows_filled, 0); - assert_eq!(again.version, declared); + assert_eq!(again.version, first.version); } - /// The contract's boundary: a filled fragment is not revisited, so - /// mutating an input leaves the value computed at fill time. + /// A filled row whose input moved is recomputed: the update rewrites + /// the row into a fragment the stamp never signed, and only that one. #[tokio::test] - async fn test_refresh_does_not_observe_input_mutation() { - let table = table_with("refresh_mutation", vec![1]).await; + async fn test_refresh_recomputes_a_row_whose_input_moved() { + let table = table_with("refresh_mutation", vec![1, 2]).await; + declare_doubled(&table).await.unwrap(); + append(&table, vec![5]).await; + table.refresh_column("doubled").await.unwrap(); + assert_eq!( + read(&table, "doubled").await, + vec![Some(2), Some(4), Some(10)] + ); + + table + .update() + .column("x", "7") + .only_if("x = 5") + .execute() + .await + .unwrap(); + + let again = table.refresh_column("doubled").await.unwrap(); + assert_eq!(again.rows_filled, 1); + assert_eq!( + read(&table, "doubled").await, + vec![Some(2), Some(4), Some(14)] + ); + + let settled = table.refresh_column("doubled").await.unwrap(); + assert_eq!(settled.rows_filled, 0); + assert_eq!(settled.version, again.version); + } + + /// Each stamp is a sidecar under `_computed/`; pruning old versions + /// removes the sidecars only they referenced, and keeps the current one. + #[tokio::test] + async fn test_pruning_drops_the_sidecars_of_pruned_versions() { + let dir = tempfile::tempdir().unwrap(); + let conn = connect(dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let batch = record_batch!(("x", Int32, [1, 2])).unwrap(); + let table = conn + .create_table("sidecars", batch) + .execute() + .await + .unwrap(); declare_doubled(&table).await.unwrap(); table.refresh_column("doubled").await.unwrap(); - assert_eq!(read(&table, "doubled").await, vec![Some(2)]); + append(&table, vec![5]).await; + table.refresh_column("doubled").await.unwrap(); + let sidecars = || { + std::fs::read_dir(dir.path().join("sidecars.lance").join("_computed")) + .unwrap() + .count() + }; + assert_eq!(sidecars(), 2); - table.update().column("x", "3").execute().await.unwrap(); + table + .optimize(crate::table::OptimizeAction::Prune { + older_than: Some(chrono::Duration::zero()), + delete_unverified: Some(true), + error_if_tagged_old_versions: None, + }) + .await + .unwrap(); + assert_eq!(sidecars(), 1); + assert_eq!( + table.refresh_column("doubled").await.unwrap().rows_filled, + 0 + ); + } + + /// A deleted row is never computed and the rows that stay keep their + /// values: a delete recomputes nothing and stamps nothing. + #[tokio::test] + async fn test_a_delete_recomputes_nothing() { + let table = table_with("refresh_delete", vec![1, 2, 3]).await; + declare_doubled(&table).await.unwrap(); + let filled = table.refresh_column("doubled").await.unwrap(); + + table.delete("x = 2").await.unwrap(); + let deleted = table.version().await.unwrap(); let again = table.refresh_column("doubled").await.unwrap(); assert_eq!(again.rows_filled, 0); - assert_eq!(read(&table, "doubled").await, vec![Some(2)]); + assert_eq!(again.version, deleted); + assert!(deleted > filled.version); + assert_eq!(read(&table, "doubled").await, vec![Some(2), Some(6)]); + } + + /// Compaction copies inputs unchanged, so a fragment it builds from + /// signed ones is fresh: the refresh recomputes nothing and only records + /// the new fragment. + #[tokio::test] + async fn test_a_compaction_of_signed_fragments_recomputes_nothing() { + let table = table_with("refresh_compact_signed", vec![1, 2]).await; + declare_doubled(&table).await.unwrap(); + append(&table, vec![5]).await; + table.refresh_column("doubled").await.unwrap(); + + table + .optimize(crate::table::OptimizeAction::Compact { + options: crate::table::CompactionOptions::default(), + remap_options: None, + }) + .await + .unwrap(); + let compacted = table.version().await.unwrap(); + + let carried = table.refresh_column("doubled").await.unwrap(); + assert_eq!(carried.rows_filled, 0); + assert_eq!(carried.version, compacted + 1); + let settled = table.refresh_column("doubled").await.unwrap(); + assert_eq!(settled.version, carried.version); + assert_eq!( + read(&table, "doubled").await, + vec![Some(2), Some(4), Some(10)] + ); + } + + /// A column declared before signatures existed has no map. Its first + /// refresh keeps the null-fill contract and enrolls what it read from; + /// from then on a moved input is recomputed like any other. + #[tokio::test] + async fn test_an_unsigned_column_is_enrolled_by_its_first_refresh() { + let table = table_with("refresh_legacy", vec![1, 2]).await; + declare_doubled(&table).await.unwrap(); + table.refresh_column("doubled").await.unwrap(); + table + .update() + .column("x", "3") + .only_if("x = 1") + .execute() + .await + .unwrap(); + + let native = table.as_native().unwrap(); + let mut dataset = (*native.dataset.get().await.unwrap()).clone(); + let declaration = dataset + .schema() + .field("doubled") + .unwrap() + .metadata + .iter() + .filter(|(key, _)| !key.starts_with("computed_refresh.")) + .map(|(key, value)| (key.clone(), value.clone())) + .collect::>(); + dataset + .update_field_metadata() + .replace("doubled", declaration) + .unwrap() + .await + .unwrap(); + table.checkout_latest().await.unwrap(); + + // Null-fill only: the moved row keeps the value it was filled with. + let enrolled = table.refresh_column("doubled").await.unwrap(); + assert_eq!(enrolled.rows_filled, 0); + assert_eq!(read(&table, "doubled").await, vec![Some(2), Some(4)]); + + table + .update() + .column("x", "5") + .only_if("x = 3") + .execute() + .await + .unwrap(); + let again = table.refresh_column("doubled").await.unwrap(); + assert_eq!(again.rows_filled, 1); + assert_eq!(read(&table, "doubled").await, vec![Some(4), Some(10)]); } /// A row rewrite before the first refresh materializes the declared @@ -831,11 +1093,11 @@ mod tests { assert_eq!(read(&table, "doubled").await, vec![Some(6)]); } - /// The contract holds row by row, not fragment by fragment: revisiting a - /// fragment to fill one row must not recompute a filled row sitting beside - /// it, even where the input behind it has since changed. + /// A fragment compacted out of one the stamp never signed cannot vouch + /// for any of its rows: every live row is recomputed, the moved one + /// included. #[tokio::test] - async fn test_refresh_does_not_recompute_a_filled_row_beside_an_unfilled_one() { + async fn test_a_compaction_of_an_unsigned_fragment_recomputes_it() { let table = table_with("refresh_mixed", vec![1, 2]).await; declare_doubled(&table).await.unwrap(); table.refresh_column("doubled").await.unwrap(); @@ -857,18 +1119,70 @@ mod tests { .unwrap(); let result = table.refresh_column("doubled").await.unwrap(); - assert_eq!(result.rows_filled, 1); - // 2 is the mutated row keeping the value it was filled with, not 200. + assert_eq!(result.rows_filled, 3); + assert_eq!( + read(&table, "doubled").await, + vec![Some(4), Some(10), Some(200)] + ); + } + + /// The gate's reproducer: a raw lance append may carry a value for the + /// computed column. Compaction cannot certify it, so the product is + /// recomputed and the supplied value replaced. + #[tokio::test] + async fn test_raw_append_values_are_not_trusted_after_compaction() { + use arrow_array::RecordBatchIterator; + use lance::Dataset; + use lance::dataset::{WriteMode, WriteParams}; + + let dir = tempfile::tempdir().unwrap(); + let conn = connect(dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let batch = record_batch!(("x", Int32, [1, 2])).unwrap(); + let table = conn + .create_table("raw_append", batch) + .execute() + .await + .unwrap(); + declare_doubled(&table).await.unwrap(); + table.refresh_column("doubled").await.unwrap(); + + let batch = record_batch!(("x", Int32, [5]), ("doubled", Int32, [Some(999_i32)])).unwrap(); + let schema = batch.schema(); + let uri = table.uri().await.unwrap(); + Dataset::write( + RecordBatchIterator::new(vec![Ok(batch)], schema), + &uri, + Some(WriteParams { + mode: WriteMode::Append, + ..Default::default() + }), + ) + .await + .unwrap(); + table.checkout_latest().await.unwrap(); + table + .optimize(crate::table::OptimizeAction::Compact { + options: crate::table::CompactionOptions::default(), + remap_options: None, + }) + .await + .unwrap(); + + let result = table.refresh_column("doubled").await.unwrap(); + assert_eq!(result.rows_filled, 3); assert_eq!( read(&table, "doubled").await, vec![Some(2), Some(4), Some(10)] ); } - /// Filling a fragment must not disturb the values it already holds, which - /// is what makes a compaction-mixed fragment safe to revisit. + /// An appended fragment holds no values, so compacting it into a signed + /// one leaves the product fresh: only the appended rows are filled. #[tokio::test] - async fn test_refresh_preserves_already_filled_rows() { + async fn test_a_compaction_with_an_appended_fragment_fills_only_its_rows() { let table = table_with("refresh_preserves", vec![1, 2]).await; declare_doubled(&table).await.unwrap(); table.refresh_column("doubled").await.unwrap(); @@ -995,7 +1309,8 @@ mod tests { assert_eq!(result.rows_failed, 0); assert_eq!(result.rows_remaining, 0); assert_eq!(result.source_version, 2); - assert_eq!(result.published_version, Some(3)); + // The fill lands at 3; the stamp recording its inputs is published at 4. + assert_eq!(result.published_version, Some(4)); assert_eq!(job.status().await.unwrap(), "finished"); assert_eq!( read(&table, "doubled").await, @@ -1059,31 +1374,32 @@ mod tests { assert_eq!(read(&table, "quotient").await, vec![Some(10)]); } - /// The gate's reproducer: an already-filled row's value must not be - /// re-evaluated either -- its input may have mutated into one the - /// expression chokes on. + /// A filled row whose input moved is re-evaluated, and a row whose + /// input did not move is not: the untouched fragment is never read, so + /// its poison input is never reached. #[tokio::test] - async fn test_a_filled_rows_value_is_never_evaluated() { - let table = table_with("refresh_filled_poison", vec![1, 2]).await; + async fn test_only_a_moved_rows_value_is_re_evaluated() { + let table = table_with("refresh_filled_poison", vec![1, 0]).await; table .add_columns() - .computed("quotient", "10 / x") + .computed("quotient", "10 / coalesce(nullif(x, 0), 1)") .execute() .await .unwrap(); table.refresh_column("quotient").await.unwrap(); + assert_eq!(read(&table, "quotient").await, vec![Some(10), Some(10)]); + append(&table, vec![5]).await; table .update() - .column("x", "0") + .column("x", "2") .only_if("x = 1") .execute() .await .unwrap(); - append(&table, vec![5]).await; let result = table.refresh_column("quotient").await.unwrap(); - assert_eq!(result.rows_filled, 1); + assert_eq!(result.rows_filled, 2); assert_eq!( read(&table, "quotient").await, vec![Some(2), Some(5), Some(10)] From 7575c2597af5f62db5168ea2f8eb8f5088af03be Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Mon, 14 Sep 2026 09:41:37 -0700 Subject: [PATCH 04/43] feat: recompute computed column rows whose inputs changed (#4161) refresh_column fills nulls, so once a row has a value nothing revisits it: an update to one of its inputs, or a definition change, leaves the computed value stale for good. This stamps the column's field metadata with the definition it was computed under and a per-fragment signature of the input storage it was read from (input data files and overlays; not the deletion file, since a delete changes no surviving value). A refresh recomputes every live row of a fragment whose stamp disagrees with the manifest, then records what it computed from in a second commit after the fill. A compacted fragment inherits freshness through the Rewrite lineage when every fragment it was built from was signed, or was appended since the stamp, never had an input moved, and left its rows of the product unfilled (a raw append may supply a value; the product's data is the evidence, and the null fill covers those rows); otherwise it recomputes. A column declared before the stamps existed keeps the null-fill contract on its first refresh, which enrolls it as it stood. The map is one entry per fragment per column, so it is kept out of the manifest: each stamp writes an immutable sidecar under `_computed/`, named by its content digest, and the field metadata holds the digest. Pruning old versions also drops the sidecars no remaining version references, keeping any younger than seven days as lance keeps unverified files, since a sidecar is put before the commit that references it. The stamp commit is metadata-only, so a materialized view's drift check treats it like the fill. The core lives in `table::freshness` so a remote refresh can share the contract. From c44b1923349c6355e6af02fadf03d99a7cf9bbc6 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Mon, 14 Sep 2026 16:46:21 +0000 Subject: [PATCH 05/43] =?UTF-8?q?Bump=20version:=200.39.0-beta.7=20?= =?UTF-8?q?=E2=86=92=200.39.0-beta.8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index a2c5fdd61..23f097af5 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.39.0-beta.7" +current_version = "0.39.0-beta.8" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index efa59c4e3..b3ff0e299 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5452,7 +5452,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.39.0-beta.7" +version = "0.39.0-beta.8" dependencies = [ "ahash", "anyhow", @@ -5544,7 +5544,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.39.0-beta.7" +version = "0.39.0-beta.8" dependencies = [ "arrow-array", "arrow-buffer", @@ -5569,7 +5569,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.39.0-beta.7" +version = "0.39.0-beta.8" dependencies = [ "arrow", "async-trait", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 59c995344..1b5f14153 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.39.0-beta.7 + 0.39.0-beta.8 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index afe9b8c5c..5f9702aaa 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.39.0-beta.7 + 0.39.0-beta.8 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index e66e1bbd4..37c182feb 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.39.0-beta.7 + 0.39.0-beta.8 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 340d8e500..53fc32c6a 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.39.0-beta.7" +version = "0.39.0-beta.8" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index e1ea88e62..7e8684be8 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.39.0-beta.7", + "version": "0.39.0-beta.8", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 7bbbd3221..274875fec 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.39.0-beta.7", + "version": "0.39.0-beta.8", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 58c488e47..c01cac42f 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.39.0-beta.7", + "version": "0.39.0-beta.8", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index ca364168e..1398953d4 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.39.0-beta.7", + "version": "0.39.0-beta.8", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index c088a4d7d..9823eda62 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.39.0-beta.7", + "version": "0.39.0-beta.8", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index af28f68d3..ce6d2922f 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.39.0-beta.7", + "version": "0.39.0-beta.8", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index ff612c4df..8f6a919b2 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.39.0-beta.7", + "version": "0.39.0-beta.8", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index e9621ba60..db979c1e7 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.39.0-beta.7", + "version": "0.39.0-beta.8", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index ff7a98234..d7ec8cd61 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.39.0-beta.7" +version = "0.39.0-beta.8" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 998d041e2..f9c79b042 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.39.0-beta.7" +version = "0.39.0-beta.8" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 37771fd4fcbc969485426c6b79ac3c3c6cd53356 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Mon, 14 Sep 2026 20:04:15 -0700 Subject: [PATCH 06/43] fix(deps): update rustls and cap aws-smithy-types to unbreak CI (#4177) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two upstream dependency releases broke CI on `main`. Both fixes are dependency constraints, so they ride together. ## `deny` — RUSTSEC-2026-0285 rustls 0.23.40 accepts TLS 1.3 handshake messages sent at the wrong encryption level ([advisory](https://rustsec.org/advisories/RUSTSEC-2026-0285)), patched in 0.23.45. rustls 0.23.45 requires `aws-lc-rs >= 1.18`, which the nodejs crate pinned to `=1.16.3`, so this also bumps that pin and its `aws-lc-sys` companion to `=1.18.1` / `=0.45.0`. The pin comment already calls for periodic updates on security patches. The workspace's other rustls (0.21.12) is below the advisory's affected range (`unaffected = ["< 0.23.13"]`). ## `build-no-lock` — aws-smithy-types 1.7.0 `aws-smithy-types` 1.7.0 and `aws-smithy-json` 0.64.0 both released 2026-09-14. 1.7.0 made `Document` `non_exhaustive`, which `aws-smithy-json` 0.63 does not compile against: ``` error[E0004]: non-exhaustive patterns: `&_` not covered --> aws-smithy-json-0.63.0/src/serialize.rs:36:15 note: `aws_smithy_types::Document` defined here --> aws-smithy-types-1.7.0/src/document/mod.rs:91:1 ``` Every `aws-sdk-*` crate moved to `aws-smithy-json ^0.64`, but `aws-config` 1.12.0 still requires `^0.63`, so a lockfile-free resolve pairs json 0.63.0 with types 1.7.0 and fails. This caps `aws-smithy-types` below 1.7 as a constraint-only dev-dependency, matching the existing `aws-smithy-runtime` entry. Revert once `aws-config` moves to `aws-smithy-json` 0.64. Note this break is not specific to this PR — `build-no-lock` fails the same way on unrelated branches (e.g. `jon/secrets-client-api` run 34903587364), which passed it hours earlier. ## Verification Resolution only, no local build: - Locked resolve unchanged: `aws-smithy-types` stays 1.4.8; the only `Cargo.lock` delta from the cap is the new dev-dep edge. - Fresh resolve (`rm Cargo.lock`): `aws-smithy-json` 0.63.0 with `aws-smithy-types` 1.6.3, `aws-sdk-*` one release back, `rustls` 0.23.45 retained. --- Cargo.lock | 72 +++++++++++++++++++++-------------------- nodejs/Cargo.toml | 5 +-- rust/lancedb/Cargo.toml | 4 +++ 3 files changed, 44 insertions(+), 37 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b3ff0e299..160ee2253 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -141,7 +141,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -152,7 +152,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -662,9 +662,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.16.3" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -673,14 +673,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.40.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -1008,7 +1009,7 @@ dependencies = [ "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.40", + "rustls 0.23.45", "rustls-native-certs", "rustls-pki-types", "tokio", @@ -1822,7 +1823,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3084,7 +3085,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3292,7 +3293,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -4262,7 +4263,7 @@ dependencies = [ "http 1.5.0", "hyper 1.9.0", "hyper-util", - "rustls 0.23.40", + "rustls 0.23.45", "rustls-native-certs", "tokio", "tokio-rustls 0.26.4", @@ -4300,7 +4301,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.5.10", + "socket2 0.6.3", "system-configuration", "tokio", "tower-service", @@ -4599,7 +4600,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5474,6 +5475,7 @@ dependencies = [ "aws-sdk-kms", "aws-sdk-s3", "aws-smithy-runtime", + "aws-smithy-types", "bytes", "candle-core", "candle-nn", @@ -6305,7 +6307,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7969,8 +7971,8 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.40", - "socket2 0.5.10", + "rustls 0.23.45", + "socket2 0.6.3", "thiserror 2.0.18", "tokio", "tracing", @@ -7990,7 +7992,7 @@ dependencies = [ "rand 0.9.5", "ring", "rustc-hash", - "rustls 0.23.40", + "rustls 0.23.45", "rustls-pki-types", "slab", "thiserror 2.0.18", @@ -8008,7 +8010,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.3", "tracing", "windows-sys 0.60.2", ] @@ -8533,7 +8535,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.40", + "rustls 0.23.45", "rustls-native-certs", "rustls-pki-types", "serde", @@ -8576,7 +8578,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.40", + "rustls 0.23.45", "rustls-pki-types", "rustls-platform-verifier", "serde", @@ -8791,7 +8793,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -8808,16 +8810,16 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "log", "once_cell", "ring", "rustls-pki-types", - "rustls-webpki 0.103.13", + "rustls-webpki 0.103.15", "subtle", "zeroize", ] @@ -8855,14 +8857,14 @@ dependencies = [ "jni", "log", "once_cell", - "rustls 0.23.40", + "rustls 0.23.45", "rustls-native-certs", "rustls-platform-verifier-android", - "rustls-webpki 0.103.13", + "rustls-webpki 0.103.15", "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -8883,9 +8885,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", @@ -9444,7 +9446,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -9551,7 +9553,7 @@ dependencies = [ "cfg-if 1.0.4", "libc", "psm", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -9843,7 +9845,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -10099,7 +10101,7 @@ version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls 0.23.40", + "rustls 0.23.45", "tokio", ] @@ -10513,7 +10515,7 @@ dependencies = [ "flate2", "log", "once_cell", - "rustls 0.23.40", + "rustls 0.23.45", "rustls-pki-types", "serde", "serde_json", @@ -10847,7 +10849,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 53fc32c6a..636289907 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -37,8 +37,9 @@ lzma-sys = { version = "0.1", features = ["static"] } log.workspace = true # Pin to resolve build failures; update periodically for security patches. -aws-lc-sys = "=0.40.0" -aws-lc-rs = "=1.16.3" +# rustls >= 0.23.45 (RUSTSEC-2026-0285) needs aws-lc-rs >= 1.18. +aws-lc-sys = "=0.45.0" +aws-lc-rs = "=1.18.1" [build-dependencies] napi-build = "2.3.1" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index f9c79b042..9bc4c4a8c 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -111,6 +111,10 @@ aws-sdk-s3 = { version = "1.55.0" } aws-sdk-kms = { version = "1.48.0" } aws-config = { version = "1.5.10" } aws-smithy-runtime = { version = "1.9.1" } +# Constraint only: types 1.7 breaks aws-smithy-json 0.63, which aws-config still +# requires. Bounds must stay inside 1.x and allow the MSRV job's 1.3.6 pin. +# Drop once aws-config moves to aws-smithy-json 0.64. +aws-smithy-types = { version = ">=1.0, <1.7" } datafusion.workspace = true http-body = "1" # Matching reqwest rstest = "0.23.0" From ffe94a65a1881f0dbcf4ff84ab26d05883009279 Mon Sep 17 00:00:00 2001 From: "lancedb-gatefixer[bot]" <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 14:10:48 +0800 Subject: [PATCH 07/43] fix(node): preserve optimize cleanup timestamp (#4160) ## Summary - pass the TypeScript `cleanupOlderThan` date to the native binding as an unchanged epoch timestamp - prune with Lance's absolute `before_timestamp` policy so dispatch and compaction time cannot move the cutoff - retain versions created after the supplied cutoff and document that behavior - add boundary and end-to-end regression coverage ## Root cause The TypeScript layer converted the absolute date into an elapsed duration before calling native optimize. Lance converted that duration back into a timestamp only after compaction, which silently advanced the requested cutoff and made the cleanup count depend on a millisecond timing boundary. ## Validation - `cargo fmt --all` - `cargo clippy --quiet --features remote --tests --examples -p lancedb -p lancedb-nodejs` - `pnpm build` - `pnpm lint` - `pnpm run docs` - `pnpm test __test__/table.test.ts --runInBand` (309 passed) Fixes #4159 --------- Co-authored-by: Gatefixer <313497061+lancedb-gatefixer[bot]@users.noreply.github.com> Co-authored-by: Xuanwo --- docs/src/js/interfaces/OptimizeOptions.md | 3 +- nodejs/__test__/table.test.ts | 21 +++++++++- nodejs/lancedb/table.ts | 13 ++---- nodejs/src/table.rs | 50 ++++++++++++----------- rust/lancedb/src/table.rs | 27 ++++++++++++ rust/lancedb/src/table/optimize.rs | 30 +++++++++++++- rust/lancedb/src/table/refresh.rs | 37 +++++++++++++++++ 7 files changed, 145 insertions(+), 36 deletions(-) diff --git a/docs/src/js/interfaces/OptimizeOptions.md b/docs/src/js/interfaces/OptimizeOptions.md index 700632342..110eb3813 100644 --- a/docs/src/js/interfaces/OptimizeOptions.md +++ b/docs/src/js/interfaces/OptimizeOptions.md @@ -26,7 +26,8 @@ const olderThan = new Date(); olderThan.setDate(olderThan.getDate() - 1)); tbl.optimize({cleanupOlderThan: olderThan}); -// Delete all versions except the current version +// Delete versions committed before this point. Versions created by the +// optimize call itself are newer than the cutoff and will be retained. tbl.optimize({cleanupOlderThan: new Date()}); ``` diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index cae01d9d5..ac74c65ef 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -53,6 +53,7 @@ import { Operator, instanceOfFullTextQuery, } from "../lancedb/query"; +import { LocalTable } from "../lancedb/table"; describe.each([arrow15, arrow16, arrow17, arrow18])( "Given a table", @@ -2789,7 +2790,7 @@ describe("when optimizing a dataset", () => { it("cleanups old versions", async () => { const stats = await table.optimize({ cleanupOlderThan: new Date() }); expect(stats.prune.bytesRemoved).toBeGreaterThan(0); - expect(stats.prune.oldVersionsRemoved).toBe(3); + expect(stats.prune.oldVersionsRemoved).toBe(2); }); it("delete unverified", async () => { @@ -2810,6 +2811,24 @@ describe("when optimizing a dataset", () => { }); }); +it("passes cleanupOlderThan to the native binding as an absolute timestamp", async () => { + const optimize = jest.fn().mockResolvedValue({ + compaction: { + filesAdded: 0, + filesRemoved: 0, + fragmentsAdded: 0, + fragmentsRemoved: 0, + }, + prune: { bytesRemoved: 0, oldVersionsRemoved: 0 }, + }); + const table = new LocalTable({ optimize } as never); + const cutoff = new Date("2020-01-02T03:04:05.678Z"); + + await table.optimize({ cleanupOlderThan: cutoff, deleteUnverified: true }); + + expect(optimize).toHaveBeenCalledWith(cutoff.getTime(), true); +}); + describe.each([arrow15, arrow16, arrow17, arrow18])( "when optimizing a dataset", // biome-ignore lint/suspicious/noExplicitAny: diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 8d8b0d675..a70243402 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -148,7 +148,8 @@ export interface OptimizeOptions { * olderThan.setDate(olderThan.getDate() - 1)); * tbl.optimize({cleanupOlderThan: olderThan}); * - * // Delete all versions except the current version + * // Delete versions committed before this point. Versions created by the + * // optimize call itself are newer than the cutoff and will be retained. * tbl.optimize({cleanupOlderThan: new Date()}); */ cleanupOlderThan: Date; @@ -1486,16 +1487,8 @@ export class LocalTable extends Table { } async optimize(options?: Partial): Promise { - let cleanupOlderThanMs; - if ( - options?.cleanupOlderThan !== undefined && - options?.cleanupOlderThan !== null - ) { - cleanupOlderThanMs = - new Date().getTime() - options.cleanupOlderThan.getTime(); - } return await this.inner.optimize( - cleanupOlderThanMs, + options?.cleanupOlderThan?.getTime(), options?.deleteUnverified, ); } diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 7ae4402ab..cf7ec4020 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -7,7 +7,7 @@ use chrono::{DateTime, Utc}; use lancedb::ipc::{ipc_file_to_batches, ipc_file_to_schema}; use lancedb::table::{ - AddDataMode, ColumnAlteration as LanceColumnAlteration, Duration, + AddDataMode, ColumnAlteration as LanceColumnAlteration, FieldMetadataUpdate as LanceFieldMetadataUpdate, FtsToken as LanceDbFtsToken, NewColumnTransform, OptimizeAction, OptimizeOptions, Ref, Table as LanceDbTable, }; @@ -677,22 +677,20 @@ impl Table { #[napi(catch_unwind)] pub async fn optimize( &self, - older_than_ms: Option, + before_timestamp_ms: Option, delete_unverified: Option, ) -> napi::Result { let inner = self.inner_ref()?; - let older_than = if let Some(ms) = older_than_ms { - if ms == i64::MIN { - return Err(napi::Error::from_reason(format!( - "older_than_ms can not be {}", - i32::MIN, - ))); - } - Duration::try_milliseconds(ms) - } else { - None - }; + let before_timestamp = before_timestamp_ms + .map(|ms| { + DateTime::from_timestamp_millis(ms).ok_or_else(|| { + napi::Error::from_reason(format!( + "cleanupOlderThan timestamp is out of range: {ms}" + )) + }) + }) + .transpose()?; let compaction_stats = inner .optimize(OptimizeAction::Compact { @@ -703,16 +701,22 @@ impl Table { .default_error()? .compaction .unwrap(); - let prune_stats = inner - .optimize(OptimizeAction::Prune { - older_than, - delete_unverified, - error_if_tagged_old_versions: None, - }) - .await - .default_error()? - .prune - .unwrap(); + let prune_stats = if let Some(before_timestamp) = before_timestamp { + inner + .optimize_prune_before(before_timestamp, delete_unverified, None) + .await + } else { + inner + .optimize(OptimizeAction::Prune { + older_than: None, + delete_unverified, + error_if_tagged_old_versions: None, + }) + .await + } + .default_error()? + .prune + .unwrap(); inner .optimize(lancedb::table::OptimizeAction::Index( OptimizeOptions::default(), diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 508c2ca49..5732955f5 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -1741,6 +1741,33 @@ impl Table { self.inner.optimize(action).await } + /// Prune versions committed before an absolute timestamp. + /// + /// This is an internal entry point for language bindings whose public API + /// accepts an absolute cleanup cutoff. + #[doc(hidden)] + pub async fn optimize_prune_before( + &self, + before_timestamp: chrono::DateTime, + delete_unverified: Option, + error_if_tagged_old_versions: Option, + ) -> Result { + let native = self.as_native().ok_or_else(|| Error::NotSupported { + message: "optimize is not supported on LanceDB cloud.".into(), + })?; + let prune = optimize::cleanup_old_versions_before( + native, + before_timestamp, + delete_unverified, + error_if_tagged_old_versions, + ) + .await?; + Ok(OptimizeStats { + compaction: None, + prune: Some(prune), + }) + } + /// Add new columns to the table, providing values to fill in. pub fn add_columns(&self) -> AddColumnsBuilder { AddColumnsBuilder::new(self.inner.clone()) diff --git a/rust/lancedb/src/table/optimize.rs b/rust/lancedb/src/table/optimize.rs index 6e3fc0048..2a502b059 100644 --- a/rust/lancedb/src/table/optimize.rs +++ b/rust/lancedb/src/table/optimize.rs @@ -8,7 +8,8 @@ use std::sync::Arc; -use lance::dataset::cleanup::RemovalStats; +use chrono::{DateTime, Utc}; +use lance::dataset::cleanup::{CleanupPolicyBuilder, RemovalStats}; use lance::dataset::optimize::{CompactionMetrics, IndexRemapperOptions, compact_files}; use lance::index::DatasetIndexExt; use lance_index::optimize::OptimizeOptions; @@ -147,6 +148,33 @@ pub(crate) async fn cleanup_old_versions( Ok(stats) } +/// Remove dataset versions committed before an absolute timestamp. +pub(crate) async fn cleanup_old_versions_before( + table: &NativeTable, + before_timestamp: DateTime, + delete_unverified: Option, + error_if_tagged_old_versions: Option, +) -> Result { + table.dataset.ensure_mutable()?; + let dataset = table.dataset.get().await?; + let mut policy = CleanupPolicyBuilder::default().before_timestamp(before_timestamp); + if let Some(delete_unverified) = delete_unverified { + policy = policy.delete_unverified(delete_unverified); + } + if let Some(error_if_tagged_old_versions) = error_if_tagged_old_versions { + policy = policy.error_if_tagged_old_versions(error_if_tagged_old_versions); + } + let stats = dataset.cleanup_with_policy(policy.build()).await?; + // Computed-column signature sidecars live outside lance's directories; + // drop the ones the surviving versions no longer reference. + let removed = + super::freshness::prune_sidecars(&dataset, delete_unverified.unwrap_or(false)).await?; + if removed > 0 { + log::debug!("removed {removed} unreferenced computed-column signature sidecars"); + } + Ok(stats) +} + /// Compact files in the dataset. /// /// This can be run after making several small appends to optimize the table diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index 2ed479256..996f2e383 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -978,6 +978,43 @@ mod tests { ); } + /// Absolute timestamp pruning applies the same computed-column sidecar + /// cleanup as duration-based pruning. + #[tokio::test] + async fn test_absolute_pruning_drops_the_sidecars_of_pruned_versions() { + let dir = tempfile::tempdir().unwrap(); + let conn = connect(dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let batch = record_batch!(("x", Int32, [1, 2])).unwrap(); + let table = conn + .create_table("sidecars", batch) + .execute() + .await + .unwrap(); + declare_doubled(&table).await.unwrap(); + table.refresh_column("doubled").await.unwrap(); + append(&table, vec![5]).await; + table.refresh_column("doubled").await.unwrap(); + let sidecars = || { + std::fs::read_dir(dir.path().join("sidecars.lance").join("_computed")) + .unwrap() + .count() + }; + assert_eq!(sidecars(), 2); + + table + .optimize_prune_before(chrono::Utc::now(), Some(true), None) + .await + .unwrap(); + assert_eq!(sidecars(), 1); + assert_eq!( + table.refresh_column("doubled").await.unwrap().rows_filled, + 0 + ); + } + /// A deleted row is never computed and the rows that stay keep their /// values: a delete recomputes nothing and stamps nothing. #[tokio::test] From 56bbd19ff4efe2b1a05a10ecaa0f27ff39616e29 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Tue, 15 Sep 2026 02:10:49 -0700 Subject: [PATCH 08/43] test(python): cover namespace operations without pylance (#4174) The "Test without pylance or pandas" CI job only ran `test_table.py`, so the regression fixed in #3606 had no guard: sync namespace operations used to route through the Python `lance_namespace` client, whose `dir` implementation ships in the optional `pylance` extra, so `lancedb.connect(path).list_namespaces()` failed with `No module named 'lance'` while the async API worked. Adds `python/python/tests/test_namespace_no_pylance.py` and runs it in that job. Its `without_pylance` fixture blocks `lance` imports, so the guard also fires in environments that do have `pylance` installed. Coverage: the original reproducer, nested namespace lifecycle, namespaced and root table lifecycle, the async path, and the one API that legitimately still needs `pylance` (`namespace_client()`). Verified the guard actually catches the regression: against `lancedb==0.33.0`, three of these tests fail with the original error; against a fixed build all pass. Co-authored-by: Xuanwo --- .github/workflows/python.yml | 2 +- .../python/tests/test_namespace_no_pylance.py | 149 ++++++++++++++++++ 2 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 python/python/tests/test_namespace_no_pylance.py diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index db8919ddc..3594bd826 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -156,7 +156,7 @@ jobs: - name: Test without pylance or pandas run: | pip uninstall -y pylance pandas - pytest -vv python/tests/test_table.py + pytest -vv python/tests/test_table.py python/tests/test_namespace_no_pylance.py # Make sure wheels are not included in the Rust cache - name: Delete wheels run: rm -rf target/wheels diff --git a/python/python/tests/test_namespace_no_pylance.py b/python/python/tests/test_namespace_no_pylance.py new file mode 100644 index 000000000..7f5b086d6 --- /dev/null +++ b/python/python/tests/test_namespace_no_pylance.py @@ -0,0 +1,149 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""Namespace operations must not require the optional ``pylance`` dependency. + +The sync ``lancedb.connect()`` connection used to route namespace operations +through the Python ``lance_namespace`` client, whose ``dir`` implementation +lives in ``lance.namespace`` (shipped by the optional ``pylance`` extra). On an +install without that extra, even ``db.list_namespaces()`` failed, while the +async API worked because it goes straight to the native Rust connection. + +The ``without_pylance`` fixture below simulates a missing ``pylance`` so these +tests fail on any environment when the native routing regresses. The ground +truth remains the "Test without pylance or pandas" CI job, which runs this file +with ``pylance`` and ``pandas`` actually uninstalled -- so nothing here may +import either at module scope. +""" + +import sys +from importlib import import_module +from importlib.abc import MetaPathFinder + +import lancedb +import pyarrow as pa +import pytest + + +def _is_lance(module_name: str) -> bool: + # "lance_namespace" is a separate, non-optional package -- leave it alone. + return module_name == "lance" or module_name.startswith("lance.") + + +class _BlockLanceImports(MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if _is_lance(fullname): + raise ModuleNotFoundError(f"No module named {fullname!r}", name=fullname) + return None + + +@pytest.fixture +def without_pylance(monkeypatch): + """Make ``lance`` unimportable, as on an install without the pylance extra. + + Uninstalled means absent from ``sys.modules`` too, not merely unimportable: + LanceDB has code that branches on ``"lance" in sys.modules``, so a fixture + that poisons the entry instead of removing it would fail tests that a real + install passes. + """ + for name in list(sys.modules): + if _is_lance(name): + monkeypatch.delitem(sys.modules, name) + monkeypatch.setattr(sys, "meta_path", [_BlockLanceImports(), *sys.meta_path]) + + +def _schema() -> pa.Schema: + return pa.schema([pa.field("id", pa.int64())]) + + +def test_fixture_matches_an_uninstalled_pylance(without_pylance): + """Guard the guard: the other tests are meaningless if lance stays importable.""" + assert "lance" not in sys.modules + with pytest.raises(ModuleNotFoundError): + import_module("lance.namespace") + + +def test_list_namespaces_on_sync_connection(tmp_path, without_pylance): + """The original reproducer: this alone used to raise on a sync connection.""" + db = lancedb.connect(tmp_path) + assert db.list_namespaces().namespaces == [] + + +def test_sync_namespace_lifecycle(tmp_path, without_pylance): + db = lancedb.connect(tmp_path) + + db.create_namespace(["child"]) + assert db.list_namespaces().namespaces == ["child"] + assert db.list_namespaces(namespace_path=["child"]).namespaces == [] + db.describe_namespace(["child"]) + + db.create_namespace(["child", "grandchild"]) + assert db.list_namespaces(namespace_path=["child"]).namespaces == ["grandchild"] + + db.drop_namespace(["child", "grandchild"]) + db.drop_namespace(["child"]) + assert db.list_namespaces().namespaces == [] + + +def test_sync_namespaced_table_lifecycle(tmp_path, without_pylance): + db = lancedb.connect(tmp_path) + db.create_namespace(["child"]) + + table = db.create_table("tbl", schema=_schema(), namespace_path=["child"]) + assert table.namespace == ["child"] + table.add([{"id": 1}]) + + assert db.list_tables(namespace_path=["child"]).tables == ["tbl"] + assert db.list_tables().tables == [] + + opened = db.open_table("tbl", namespace_path=["child"]) + assert opened.namespace == ["child"] + assert opened.count_rows() == 1 + assert opened.search().limit(5).to_arrow().num_rows == 1 + + db.drop_table("tbl", namespace_path=["child"]) + assert db.list_tables(namespace_path=["child"]).tables == [] + db.drop_namespace(["child"]) + + +def test_sync_root_table_lifecycle(tmp_path, without_pylance): + """Root-namespace tables share the namespace plumbing, so cover them too.""" + db = lancedb.connect(tmp_path) + + table = db.create_table("tbl", schema=_schema()) + table.add([{"id": 1}]) + + assert db.table_names() == ["tbl"] + assert "tbl" in db + assert db["tbl"].count_rows() == 1 + + db.drop_table("tbl") + assert db.table_names() == [] + + +@pytest.mark.asyncio +async def test_async_namespace_lifecycle(tmp_path, without_pylance): + db = await lancedb.connect_async(tmp_path) + + await db.create_namespace(["child"]) + assert (await db.list_namespaces()).namespaces == ["child"] + + table = await db.create_table("tbl", schema=_schema(), namespace_path=["child"]) + await table.add([{"id": 1}]) + assert (await db.list_tables(namespace_path=["child"])).tables == ["tbl"] + assert await table.count_rows() == 1 + + await db.drop_table("tbl", namespace_path=["child"]) + await db.drop_namespace(["child"]) + assert (await db.list_namespaces()).namespaces == [] + + +def test_namespace_client_still_requires_pylance(tmp_path, without_pylance): + """``namespace_client()`` is the one namespace API that opts into pylance. + + It hands out a Python ``LanceNamespace``, so it cannot be served natively. + Pinned here so the boundary stays explicit and the error stays actionable. + """ + db = lancedb.connect(tmp_path) + with pytest.raises(ValueError, match="lance.namespace.DirectoryNamespace"): + db.namespace_client() From 21ecafe9ae883953ee7da2c7aeeb6248de0d764d Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Tue, 15 Sep 2026 04:13:44 -0700 Subject: [PATCH 09/43] chore: update lance dependency to v13.0.0-beta.1 (#4183) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update the Rust workspace Lance dependencies and Java lance-core from v12.0.0-beta.18 to [v13.0.0-beta.1](https://github.com/lance-format/lance/releases/tag/v13.0.0-beta.1), and refresh Cargo.lock; no compatibility fixes were required. Validation passed: `cargo clippy --quiet --workspace --tests --all-features -- -D warnings`, `cargo fmt --all --quiet`, and `git diff --check`. Also fixes the flaky Node test `when optimizing a dataset › cleanups old versions` that failed the NPM Publish Linux test jobs on this PR. The test captured `new Date()` (millisecond precision) in the same millisecond as the last commit, while Lance compares version timestamps at nanosecond precision, so that version was not pruned. The test now waits for the clock to tick to the next millisecond before taking the cutoff. This is a pre-existing flake on `main` since #4160, unrelated to the Lance upgrade. --------- Co-authored-by: Yang Cen Co-authored-by: Claude Fable 5.1 --- Cargo.lock | 85 ++++++++++++++++++----------------- Cargo.toml | 28 ++++++------ java/pom.xml | 2 +- nodejs/__test__/table.test.ts | 16 ++++++- 4 files changed, 73 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 160ee2253..6dad3d6b6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3491,8 +3491,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "rand 0.9.5", @@ -4863,8 +4863,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arc-swap", "arrow", @@ -4936,8 +4936,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "arrow-buffer", @@ -4959,7 +4959,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "arrow-buffer", @@ -4973,7 +4973,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "arrow-schema", @@ -4982,8 +4982,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrayref", "crunchy", @@ -4993,8 +4993,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "arrow-buffer", @@ -5031,8 +5031,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow", "arrow-array", @@ -5062,8 +5062,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow", "arrow-array", @@ -5080,8 +5080,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "proc-macro2", "quote", @@ -5090,8 +5090,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-arith", "arrow-array", @@ -5124,8 +5124,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-arith", "arrow-array", @@ -5150,14 +5150,15 @@ dependencies = [ "prost", "prost-build", "prost-types", + "serde", "tokio", "tracing", ] [[package]] name = "lance-index" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arc-swap", "arrow", @@ -5221,8 +5222,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "arrow-schema", @@ -5244,8 +5245,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow", "arrow-array", @@ -5285,8 +5286,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "arrow-schema", @@ -5300,8 +5301,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow", "async-trait", @@ -5315,8 +5316,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow", "arrow-ipc", @@ -5369,8 +5370,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "arrow-buffer", @@ -5384,8 +5385,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow", "arrow-array", @@ -5425,8 +5426,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "arrow-array", "arrow-schema", @@ -5439,8 +5440,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "12.0.0-beta.18" -source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.18#1047c4bdbf36331cff2d9e21ab525c45ed741f93" +version = "13.0.0-beta.1" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 1068d48ee..e6560dbf7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=12.0.0-beta.18", default-features = false, "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=12.0.0-beta.18", default-features = false, "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=12.0.0-beta.18", default-features = false, "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=12.0.0-beta.18", "tag" = "v12.0.0-beta.18", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=13.0.0-beta.1", default-features = false, "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=13.0.0-beta.1", default-features = false, "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=13.0.0-beta.1", default-features = false, "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 37c182feb..9b1c19f96 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 12.0.0-beta.18 + 13.0.0-beta.1 false 2.30.0 1.7 diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index ac74c65ef..852bdec80 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -2766,6 +2766,15 @@ describe("when dealing with tags", () => { }); }); +/** Returns a Date strictly later than every instant observed before the call. */ +async function nextMillisecond(): Promise { + const start = Date.now(); + while (Date.now() <= start) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + return new Date(); +} + describe("when optimizing a dataset", () => { let tmpDir: tmp.DirResult; let table: Table; @@ -2788,7 +2797,12 @@ describe("when optimizing a dataset", () => { }); it("cleanups old versions", async () => { - const stats = await table.optimize({ cleanupOlderThan: new Date() }); + // Lance stores version timestamps with nanosecond precision while a JS + // Date only has millisecond precision. A cutoff captured in the same + // millisecond as the last commit would truncate to *before* that commit + // and leave it in place, so wait for the clock to tick over first. + const cutoff = await nextMillisecond(); + const stats = await table.optimize({ cleanupOlderThan: cutoff }); expect(stats.prune.bytesRemoved).toBeGreaterThan(0); expect(stats.prune.oldVersionsRemoved).toBe(2); }); From 09e5418943782507e34c895c6fe5c8c0e1f759ee Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:00:31 +0800 Subject: [PATCH 10/43] build(deps): bump the rust-minor-patch group across 1 directory with 5 updates (#4178) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the rust-minor-patch group with 5 updates in the / directory: | Package | From | To | | --- | --- | --- | | [uuid](https://github.com/uuid-rs/uuid) | `1.26.0` | `1.26.1` | | [serde_with](https://github.com/jonasbb/serde_with) | `3.22.0` | `3.23.0` | | [aws-smithy-types](https://github.com/smithy-lang/smithy-rs) | `1.4.8` | `1.6.3` | | [napi-derive](https://github.com/napi-rs/napi-rs) | `3.6.3` | `3.6.5` | | [napi-build](https://github.com/napi-rs/napi-rs) | `2.4.1` | `2.4.2` | Updates `uuid` from 1.26.0 to 1.26.1
Release notes

Sourced from uuid's releases.

v1.26.1

What's Changed

New Contributors

Full Changelog: https://github.com/uuid-rs/uuid/compare/v1.26.0...v1.26.1

Commits
  • 9f92712 Merge pull request #910 from uuid-rs/cargo/v1.26.1
  • d4df8f0 prepare for 1.26.1 release
  • 5613f23 Merge pull request #909 from uuid-rs/fix/ts-conversion-overflow
  • fda00eb don't panic in overflowing Timestamp to SystemTime conversion
  • c82e88c Merge pull request #907 from lenamonj/v7-counter-placement
  • ac065a6 Align the counter diagram
  • 34ec102 Seat the v7 counter below the version nibble
  • See full diff in compare view

Updates `serde_with` from 3.22.0 to 3.23.0
Release notes

Sourced from serde_with's releases.

serde_with v3.23.0

Changed

  • Update syn and darling dependencies to use syn v3 (#992)
  • Update dev-dependencies to newer versions (#993)
  • Update base64 to a newer version. This should not have any API change, but some error messages might change. (#993)
  • serde_as can now parse cfg_attr(true, ...) and cfg_attr(false, ...) (#995) true/false are new literals as of Rust 1.88 but need to be parsed explicitly with the syn types. This is used when emitting schemars annotations.
Commits
  • ea5dfdc Bump version to v3.23.0 (#1005)
  • e52e85b Bump version to v3.23.0
  • 39955b6 Bump rmp dev-dependency (#1004)
  • b51e5fb Bump rmp dev-dependency
  • ef598c6 Use setup-rust-toolchain v2 instead of v1 (#1003)
  • 9ea2658 Fix cargo lint about workspace lints being inherited in the test crate
  • e5ad81a Use setup-rust-toolchain v2 instead of v1
  • 8100141 Bump the github-actions group across 1 directory with 2 updates (#1002)
  • 940f4a6 Bump jsonschema from 0.49.8 to 0.52.0 (#1001)
  • 6f42b94 Bump the github-actions group across 1 directory with 2 updates
  • Additional commits viewable in compare view

Updates `aws-smithy-types` from 1.4.8 to 1.6.3
Commits

Updates `napi-derive` from 3.6.3 to 3.6.5
Release notes

Sourced from napi-derive's releases.

napi-derive-v3.6.5

Other

  • update Cargo.toml dependencies

napi-derive-v3.6.4

Fixed

  • (deps) update rust crate convert_case to 0.12 (#3469)
Commits
  • 1492b22 chore: release (#3502)
  • 828983a fix(napi): return errors from the serde deserializer for unexpected JS value ...
  • 606b142 fix(napi): validate wrapped payload provenance in Object::unwrap/remove_wrapp...
  • a75d89f fix(napi): point from_external slices at the engine-owned copy after finalize...
  • e414c8c fix(deps): update dependency obug to v3 (#3499)
  • a5fedde chore(deps): update release-plz/action action to v0.5.136 (#3505)
  • 31c27a1 chore: release (#3470)
  • 7e3f293 chore(release): publish
  • f772ee0 fix(cli): align generated file formats (#3501)
  • 1cf5ec5 fix(cli): use accessible WASI preopen root on Android (#3485)
  • Additional commits viewable in compare view

Updates `napi-build` from 2.4.1 to 2.4.2
Release notes

Sourced from napi-build's releases.

napi-build-v2.4.2

Fixed

  • (cli,build) make wasm32-wasip1-threads link with wasi-sdk 34 and Rust nightly (#3492)
Commits
  • 31c27a1 chore: release (#3470)
  • 7e3f293 chore(release): publish
  • f772ee0 fix(cli): align generated file formats (#3501)
  • 1cf5ec5 fix(cli): use accessible WASI preopen root on Android (#3485)
  • cf9245d chore(deps): update vitest monorepo to v5 (#3481)
  • 78fa3ad fix(macro): recognize fully-qualified napi::Env as the special Env parameter ...
  • 37283c7 chore(deps): lock file maintenance (#3474)
  • 1932d2f chore(deps): update release-plz/action action to v0.5.135 (#3500)
  • 0238e8b fix(deps): update dependency js-yaml to v5 (#3344)
  • 9ed9d35 chore(deps): update dependency electron to v44 (#3468)
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Xuanwo --- Cargo.lock | 60 +++++++++++++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6dad3d6b6..24889deec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1114,9 +1114,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.4.8" +version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "056b66dbce2f81cc0c1e2b05bb402eb58f8a3530479d650efadd5bbae9a4050b" +checksum = "8f94d16e797ec62cd999fc9d5942b48fa7050c3093ddadff48e4d7528d16fcb9" dependencies = [ "base64-simd", "bytes", @@ -1941,9 +1941,9 @@ checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" [[package]] name = "convert_case" -version = "0.11.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "affbf0190ed2caf063e3def54ff444b449371d55c58e513a95ab98eca50adb49" +checksum = "1af709f1f33454bf52eadfc8c78b3b9ef9cb26fb54d16dc9cd9a7299f899fd1b" dependencies = [ "unicode-segmentation", ] @@ -2263,12 +2263,12 @@ dependencies = [ [[package]] name = "darling" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +checksum = "ed17f5901b6630b993ca003def43f2f8ef4014fc13b047b57aad617ff32bc2ec" dependencies = [ - "darling_core 0.23.0", - "darling_macro 0.23.0", + "darling_core 0.24.1", + "darling_macro 0.24.1", ] [[package]] @@ -2287,15 +2287,15 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +checksum = "6837e2cf7485aaae18f86181d2f0e9a7ed297a025e220aeabf63fdebd3a2ddff" dependencies = [ "ident_case", "proc-macro2", "quote", "strsim", - "syn 2.0.117", + "syn 3.0.3", ] [[package]] @@ -2311,13 +2311,13 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +checksum = "2ac7135c3ef02b2f7833bbeb1be5ba7f966dcde8a87c6b87f65a778d71a02785" dependencies = [ - "darling_core 0.23.0", + "darling_core 0.24.1", "quote", - "syn 2.0.117", + "syn 3.0.3", ] [[package]] @@ -6172,15 +6172,15 @@ dependencies = [ [[package]] name = "napi-build" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60fdf9b392c50e7c4170fa633bd909490ed7835cea4c046776d1a4dd8d2ae0ab" +checksum = "860e7c40864f95cfb83cde99f9ebadd88ef3d9bdccd7dd2cee0cc96a2dd4ffa7" [[package]] name = "napi-derive" -version = "3.6.3" +version = "3.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa55ea69990c90b888e9e77044410e304ce7f35de599dc6d0b5c1923d2e59af" +checksum = "350057056a30368aa76c11a0d406b0aa61321710be2c36656ab4f6efe0b785a2" dependencies = [ "convert_case", "ctor", @@ -6192,9 +6192,9 @@ dependencies = [ [[package]] name = "napi-derive-backend" -version = "6.1.2" +version = "6.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df4056ac7c18e4438ccf0edaed4340ca0d269278c8ec19284f7b23cb039fd0ae" +checksum = "4c1c87a71568f3fe5c736b10878ff55064b250bb4ea4b48c8055713e07b9e463" dependencies = [ "convert_case", "proc-macro2", @@ -9156,11 +9156,11 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.22.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee78f1fbe43ac4a0e47aadb3dbd357b69eb0d3793e948624cd03dd2750ab1c0a" +checksum = "935177bb8c0cd8ca1a4e6d1a2ac8988bea69cab4f9d3a31311e012ad27868ea4" dependencies = [ - "base64 0.22.1", + "base64 0.23.1", "bs58", "chrono", "hex", @@ -9177,14 +9177,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.22.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8705578779c2b6bd90d84d66eb2e206b708b1a4d7b9f17641b293545bf1c7e46" +checksum = "1d607aa01a3cb0ad757d6fd216136910db3c97b102fe686585689615a02dbcdc" dependencies = [ - "darling 0.23.0", + "darling 0.24.1", "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.3", ] [[package]] @@ -10563,9 +10563,9 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.26.0" +version = "1.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5772d71c9be8a8a6ac2117d949c5b224c1b72241bb611d9a3012edcf8af7812" +checksum = "2ef6dac1e96601b4fb3acccccff2139741fcb757cb9a36089bf5be91cfb285ce" dependencies = [ "getrandom 0.4.2", "js-sys", From 575286922b08c66559fa051029c6fa7e5a76dfe5 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Tue, 15 Sep 2026 08:31:28 -0700 Subject: [PATCH 11/43] fix: redact authentication headers from debug logs (#4179) Prevent bearer tokens, API keys, cookies, and proxy credentials from appearing in request debug output through a reusable `redact_sensitive_headers` utility. The utility is applied after default/configured header construction and dynamic header merging, so it also covers retry requests. Invalid dynamic-header values are no longer echoed, and regression coverage verifies that credentials are redacted while ordinary diagnostic headers remain visible. Extracted from the OAuth discussion in #4173, following [Colin's review](https://github.com/lancedb/lancedb/pull/4173#issuecomment-5674048100). --- rust/lancedb/src/remote/client.rs | 101 +++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 3 deletions(-) diff --git a/rust/lancedb/src/remote/client.rs b/rust/lancedb/src/remote/client.rs index 57dd89890..704afe948 100644 --- a/rust/lancedb/src/remote/client.rs +++ b/rust/lancedb/src/remote/client.rs @@ -15,6 +15,25 @@ use crate::remote::retry::{ResolvedRetryConfig, RetryCounter}; const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id"); +pub(crate) fn redact_sensitive_headers(headers: &mut HeaderMap) { + const SENSITIVE_HEADERS: [&str; 5] = [ + "authorization", + "proxy-authorization", + "cookie", + "set-cookie", + "x-api-key", + ]; + + for (name, value) in headers.iter_mut() { + if SENSITIVE_HEADERS + .iter() + .any(|sensitive| name.as_str().eq_ignore_ascii_case(sensitive)) + { + value.set_sensitive(true); + } + } +} + /// Configuration for TLS/mTLS settings. #[derive(Clone, Debug)] pub struct TlsConfig { @@ -681,6 +700,7 @@ impl RestfulLanceDbClient { ); } + redact_sensitive_headers(&mut headers); Ok(headers) } @@ -714,13 +734,14 @@ impl RestfulLanceDbClient { if let Ok(header_value) = HeaderValue::from_str(&value) { request_headers.insert(header_name, header_value); } else { - debug!("Invalid header value for key {}: {}", key, value); + debug!("Invalid header value for key {}", key); } } else { debug!("Invalid header name: {}", key); } } } + redact_sensitive_headers(request.headers_mut()); Ok(request) } @@ -1180,7 +1201,7 @@ mod tests { assert!(!headers.contains_key("x-api-key")); let headers = RestfulLanceDbClient::::default_headers( - "api-key", + "static-secret-value", "us-east-1", "db-name", false, @@ -1189,7 +1210,55 @@ mod tests { &ClientConfig::default(), ) .unwrap(); - assert_eq!(headers.get("x-api-key").unwrap(), "api-key"); + let api_key = headers.get("x-api-key").unwrap(); + assert_eq!(api_key, "static-secret-value"); + assert!(api_key.is_sensitive()); + assert!(!format!("{headers:?}").contains("static-secret-value")); + } + + #[test] + fn test_configured_authentication_headers_are_sensitive() { + let config = ClientConfig { + extra_headers: HashMap::from([ + ( + "Authorization".to_string(), + "Bearer configured-secret".to_string(), + ), + ("X-API-Key".to_string(), "configured-api-key".to_string()), + ( + "Cookie".to_string(), + "session=configured-cookie".to_string(), + ), + ( + "Proxy-Authorization".to_string(), + "Basic configured-proxy-secret".to_string(), + ), + ("X-Custom".to_string(), "visible-value".to_string()), + ]), + ..Default::default() + }; + let headers = RestfulLanceDbClient::::default_headers( + "", + "us-east-1", + "db-name", + false, + &RemoteOptions::default(), + None, + &config, + ) + .unwrap(); + + assert!(headers.get("authorization").unwrap().is_sensitive()); + assert!(headers.get("x-api-key").unwrap().is_sensitive()); + assert!(headers.get("cookie").unwrap().is_sensitive()); + assert!(headers.get("proxy-authorization").unwrap().is_sensitive()); + assert!(!headers.get("x-custom").unwrap().is_sensitive()); + let debug = format!("{headers:?}"); + assert!(!debug.contains("configured-secret")); + assert!(!debug.contains("configured-api-key")); + assert!(!debug.contains("configured-cookie")); + assert!(!debug.contains("configured-proxy-secret")); + assert!(debug.contains("visible-value")); } #[test] @@ -1303,6 +1372,7 @@ mod tests { // Test that dynamic headers override existing headers let mut headers = HashMap::new(); headers.insert("Authorization".to_string(), "Bearer new-token".to_string()); + headers.insert("X-API-Key".to_string(), "new-api-key".to_string()); headers.insert("X-Custom".to_string(), "custom-value".to_string()); let provider = TestHeaderProvider::new(headers); @@ -1338,6 +1408,31 @@ mod tests { updated_request.headers().get("X-Custom").unwrap(), "custom-value" ); + assert!( + updated_request + .headers() + .get("Authorization") + .unwrap() + .is_sensitive() + ); + assert!( + updated_request + .headers() + .get("X-API-Key") + .unwrap() + .is_sensitive() + ); + assert!( + !updated_request + .headers() + .get("X-Custom") + .unwrap() + .is_sensitive() + ); + let debug = format!("{updated_request:?}"); + assert!(!debug.contains("new-token")); + assert!(!debug.contains("new-api-key")); + assert!(debug.contains("custom-value")); // Existing headers should still be present assert_eq!( updated_request.headers().get("X-Existing").unwrap(), From 2f88b71c216cd4cc66cf5ff560395b745624fb10 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Tue, 15 Sep 2026 11:01:10 -0700 Subject: [PATCH 12/43] feat: add persistent OAuth token cache and session APIs (#4182) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stacked on #4173 (diff includes it until that merges; will rebase after). Addresses the token-cache part of [Colin's review](https://github.com/lancedb/lancedb/pull/4173#issuecomment-5674048100). Adds an explicit, opt-in persistent OAuth token cache shared by Rust, Python, and Node clients, plus `login` / `status` / `logout` session APIs, so short-lived processes (CLIs, scripts, notebooks) reuse one session instead of restarting a browser or device flow on every start. - **Opt-in and minimal**: existing callers stay memory-only and lazy. Only refresh tokens are persisted (never access tokens, never client secrets), so there are no local token-expiry decisions to get wrong when clocks move. Each process start performs one silent refresh grant. - **Hardened file backend**: private directory (`0700`), per-record files (`0600`), owner validation, symlink rejection, and atomic `rename` replacement. Corrupt, truncated, unknown-version, or permission-invalid records fail with actionable errors naming the file. Native keyring backends were evaluated (keyring crate routes Linux through D-Bus/zbus: heavy deps, headless/CI flakiness) and are deferred; the file store is the explicit opt-in, not a downgrade from a keyring. - **Cache key**: SHA-256 of the canonical identity (issuer, client ID, sorted/de-duplicated scopes, flow, public/confidential), so no secret appears in a filename and distinct identities never collide. Versioned record schema (`version: 1`). One record per identity: last login wins, documented. - **Cross-process rotation locking**: per-key `fs4` file lock (`flock` / `LockFileEx`) around the refresh critical section — acquire, reread the durable record, refresh exactly once, atomically store the rotated refresh token, release. The OS releases locks on process death, so crashes cannot strand stale locks. Only confirmed `invalid_grant`/`invalid_token` deletes a record and reauthenticates; transport, 5xx, 429, and parse failures retain it. - **Session APIs**: `OAuthSession::login/status/logout` in Rust, `lancedb.remote.OAuthSession` (async) in Python, `OAuthSession` class in Node. `status` returns non-secret metadata only. `logout` removes only the local credential — provider revocation (RFC 7009) is a deliberate follow-up, and local logout never terminates browser SSO. Azure managed identity is rejected for persistence (machine identity stays in memory); client credentials have nothing refreshable to persist and stay memory-only. - No CLI binary exists in this repo, so this ships library APIs plus doc examples in all three languages. Tests: Rust unit + mock-IdP integration (cache-key canonicalization/separation, record versioning/corruption/truncation/symlink/owner/perms, lock serialization + release, two concurrent providers proving no `invalid_grant` and correct rotation, transient-failure retention, `invalid_grant` delete + reauthenticate, login/status/logout lifecycle, client-credentials no-op, IMDS rejection, secret redaction); Python lifecycle + a true two-subprocess cross-process reuse test (second process refreshes once, never hits the device endpoint); Node lifecycle + device-flow login test. Local builds were skipped in development; CI validates all bindings. --------- Co-authored-by: Xuanwo --- Cargo.lock | 84 + docs/src/js/classes/OAuthSession.md | 105 + docs/src/js/enumerations/OAuthFlowType.md | 20 + docs/src/js/globals.md | 4 + docs/src/js/interfaces/NativeOAuthConfig.md | 44 +- docs/src/js/interfaces/OAuthConfig.md | 57 + docs/src/js/interfaces/SessionLogout.md | 20 + docs/src/js/interfaces/SessionStatus.md | 74 + docs/src/js/interfaces/TokenCacheOptions.md | 41 + nodejs/__test__/oauth.test.ts | 211 ++ nodejs/lancedb/index.ts | 9 +- nodejs/lancedb/oauth.ts | 175 ++ nodejs/src/remote.rs | 224 ++- nodejs/typedoc.json | 3 +- python/python/lancedb/_lancedb.pyi | 27 + python/python/lancedb/remote/__init__.py | 4 +- python/python/lancedb/remote/oauth.py | 138 ++ python/src/lib.rs | 3 + python/src/oauth.rs | 252 ++- python/tests/test_oauth.py | 295 +++ rust/lancedb/Cargo.toml | 8 +- rust/lancedb/src/connection.rs | 2 + rust/lancedb/src/remote.rs | 4 +- rust/lancedb/src/remote/oauth.rs | 2004 ++++++++++++++++++- rust/lancedb/src/remote/token_cache.rs | 1753 ++++++++++++++++ 25 files changed, 5464 insertions(+), 97 deletions(-) create mode 100644 docs/src/js/classes/OAuthSession.md create mode 100644 docs/src/js/interfaces/SessionLogout.md create mode 100644 docs/src/js/interfaces/SessionStatus.md create mode 100644 docs/src/js/interfaces/TokenCacheOptions.md create mode 100644 nodejs/__test__/oauth.test.ts create mode 100644 rust/lancedb/src/remote/token_cache.rs diff --git a/Cargo.lock b/Cargo.lock index 24889deec..090e2c654 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3088,6 +3088,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "dispatch2" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" +dependencies = [ + "bitflags 2.11.1", + "objc2", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -3483,6 +3493,16 @@ version = "1.20260804.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "82eb03a32a1d50555353c85a7b9d3279a6f1e91af9890b789acdf544ed57c8d7" +[[package]] +name = "fs4" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4" +dependencies = [ + "rustix", + "windows-sys 0.59.0", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -5477,6 +5497,7 @@ dependencies = [ "aws-sdk-s3", "aws-smithy-runtime", "aws-smithy-types", + "base64 0.22.1", "bytes", "candle-core", "candle-nn", @@ -5491,6 +5512,7 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-plan", "datafusion-sql", + "fs4", "futures", "half", "hf-hub", @@ -5543,6 +5565,7 @@ dependencies = [ "urlencoding", "uuid", "walkdir", + "webbrowser", ] [[package]] @@ -6227,6 +6250,12 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "nibble_vec" version = "0.1.0" @@ -6409,6 +6438,26 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "objc2" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f" +dependencies = [ + "objc2-encode", +] + +[[package]] +name = "objc2-app-kit" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d49e936b501e5c5bf01fda3a9452ff86dc3ea98ad5f283e1455153142d97518c" +dependencies = [ + "bitflags 2.11.1", + "objc2", + "objc2-foundation", +] + [[package]] name = "objc2-core-foundation" version = "0.3.2" @@ -6416,6 +6465,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.11.1", + "dispatch2", + "objc2", +] + +[[package]] +name = "objc2-encode" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33" + +[[package]] +name = "objc2-foundation" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" +dependencies = [ + "bitflags 2.11.1", + "objc2", + "objc2-core-foundation", ] [[package]] @@ -10788,6 +10856,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webbrowser" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62c35be770821a214dbc362fc26908c853e776c0004294d0b10b8a6bad582f94" +dependencies = [ + "jni", + "log", + "ndk-context", + "objc2", + "objc2-app-kit", + "objc2-foundation", + "url", + "web-sys", +] + [[package]] name = "webpki-root-certs" version = "1.0.7" diff --git a/docs/src/js/classes/OAuthSession.md b/docs/src/js/classes/OAuthSession.md new file mode 100644 index 000000000..e80202bea --- /dev/null +++ b/docs/src/js/classes/OAuthSession.md @@ -0,0 +1,105 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / OAuthSession + +# Class: OAuthSession + +Explicit OAuth session lifecycle for the persistent token cache: eager +`login`, non-secret `status`, and local `logout`. + +A session is built from the same [OAuthConfig](../interfaces/OAuthConfig.md) used to connect +(including its `tokenCache` options). A connection created with the same +configuration shares the cache, so logging in here prepares tokens for +later processes without any database request. + +`login` always runs the configured interactive flow and replaces the cached +session (the most recent login wins). `logout` removes only the local +credential; it does not revoke anything with the provider and does not sign +out of a browser SSO session. + +## Example + +```typescript +const config: OAuthConfig = { + issuerUrl: "https://issuer.example.com", + clientId: "my-app", + scopes: ["openid", "offline_access"], + flow: OAuthFlowType.DeviceCode, + tokenCache: { cacheDir: "/tmp/my-app/oauth-cache" }, +}; +const session = new OAuthSession(config); +const status = await session.login(); +``` + +## Constructors + +### new OAuthSession() + +```ts +new OAuthSession(config): OAuthSession +``` + +Create a session manager for the given OAuth configuration. + +#### Parameters + +* **config**: [`OAuthConfig`](../interfaces/OAuthConfig.md) + +#### Returns + +[`OAuthSession`](OAuthSession.md) + +## Methods + +### login() + +```ts +login(): Promise +``` + +Eagerly run the configured authentication flow and store the session. + +A successful login always replaces any prior cached session for this +identity; if the provider does not issue a refresh token (for example +without `offline_access`), the previous record is removed and the status +reports `refreshable == false`. + +#### Returns + +`Promise`<[`SessionStatus`](../interfaces/SessionStatus.md)> + +*** + +### logout() + +```ts +logout(): Promise +``` + +Remove the matching local cached credential. + +This only deletes the local cache entry. It does not revoke the refresh +token with the provider and does not sign out of a browser SSO session. +Repeated calls succeed; `removed` reports whether a credential existed. + +#### Returns + +`Promise`<[`SessionLogout`](../interfaces/SessionLogout.md)> + +*** + +### status() + +```ts +status(): Promise +``` + +Report whether a matching cached session exists, with safe metadata. + +This never contacts the identity provider and never exposes token values. + +#### Returns + +`Promise`<[`SessionStatus`](../interfaces/SessionStatus.md)> diff --git a/docs/src/js/enumerations/OAuthFlowType.md b/docs/src/js/enumerations/OAuthFlowType.md index fe546a140..7daae22fb 100644 --- a/docs/src/js/enumerations/OAuthFlowType.md +++ b/docs/src/js/enumerations/OAuthFlowType.md @@ -10,6 +10,16 @@ OAuth authentication flow types. ## Enumeration Members +### AuthorizationCode + +```ts +AuthorizationCode: "authorization_code"; +``` + +Interactive Authorization Code grant, using PKCE by default. + +*** + ### AzureManagedIdentity ```ts @@ -27,3 +37,13 @@ ClientCredentials: "client_credentials"; ``` Client Credentials grant (service-to-service / M2M). + +*** + +### DeviceCode + +```ts +DeviceCode: "device_code"; +``` + +Device Authorization grant for CLI and headless environments. diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 4a5effaae..422c6c428 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -35,6 +35,7 @@ - [MultiMatchQuery](classes/MultiMatchQuery.md) - [NativeJsHeaderProvider](classes/NativeJsHeaderProvider.md) - [OAuthHeaderProvider](classes/OAuthHeaderProvider.md) +- [OAuthSession](classes/OAuthSession.md) - [PermutationBuilder](classes/PermutationBuilder.md) - [PhraseQuery](classes/PhraseQuery.md) - [Query](classes/Query.md) @@ -122,6 +123,8 @@ - [RestNamespaceConfig](interfaces/RestNamespaceConfig.md) - [RetryConfig](interfaces/RetryConfig.md) - [ScannableOptions](interfaces/ScannableOptions.md) +- [SessionLogout](interfaces/SessionLogout.md) +- [SessionStatus](interfaces/SessionStatus.md) - [ShuffleOptions](interfaces/ShuffleOptions.md) - [SplitCalculatedOptions](interfaces/SplitCalculatedOptions.md) - [SplitHashOptions](interfaces/SplitHashOptions.md) @@ -131,6 +134,7 @@ - [TableStatistics](interfaces/TableStatistics.md) - [TimeoutConfig](interfaces/TimeoutConfig.md) - [TlsConfig](interfaces/TlsConfig.md) +- [TokenCacheOptions](interfaces/TokenCacheOptions.md) - [TokenResponse](interfaces/TokenResponse.md) - [TokenizeOptions](interfaces/TokenizeOptions.md) - [UpdateFieldMetadataResult](interfaces/UpdateFieldMetadataResult.md) diff --git a/docs/src/js/interfaces/NativeOAuthConfig.md b/docs/src/js/interfaces/NativeOAuthConfig.md index 6959f17dd..afe05de9f 100644 --- a/docs/src/js/interfaces/NativeOAuthConfig.md +++ b/docs/src/js/interfaces/NativeOAuthConfig.md @@ -15,6 +15,16 @@ All token acquisition and refresh is handled in the Rust layer. ## Properties +### callbackPort? + +```ts +optional callbackPort: number; +``` + +Port for the authorization_code loopback callback server. + +*** + ### clientId ```ts @@ -41,7 +51,8 @@ Client secret (required for client_credentials). optional flow: string; ``` -Authentication flow: "client_credentials" or "azure_managed_identity" +Authentication flow: "client_credentials", "authorization_code", +"device_code", or "azure_managed_identity" *** @@ -66,6 +77,16 @@ Client ID for user-assigned managed identity (azure_managed_identity). *** +### redirectUri? + +```ts +optional redirectUri: string; +``` + +Loopback redirect URI for authorization_code. + +*** + ### refreshBufferSecs? ```ts @@ -86,3 +107,24 @@ scopes: string[]; OAuth scopes to request. For Azure managed identity, exactly one scope or resource is required. For example: `["api://{app_id}/.default"]` + +*** + +### tokenCache? + +```ts +optional tokenCache: TokenCacheOptions; +``` + +Opt in to the persistent token cache so short-lived processes reuse +one session. Only refresh tokens are persisted. + +*** + +### usePkce? + +```ts +optional usePkce: boolean; +``` + +Whether authorization_code uses S256 PKCE (default: true). diff --git a/docs/src/js/interfaces/OAuthConfig.md b/docs/src/js/interfaces/OAuthConfig.md index f9d5d1c7b..f7d61c663 100644 --- a/docs/src/js/interfaces/OAuthConfig.md +++ b/docs/src/js/interfaces/OAuthConfig.md @@ -35,8 +35,32 @@ const config: OAuthConfig = { }; ``` +The authorization URL is written to stderr before LanceDB tries to open a +browser, so it can be copied in headless environments. +```typescript +const config: OAuthConfig = { + issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0", + clientId: "app-id", + scopes: ["openid", "api://lancedb-api/access"], + flow: OAuthFlowType.AuthorizationCode, +}; +``` + +Device Authorization writes the verification URL and user code to stderr +before polling begins. + ## Properties +### callbackPort? + +```ts +optional callbackPort: number; +``` + +Port for the AuthorizationCode loopback callback server (default: 8400). + +*** + ### clientId ```ts @@ -88,6 +112,16 @@ Client ID for user-assigned managed identity (AzureManagedIdentity). *** +### redirectUri? + +```ts +optional redirectUri: string; +``` + +Loopback redirect URI for AuthorizationCode. + +*** + ### refreshBufferSecs? ```ts @@ -109,3 +143,26 @@ scopes: string[]; OAuth scopes to request. For Azure managed identity, exactly one scope or resource is required. For example: `["api://{app_id}/.default"]` + +*** + +### tokenCache? + +```ts +optional tokenCache: TokenCacheOptions; +``` + +Opt in to the persistent token cache so short-lived processes reuse one +session. Only refresh tokens are persisted. Only supported by +AuthorizationCode and DeviceCode; Azure managed identity is rejected. +Default: unset (memory only). + +*** + +### usePkce? + +```ts +optional usePkce: boolean; +``` + +Protect AuthorizationCode with S256 PKCE (default: true). diff --git a/docs/src/js/interfaces/SessionLogout.md b/docs/src/js/interfaces/SessionLogout.md new file mode 100644 index 000000000..c91fe5108 --- /dev/null +++ b/docs/src/js/interfaces/SessionLogout.md @@ -0,0 +1,20 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / SessionLogout + +# Interface: SessionLogout + +Result of [OAuthSession.logout](../classes/OAuthSession.md#logout). + +## Properties + +### removed + +```ts +removed: boolean; +``` + +Whether a cached credential was removed. `false` means no matching +session was cached; logout is idempotent. diff --git a/docs/src/js/interfaces/SessionStatus.md b/docs/src/js/interfaces/SessionStatus.md new file mode 100644 index 000000000..3844bfae6 --- /dev/null +++ b/docs/src/js/interfaces/SessionStatus.md @@ -0,0 +1,74 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / SessionStatus + +# Interface: SessionStatus + +Safe, non-secret view of a cached OAuth session, returned by +[OAuthSession.status](../classes/OAuthSession.md#status) and [OAuthSession.login](../classes/OAuthSession.md#login). + +## Properties + +### clientId + +```ts +clientId: string; +``` + +Client ID of the cached session. + +*** + +### flow + +```ts +flow: string; +``` + +Flow that produced the cached session. + +*** + +### issuerUrl + +```ts +issuerUrl: string; +``` + +Canonical issuer URL of the cached session. + +*** + +### obtainedAt? + +```ts +optional obtainedAt: number; +``` + +When the cached session was obtained, as Unix seconds. + +*** + +### refreshable + +```ts +refreshable: boolean; +``` + +Whether a cached session exists that can obtain tokens without +interactive authentication. Because access tokens are not persisted, +this is `true` exactly when a refresh token is cached; the next +connection refreshes with it rather than opening a browser or device +prompt. + +*** + +### scopes + +```ts +scopes: string[]; +``` + +Canonical (sorted, de-duplicated) scope set of the cached session. diff --git a/docs/src/js/interfaces/TokenCacheOptions.md b/docs/src/js/interfaces/TokenCacheOptions.md new file mode 100644 index 000000000..95b96ef2f --- /dev/null +++ b/docs/src/js/interfaces/TokenCacheOptions.md @@ -0,0 +1,41 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / TokenCacheOptions + +# Interface: TokenCacheOptions + +Options for the persistent OAuth token cache. + +The cache is opt-in: it is only used when set as `tokenCache` on +[OAuthConfig](OAuthConfig.md). Only refresh tokens are persisted, in a private +directory with owner-only permissions, so short-lived processes can reuse +an authenticated session instead of re-prompting on every start. + +Multiple identities (issuer, client, scopes, flow, client authentication) +get separate cache entries. Within one identity the most recent login wins. + +## Properties + +### cacheDir? + +```ts +optional cacheDir: string; +``` + +Directory that holds cached credentials. Defaults to +`$XDG_CACHE_HOME/lancedb/oauth`, `$HOME/.cache/lancedb/oauth` on Unix, +or `%LOCALAPPDATA%\\lancedb\\oauth` on Windows. The directory is created +with owner-only permissions (`0700`) when missing. + +*** + +### lockTimeoutSecs? + +```ts +optional lockTimeoutSecs: number; +``` + +How long to wait for the cross-process refresh lock before failing, in +seconds (default: 30). diff --git a/nodejs/__test__/oauth.test.ts b/nodejs/__test__/oauth.test.ts new file mode 100644 index 000000000..83ea2791b --- /dev/null +++ b/nodejs/__test__/oauth.test.ts @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import * as fs from "fs"; +import * as http from "http"; +import * as os from "os"; +import * as path from "path"; +import { OAuthConfig, OAuthFlowType, OAuthSession } from "../lancedb/oauth"; + +function tempCacheDir(): string { + return fs.mkdtempSync(path.join(os.tmpdir(), "lancedb-oauth-cache-")); +} + +function deviceConfig(issuerUrl: string, cacheDir: string): OAuthConfig { + return { + issuerUrl, + clientId: "client-id", + scopes: ["openid"], + flow: OAuthFlowType.DeviceCode, + tokenCache: { cacheDir }, + }; +} + +describe("OAuthSession", () => { + beforeAll(() => { + // Point the Rust browser helper at a no-op so device-flow logins never + // open a real browser window during tests. + process.env.LANCEDB_OAUTH_BROWSER = "/usr/bin/true"; + }); + + it("reports an absent session and logout is idempotent", async () => { + const cacheDir = tempCacheDir(); + const session = new OAuthSession( + deviceConfig("https://issuer.example.com", cacheDir), + ); + + const status = await session.status(); + expect(status.refreshable).toBe(false); + expect(status.issuerUrl).toBe("https://issuer.example.com"); + expect(status.clientId).toBe("client-id"); + expect(status.scopes).toEqual(["openid"]); + expect(status.flow).toBe("device_code"); + expect(status.obtainedAt).toBeUndefined(); + + const logout = await session.logout(); + expect(logout.removed).toBe(false); + }); + + it("requires token cache options", () => { + const config: OAuthConfig = { + issuerUrl: "https://issuer.example.com", + clientId: "client-id", + scopes: ["openid"], + flow: OAuthFlowType.DeviceCode, + }; + expect(() => new OAuthSession(config)).toThrow(/token/); + }); + + it("rejects azure managed identity persistence", () => { + const config: OAuthConfig = { + issuerUrl: "https://login.microsoftonline.com/tenant/v2.0", + clientId: "app-id", + scopes: ["api://app/.default"], + flow: OAuthFlowType.AzureManagedIdentity, + tokenCache: { cacheDir: tempCacheDir() }, + }; + expect(() => new OAuthSession(config)).toThrow(/AzureManagedIdentity/); + }); + + it("logs in via device flow, caches, and logs out", async () => { + const server = new MockIdp(); + await server.start(); + try { + const cacheDir = tempCacheDir(); + const issuerUrl = server.issuerUrl(); + + const session = new OAuthSession(deviceConfig(issuerUrl, cacheDir)); + const status = await session.login(); + expect(status.refreshable).toBe(true); + expect(status.obtainedAt).toBeGreaterThan(0); + expect(server.state.deviceAuthorizations).toBe(1); + + // An independent session (a fresh "process") sees the cached login. + const other = new OAuthSession(deviceConfig(issuerUrl, cacheDir)); + const cached = await other.status(); + expect(cached.refreshable).toBe(true); + + const logout = await other.logout(); + expect(logout.removed).toBe(true); + const again = await session.logout(); + expect(again.removed).toBe(false); + expect((await session.status()).refreshable).toBe(false); + + // Only the initial login used the interactive device flow. + expect(server.state.deviceAuthorizations).toBe(1); + expect(server.state.refreshGrants).toBe(0); + } finally { + server.close(); + } + }, 15000); +}); + +/** Mock IdP with discovery, device authorization, and rotating refresh. */ +class MockIdp { + readonly state = { + deviceAuthorizations: 0, + refreshGrants: 0, + accessTokensIssued: 0, + currentRefresh: null as string | null, + }; + private server?: http.Server; + private port = 0; + + issuerUrl(): string { + return `http://127.0.0.1:${this.port}`; + } + + async start(): Promise { + const server = http.createServer((req, res) => { + const chunks: Buffer[] = []; + req.on("data", (chunk) => chunks.push(chunk)); + req.on("end", () => { + const body = Buffer.concat(chunks).toString(); + const params = new URLSearchParams(body); + this.handle(req.url ?? "", params, res); + }); + }); + await new Promise((resolve) => { + server.listen(0, "127.0.0.1", () => resolve()); + }); + const address = server.address(); + if (address && typeof address === "object") { + this.port = address.port; + } + this.server = server; + } + + private handle( + url: string, + params: URLSearchParams, + res: http.ServerResponse, + ): void { + const respond = (status: number, payload: unknown): void => { + const body = JSON.stringify(payload); + res.writeHead(status, { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(body), + }); + res.end(body); + }; + + if (url === "/.well-known/openid-configuration") { + respond(200, { + // biome-ignore lint/style/useNamingConvention: OAuth wire format + token_endpoint: `${this.issuerUrl()}/token`, + // biome-ignore lint/style/useNamingConvention: OAuth wire format + device_authorization_endpoint: `${this.issuerUrl()}/device`, + }); + return; + } + + if (url === "/device") { + this.state.deviceAuthorizations += 1; + respond(200, { + // biome-ignore lint/style/useNamingConvention: OAuth wire format + device_code: "device-code", + // biome-ignore lint/style/useNamingConvention: OAuth wire format + user_code: "ABCD-EFGH", + // biome-ignore lint/style/useNamingConvention: OAuth wire format + verification_uri: `${this.issuerUrl()}/verify`, + // biome-ignore lint/style/useNamingConvention: OAuth wire format + expires_in: 60, + interval: 1, + }); + return; + } + + if (url === "/token") { + const grantType = params.get("grant_type") ?? ""; + if (grantType === "refresh_token") { + this.state.refreshGrants += 1; + if (params.get("refresh_token") !== this.state.currentRefresh) { + respond(400, { error: "invalid_grant" }); + return; + } + } else if (!grantType.includes("device_code")) { + respond(400, { error: "unsupported_grant_type" }); + return; + } + this.state.accessTokensIssued += 1; + const number = this.state.accessTokensIssued; + const refresh = `refresh-${number}`; + this.state.currentRefresh = refresh; + respond(200, { + // biome-ignore lint/style/useNamingConvention: OAuth wire format + access_token: `access-${number}`, + // biome-ignore lint/style/useNamingConvention: OAuth wire format + refresh_token: refresh, + // biome-ignore lint/style/useNamingConvention: OAuth wire format + expires_in: 3600, + }); + return; + } + + respond(404, {}); + } + + close(): void { + this.server?.close(); + } +} diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index d94007a11..a58cb182e 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -170,7 +170,14 @@ export { TokenResponse, } from "./header"; -export { OAuthConfig, OAuthFlowType } from "./oauth"; +export { + OAuthConfig, + OAuthFlowType, + OAuthSession, + SessionLogout, + SessionStatus, + TokenCacheOptions, +} from "./oauth"; export { MergeInsertBuilder, WriteExecutionOptions } from "./merge"; diff --git a/nodejs/lancedb/oauth.ts b/nodejs/lancedb/oauth.ts index 345eda87a..c9c78afa6 100644 --- a/nodejs/lancedb/oauth.ts +++ b/nodejs/lancedb/oauth.ts @@ -1,16 +1,52 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors +import { + OAuthConfig as NativeOAuthConfig, + OAuthSession as NativeOAuthSession, +} from "./native"; + /** * OAuth authentication flow types. */ export enum OAuthFlowType { /** Client Credentials grant (service-to-service / M2M). */ ClientCredentials = "client_credentials", + /** Interactive Authorization Code grant, using PKCE by default. */ + AuthorizationCode = "authorization_code", + /** Device Authorization grant for CLI and headless environments. */ + DeviceCode = "device_code", /** Azure Managed Identity via IMDS. */ AzureManagedIdentity = "azure_managed_identity", } +/** + * Options for the persistent OAuth token cache. + * + * The cache is opt-in: it is only used when set as `tokenCache` on + * {@link OAuthConfig}. Only refresh tokens are persisted, in a private + * directory with owner-only permissions, so short-lived processes can reuse + * an authenticated session instead of re-prompting on every start. + * + * Multiple identities (issuer, client, scopes, flow, client authentication) + * get separate cache entries. Within one identity the most recent login wins. + */ +export interface TokenCacheOptions { + /** + * Directory that holds cached credentials. Defaults to + * `$XDG_CACHE_HOME/lancedb/oauth`, `$HOME/.cache/lancedb/oauth` on Unix, + * or `%LOCALAPPDATA%\\lancedb\\oauth` on Windows. The directory is created + * with owner-only permissions (`0700`) when missing. + */ + cacheDir?: string; + + /** + * How long to wait for the cross-process refresh lock before failing, in + * seconds (default: 30). + */ + lockTimeoutSecs?: number; +} + /** * OAuth configuration for LanceDB authentication. * @@ -40,6 +76,21 @@ export enum OAuthFlowType { * flow: OAuthFlowType.AzureManagedIdentity, * }; * ``` + * + * @example Authorization Code with PKCE: + * The authorization URL is written to stderr before LanceDB tries to open a + * browser, so it can be copied in headless environments. + * ```typescript + * const config: OAuthConfig = { + * issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0", + * clientId: "app-id", + * scopes: ["openid", "api://lancedb-api/access"], + * flow: OAuthFlowType.AuthorizationCode, + * }; + * ``` + * + * Device Authorization writes the verification URL and user code to stderr + * before polling begins. */ export interface OAuthConfig { /** @@ -64,6 +115,15 @@ export interface OAuthConfig { /** Client secret (required for ClientCredentials). */ clientSecret?: string; + /** Loopback redirect URI for AuthorizationCode. */ + redirectUri?: string; + + /** Port for the AuthorizationCode loopback callback server (default: 8400). */ + callbackPort?: number; + + /** Protect AuthorizationCode with S256 PKCE (default: true). */ + usePkce?: boolean; + /** Client ID for user-assigned managed identity (AzureManagedIdentity). */ managedIdentityClientId?: string; @@ -73,4 +133,119 @@ export interface OAuthConfig { * the TTL, each request refreshes the token. */ refreshBufferSecs?: number; + + /** + * Opt in to the persistent token cache so short-lived processes reuse one + * session. Only refresh tokens are persisted. Only supported by + * AuthorizationCode and DeviceCode; Azure managed identity is rejected. + * Default: unset (memory only). + */ + tokenCache?: TokenCacheOptions; +} + +/** + * Safe, non-secret view of a cached OAuth session, returned by + * {@link OAuthSession.status} and {@link OAuthSession.login}. + */ +export interface SessionStatus { + /** + * Whether a cached session exists that can obtain tokens without + * interactive authentication. Because access tokens are not persisted, + * this is `true` exactly when a refresh token is cached; the next + * connection refreshes with it rather than opening a browser or device + * prompt. + */ + refreshable: boolean; + + /** Canonical issuer URL of the cached session. */ + issuerUrl: string; + + /** Client ID of the cached session. */ + clientId: string; + + /** Canonical (sorted, de-duplicated) scope set of the cached session. */ + scopes: string[]; + + /** Flow that produced the cached session. */ + flow: string; + + /** When the cached session was obtained, as Unix seconds. */ + obtainedAt?: number; +} + +/** Result of {@link OAuthSession.logout}. */ +export interface SessionLogout { + /** + * Whether a cached credential was removed. `false` means no matching + * session was cached; logout is idempotent. + */ + removed: boolean; +} + +/** + * Explicit OAuth session lifecycle for the persistent token cache: eager + * `login`, non-secret `status`, and local `logout`. + * + * A session is built from the same {@link OAuthConfig} used to connect + * (including its `tokenCache` options). A connection created with the same + * configuration shares the cache, so logging in here prepares tokens for + * later processes without any database request. + * + * `login` always runs the configured interactive flow and replaces the cached + * session (the most recent login wins). `logout` removes only the local + * credential; it does not revoke anything with the provider and does not sign + * out of a browser SSO session. + * + * @example + * ```typescript + * const config: OAuthConfig = { + * issuerUrl: "https://issuer.example.com", + * clientId: "my-app", + * scopes: ["openid", "offline_access"], + * flow: OAuthFlowType.DeviceCode, + * tokenCache: { cacheDir: "/tmp/my-app/oauth-cache" }, + * }; + * const session = new OAuthSession(config); + * const status = await session.login(); + * ``` + */ +export class OAuthSession { + private readonly inner: NativeOAuthSession; + + /** Create a session manager for the given OAuth configuration. */ + constructor(config: OAuthConfig) { + this.inner = new NativeOAuthSession(config as unknown as NativeOAuthConfig); + } + + /** + * Eagerly run the configured authentication flow and store the session. + * + * A successful login always replaces any prior cached session for this + * identity; if the provider does not issue a refresh token (for example + * without `offline_access`), the previous record is removed and the status + * reports `refreshable == false`. + */ + async login(): Promise { + return this.inner.login(); + } + + /** + * Report whether a matching cached session exists, with safe metadata. + * + * This never contacts the identity provider and never exposes token values. + */ + async status(): Promise { + return this.inner.status(); + } + + /** + * Remove the matching local cached credential. + * + * This only deletes the local cache entry. It does not revoke the refresh + * token with the provider and does not sign out of a browser SSO session. + * Repeated calls succeed; `removed` reports whether a credential existed. + */ + async logout(): Promise { + return this.inner.logout(); + } } diff --git a/nodejs/src/remote.rs b/nodejs/src/remote.rs index 4bdb5685e..c619aca35 100644 --- a/nodejs/src/remote.rs +++ b/nodejs/src/remote.rs @@ -6,6 +6,8 @@ use std::collections::HashMap; use lancedb::error::Error; use napi_derive::*; +use crate::error::NapiErrorExt; + /// Timeout configuration for remote HTTP client. #[napi(object)] #[derive(Debug)] @@ -141,6 +143,34 @@ impl From for lancedb::remote::TlsConfig { } } +/// Options for the persistent OAuth token cache. +/// +/// The cache is opt-in: it is only used when set as `tokenCache` on +/// `OAuthConfig`. Only refresh tokens are persisted, in a private directory +/// with owner-only permissions, so short-lived processes can reuse an +/// authenticated session instead of re-prompting on every start. +#[napi(object)] +#[derive(Clone, Debug, Default)] +pub struct TokenCacheOptions { + /// Directory that holds cached credentials. Defaults to + /// `$XDG_CACHE_HOME/lancedb/oauth`, `$HOME/.cache/lancedb/oauth` on Unix, + /// or `%LOCALAPPDATA%\lancedb\oauth` on Windows. The directory is created + /// with owner-only permissions (`0700`) when missing. + pub cache_dir: Option, + /// How long to wait for the cross-process refresh lock before failing, + /// in seconds (default: 30). + pub lock_timeout_secs: Option, +} + +impl From for lancedb::remote::TokenCacheOptions { + fn from(options: TokenCacheOptions) -> Self { + Self { + cache_dir: options.cache_dir.map(std::path::PathBuf::from), + lock_timeout_secs: options.lock_timeout_secs.map(|secs| secs as u64), + } + } +} + /// OAuth configuration for LanceDB authentication. /// /// This is the generated napi-rs binding shape. TypeScript users should prefer @@ -158,16 +188,26 @@ pub struct OAuthConfig { /// OAuth scopes to request. For Azure managed identity, exactly one scope /// or resource is required. For example: `["api://{app_id}/.default"]` pub scopes: Vec, - /// Authentication flow: "client_credentials" or "azure_managed_identity" + /// Authentication flow: "client_credentials", "authorization_code", + /// "device_code", or "azure_managed_identity" pub flow: Option, /// Client secret (required for client_credentials). pub client_secret: Option, + /// Loopback redirect URI for authorization_code. + pub redirect_uri: Option, + /// Port for the authorization_code loopback callback server. + pub callback_port: Option, + /// Whether authorization_code uses S256 PKCE (default: true). + pub use_pkce: Option, /// Client ID for user-assigned managed identity (azure_managed_identity). pub managed_identity_client_id: Option, /// Seconds before expiry to trigger proactive refresh (default: 300). /// Keep this well below the token TTL; if it is greater than or equal to /// the TTL, each request refreshes the token. pub refresh_buffer_secs: Option, + /// Opt in to the persistent token cache so short-lived processes reuse + /// one session. Only refresh tokens are persisted. + pub token_cache: Option, } impl std::fmt::Debug for OAuthConfig { @@ -181,11 +221,15 @@ impl std::fmt::Debug for OAuthConfig { "client_secret", &self.client_secret.as_deref().map(|_| ""), ) + .field("redirect_uri", &self.redirect_uri) + .field("callback_port", &self.callback_port) + .field("use_pkce", &self.use_pkce) .field( "managed_identity_client_id", &self.managed_identity_client_id, ) .field("refresh_buffer_secs", &self.refresh_buffer_secs) + .field("token_cache", &self.token_cache) .finish() } } @@ -194,10 +238,22 @@ impl TryFrom for lancedb::remote::oauth::OAuthConfig { type Error = Error; fn try_from(config: OAuthConfig) -> Result { - use lancedb::remote::oauth::OAuthFlow; + use lancedb::remote::oauth::{AuthorizationCodeOptions, OAuthFlow}; let flow = match config.flow.as_deref().unwrap_or("client_credentials") { "client_credentials" => OAuthFlow::ClientCredentials, + "authorization_code" => { + let mut options = + AuthorizationCodeOptions::new().use_pkce(config.use_pkce.unwrap_or(true)); + if let Some(redirect_uri) = config.redirect_uri { + options = options.redirect_uri(redirect_uri); + } + if let Some(callback_port) = config.callback_port { + options = options.callback_port(callback_port); + } + OAuthFlow::AuthorizationCode(options) + } + "device_code" => OAuthFlow::DeviceCode, "azure_managed_identity" => OAuthFlow::AzureManagedIdentity { client_id: config.managed_identity_client_id, }, @@ -215,10 +271,115 @@ impl TryFrom for lancedb::remote::oauth::OAuthConfig { scopes: config.scopes, flow, refresh_buffer_secs: config.refresh_buffer_secs.map(|v| v as u64), + token_cache: config.token_cache.map(Into::into), }) } } +/// Safe, non-secret view of a cached OAuth session, returned by +/// `OAuthSession.status()` and `OAuthSession.login()`. +#[napi(object)] +#[derive(Clone, Debug)] +pub struct SessionStatus { + /// Whether a cached session exists that can obtain tokens without + /// interactive authentication. + pub refreshable: bool, + /// Canonical issuer URL of the cached session. + pub issuer_url: String, + /// Client ID of the cached session. + pub client_id: String, + /// Canonical (sorted, de-duplicated) scopes of the cached session. + pub scopes: Vec, + /// Flow that produced the cached session. + pub flow: String, + /// When the cached session was obtained, as Unix seconds. + pub obtained_at: Option, +} + +/// Result of `OAuthSession.logout()`. +#[napi(object)] +#[derive(Clone, Debug)] +pub struct SessionLogout { + /// Whether a cached credential was removed. `false` means no matching + /// session was cached; logout is idempotent. + pub removed: bool, +} + +/// Explicit OAuth session lifecycle for the persistent token cache: eager +/// `login`, non-secret `status`, and local `logout`. +/// +/// A session is built from the same `OAuthConfig` used to connect (including +/// its `tokenCache` options). A connection created with the same +/// configuration shares the cache, so logging in here prepares tokens for +/// later processes without any database request. +#[napi] +pub struct OAuthSession { + inner: lancedb::remote::OAuthSession, +} + +#[napi] +impl OAuthSession { + /// Create a session manager for the given OAuth configuration. + /// + /// The configuration must enable `tokenCache` options and use a flow that + /// supports persistent sessions (authorization code or device code). + #[napi(constructor)] + pub fn new(config: OAuthConfig) -> napi::Result { + let config: lancedb::remote::oauth::OAuthConfig = config.try_into().default_error()?; + let inner = lancedb::remote::OAuthSession::new(config).default_error()?; + Ok(Self { inner }) + } + + /// Eagerly run the configured authentication flow and store the session. + /// + /// A successful login always replaces any prior cached session for this + /// identity; if the provider does not issue a refresh token (for example + /// without `offline_access`), the previous record is removed and the + /// status reports `refreshable == false`. + #[napi(catch_unwind)] + pub async fn login(&self) -> napi::Result { + let status = self.inner.login().await.default_error()?; + Ok(SessionStatus::from(status)) + } + + /// Report whether a matching cached session exists, with safe metadata. + /// + /// This never contacts the identity provider and never exposes token + /// values. + #[napi(catch_unwind)] + pub async fn status(&self) -> napi::Result { + let status = self.inner.status().await.default_error()?; + Ok(SessionStatus::from(status)) + } + + /// Remove the matching local cached credential. + /// + /// This only deletes the local cache entry. It does not revoke the + /// refresh token with the provider and does not sign out of a browser + /// SSO session. Repeated calls succeed; `removed` reports whether a + /// credential existed. + #[napi(catch_unwind)] + pub async fn logout(&self) -> napi::Result { + let logout = self.inner.logout().await.default_error()?; + Ok(SessionLogout { + removed: logout.removed, + }) + } +} + +impl From for SessionStatus { + fn from(status: lancedb::remote::SessionStatus) -> Self { + Self { + refreshable: status.refreshable, + issuer_url: status.issuer_url, + client_id: status.client_id, + scopes: status.scopes, + flow: status.flow, + obtained_at: status.obtained_at.map(|secs| secs as f64), + } + } +} + impl From for lancedb::remote::ClientConfig { fn from(config: ClientConfig) -> Self { Self { @@ -252,8 +413,12 @@ mod tests { scopes: vec!["scope".to_string()], flow: Some("typo".to_string()), client_secret: None, + redirect_uri: None, + callback_port: None, + use_pkce: None, managed_identity_client_id: None, refresh_buffer_secs: None, + token_cache: None, }; let err = lancedb::remote::oauth::OAuthConfig::try_from(config).unwrap_err(); @@ -272,12 +437,67 @@ mod tests { scopes: vec!["scope".to_string()], flow: Some("client_credentials".to_string()), client_secret: Some("super-secret".to_string()), + redirect_uri: None, + callback_port: None, + use_pkce: None, managed_identity_client_id: None, refresh_buffer_secs: None, + token_cache: None, }; let debug = format!("{config:?}"); assert!(!debug.contains("super-secret")); assert!(debug.contains("client_secret: Some(\"\")")); } + + #[test] + fn test_authorization_code_conversion_preserves_options() { + let config = OAuthConfig { + issuer_url: "https://issuer.example.com".to_string(), + client_id: "client-id".to_string(), + scopes: vec!["openid".to_string()], + flow: Some("authorization_code".to_string()), + client_secret: Some("secret".to_string()), + redirect_uri: Some("http://127.0.0.1:9000/callback".to_string()), + callback_port: Some(9000), + use_pkce: Some(false), + managed_identity_client_id: None, + refresh_buffer_secs: None, + token_cache: None, + }; + + let converted = lancedb::remote::oauth::OAuthConfig::try_from(config).unwrap(); + let lancedb::remote::oauth::OAuthFlow::AuthorizationCode(options) = converted.flow else { + panic!("expected authorization code flow"); + }; + assert_eq!( + options.redirect_uri.as_deref(), + Some("http://127.0.0.1:9000/callback") + ); + assert_eq!(options.callback_port, Some(9000)); + assert!(!options.use_pkce); + } + + #[test] + fn test_device_code_conversion() { + let config = OAuthConfig { + issuer_url: "https://issuer.example.com".to_string(), + client_id: "client-id".to_string(), + scopes: vec!["openid".to_string()], + flow: Some("device_code".to_string()), + client_secret: None, + redirect_uri: None, + callback_port: None, + use_pkce: None, + managed_identity_client_id: None, + refresh_buffer_secs: None, + token_cache: None, + }; + + let converted = lancedb::remote::oauth::OAuthConfig::try_from(config).unwrap(); + assert!(matches!( + converted.flow, + lancedb::remote::oauth::OAuthFlow::DeviceCode + )); + } } diff --git a/nodejs/typedoc.json b/nodejs/typedoc.json index e46085cda..c601ee61e 100644 --- a/nodejs/typedoc.json +++ b/nodejs/typedoc.json @@ -4,7 +4,8 @@ "lancedb/native.d.ts:VectorQuery", "lancedb/native.d.ts:TakeQuery", "lancedb/native.d.ts:RecordBatchIterator", - "lancedb/native.d.ts:NativeMergeInsertBuilder" + "lancedb/native.d.ts:NativeMergeInsertBuilder", + "lancedb/native.d.ts:TokenCacheOptions" ], "useHTMLEncodedBrackets": true, "useCodeBlocks": true, diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 0f7b110ac..08d81bfd3 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -267,6 +267,33 @@ class JobInfo: @property def created_at_millis(self) -> int: ... +class SessionStatus: + @property + def refreshable(self) -> bool: ... + @property + def issuer_url(self) -> str: ... + @property + def client_id(self) -> str: ... + @property + def scopes(self) -> List[str]: ... + @property + def flow(self) -> str: ... + @property + def obtained_at(self) -> Optional[int]: ... + def __repr__(self) -> str: ... + +class SessionLogout: + @property + def removed(self) -> bool: ... + def __repr__(self) -> str: ... + +class OAuthSession: + def __init__(self, config: Any) -> None: ... + async def login(self) -> SessionStatus: ... + async def status(self) -> SessionStatus: ... + async def logout(self) -> SessionLogout: ... + def __repr__(self) -> str: ... + class JobFailureInfo: @property def phase(self) -> Optional[str]: ... diff --git a/python/python/lancedb/remote/__init__.py b/python/python/lancedb/remote/__init__.py index 1f255b991..602d8dbab 100644 --- a/python/python/lancedb/remote/__init__.py +++ b/python/python/lancedb/remote/__init__.py @@ -9,7 +9,7 @@ from typing import List, Optional from lancedb import __version__ from .header import HeaderProvider -from .oauth import OAuthConfig, OAuthFlowType +from .oauth import OAuthConfig, OAuthFlowType, OAuthSession, TokenCacheOptions # The API reference renders this module with a single mkdocstrings directive, # which only picks up names listed here. New public names must be added to this @@ -22,6 +22,8 @@ __all__ = [ "HeaderProvider", "OAuthConfig", "OAuthFlowType", + "OAuthSession", + "TokenCacheOptions", ] diff --git a/python/python/lancedb/remote/oauth.py b/python/python/lancedb/remote/oauth.py index 9175c3614..ad9fd2b0e 100644 --- a/python/python/lancedb/remote/oauth.py +++ b/python/python/lancedb/remote/oauth.py @@ -12,10 +12,49 @@ class OAuthFlowType(str, Enum): CLIENT_CREDENTIALS = "client_credentials" """Client Credentials grant (service-to-service / M2M).""" + AUTHORIZATION_CODE = "authorization_code" + """Interactive Authorization Code grant, using PKCE by default.""" + + DEVICE_CODE = "device_code" + """Device Authorization grant for CLI and headless environments.""" + AZURE_MANAGED_IDENTITY = "azure_managed_identity" """Azure Managed Identity via IMDS.""" +@dataclass +class TokenCacheOptions: + """Options for the persistent OAuth token cache. + + The cache is opt-in: it is only used when set as ``token_cache`` on + :class:`OAuthConfig`. Only refresh tokens are persisted, in a private + directory with owner-only permissions, so short-lived processes can reuse + an authenticated session instead of re-prompting on every start. + + Parameters + ---------- + cache_dir : Optional[str] + Directory that holds cached credentials. Defaults to + ``$XDG_CACHE_HOME/lancedb/oauth``, ``$HOME/.cache/lancedb/oauth`` on + Unix, or ``%LOCALAPPDATA%\\lancedb\\oauth`` on Windows. The directory + is created with owner-only permissions (``0700``) when missing. + lock_timeout_secs : Optional[int] + How long to wait for the cross-process refresh lock before failing + (default: 30 seconds). + + Examples + -------- + >>> opts = TokenCacheOptions(cache_dir="/tmp/my-app/oauth-cache") + + Multiple identities (issuer, client, scopes, flow, client + authentication) get separate cache entries. Within one identity the most + recent login wins. + """ + + cache_dir: Optional[str] = None + lock_timeout_secs: Optional[int] = None + + @dataclass class OAuthConfig: """OAuth configuration for LanceDB authentication. @@ -38,12 +77,23 @@ class OAuthConfig: Authentication flow to use. Default: CLIENT_CREDENTIALS. client_secret : Optional[str] Client secret (required for CLIENT_CREDENTIALS). + redirect_uri : Optional[str] + Loopback redirect URI for AUTHORIZATION_CODE. The default is + ``http://127.0.0.1:{callback_port}/callback``. + callback_port : Optional[int] + Port for the AUTHORIZATION_CODE loopback callback server (default: 8400). + use_pkce : bool + Protect AUTHORIZATION_CODE with S256 PKCE (default: True). managed_identity_client_id : Optional[str] Client ID for user-assigned managed identity (AZURE_MANAGED_IDENTITY). refresh_buffer_secs : Optional[int] Seconds before expiry to trigger proactive refresh (default: 300). Keep this well below the token TTL; if it is greater than or equal to the TTL, each request refreshes the token. + token_cache : Optional[TokenCacheOptions] + Opt in to the persistent token cache so short-lived processes reuse + one session. Only supported by AUTHORIZATION_CODE and DEVICE_CODE; + azure managed identity is rejected. Default: None (memory only). Examples -------- @@ -64,6 +114,29 @@ class OAuthConfig: ... scopes=["api://lancedb-api/.default"], ... flow=OAuthFlowType.AZURE_MANAGED_IDENTITY, ... ) + + Authorization Code with PKCE: + + The authorization URL is written to standard error before LanceDB tries to + open a browser, so it can be copied in headless environments. + + >>> config = OAuthConfig( + ... issuer_url="https://login.microsoftonline.com/{tenant}/v2.0", + ... client_id="app-id", + ... scopes=["openid", "api://lancedb-api/access"], + ... flow=OAuthFlowType.AUTHORIZATION_CODE, + ... ) + + Device Authorization with a persistent cache, so later processes reuse + the session without a new device prompt: + + >>> config = OAuthConfig( + ... issuer_url="https://login.microsoftonline.com/{tenant}/v2.0", + ... client_id="app-id", + ... scopes=["openid", "offline_access", "api://lancedb-api/access"], + ... flow=OAuthFlowType.DEVICE_CODE, + ... token_cache=TokenCacheOptions(), + ... ) """ issuer_url: str @@ -71,5 +144,70 @@ class OAuthConfig: scopes: List[str] flow: OAuthFlowType = OAuthFlowType.CLIENT_CREDENTIALS client_secret: Optional[str] = field(default=None, repr=False) + redirect_uri: Optional[str] = None + callback_port: Optional[int] = None + use_pkce: bool = True managed_identity_client_id: Optional[str] = None refresh_buffer_secs: Optional[int] = None + token_cache: Optional[TokenCacheOptions] = None + + +class OAuthSession: + """Explicit OAuth session lifecycle for the persistent token cache. + + Built from the same :class:`OAuthConfig` used for + :func:`lancedb.connect_async` (including its ``token_cache`` options). + A connection created with the same configuration shares the cache, so + logging in here prepares tokens for later processes without any database + request. + + ``login`` always runs the configured interactive flow and replaces the + cached session (the most recent login wins). ``logout`` removes only the + local credential; it does not revoke anything with the provider and does + not sign out of a browser SSO session. + + Examples + -------- + >>> config = OAuthConfig( + ... issuer_url="https://issuer.example.com", + ... client_id="my-app", + ... scopes=["openid", "offline_access"], + ... flow=OAuthFlowType.DEVICE_CODE, + ... token_cache=TokenCacheOptions(), + ... ) + >>> session = OAuthSession(config) # doctest: +SKIP + >>> status = await session.login() # doctest: +SKIP + >>> status.refreshable # doctest: +SKIP + True + """ + + def __init__(self, config: OAuthConfig): + from lancedb._lancedb import OAuthSession as PyOAuthSession + + self._inner: PyOAuthSession = PyOAuthSession(config) + + async def login(self): + """Eagerly run the configured flow and store the session. + + Returns a :class:`lancedb._lancedb.SessionStatus` describing the + cached session. A successful login always replaces any prior cached + session for this identity; if the provider does not issue a refresh + token (for example without ``offline_access``), the previous record + is removed and ``refreshable`` is ``False``. + """ + return await self._inner.login() + + async def status(self): + """Report whether a cached session exists, with safe metadata. + + Never contacts the identity provider and never exposes token values. + """ + return await self._inner.status() + + async def logout(self): + """Remove the matching local cached credential. + + Returns a :class:`lancedb._lancedb.SessionLogout` whose ``removed`` + flag reports whether a credential existed. Logout is idempotent. + """ + return await self._inner.logout() diff --git a/python/src/lib.rs b/python/src/lib.rs index 06b31d033..f83776e7b 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -47,6 +47,9 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/python/src/oauth.rs b/python/src/oauth.rs index 11ea011e2..ff24a9566 100644 --- a/python/src/oauth.rs +++ b/python/src/oauth.rs @@ -1,10 +1,33 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -use pyo3::FromPyObject; +use std::path::PathBuf; +use std::sync::Arc; +use pyo3::{FromPyObject, PyResult, Python, pyclass, pymethods}; + +use crate::error::PythonErrorExt; +use crate::runtime::future_into_py; use lancedb::error::Error; -use lancedb::remote::oauth::{OAuthConfig, OAuthFlow}; +use lancedb::remote::oauth::{AuthorizationCodeOptions, OAuthConfig, OAuthFlow}; +use lancedb::remote::{OAuthSession, SessionLogout, SessionStatus, TokenCacheOptions}; + +/// Python-side persistent token cache options, extracted via FromPyObject. +/// Maps to `lancedb.remote.oauth.TokenCacheOptions` Python dataclass. +#[derive(FromPyObject, Default)] +pub struct PyTokenCacheOptions { + pub cache_dir: Option, + pub lock_timeout_secs: Option, +} + +impl From for TokenCacheOptions { + fn from(py: PyTokenCacheOptions) -> Self { + TokenCacheOptions { + cache_dir: py.cache_dir.map(PathBuf::from), + lock_timeout_secs: py.lock_timeout_secs, + } + } +} /// Python-side OAuth configuration, extracted via FromPyObject. /// Maps to `lancedb.remote.oauth.OAuthConfig` Python dataclass. @@ -15,8 +38,12 @@ pub struct PyOAuthConfig { pub scopes: Vec, pub flow: String, pub client_secret: Option, + pub redirect_uri: Option, + pub callback_port: Option, + pub use_pkce: bool, pub managed_identity_client_id: Option, pub refresh_buffer_secs: Option, + pub token_cache: Option, } impl TryFrom for OAuthConfig { @@ -25,6 +52,17 @@ impl TryFrom for OAuthConfig { fn try_from(py: PyOAuthConfig) -> Result { let flow = match py.flow.as_str() { "client_credentials" => OAuthFlow::ClientCredentials, + "authorization_code" => { + let mut options = AuthorizationCodeOptions::new().use_pkce(py.use_pkce); + if let Some(redirect_uri) = py.redirect_uri { + options = options.redirect_uri(redirect_uri); + } + if let Some(callback_port) = py.callback_port { + options = options.callback_port(callback_port); + } + OAuthFlow::AuthorizationCode(options) + } + "device_code" => OAuthFlow::DeviceCode, "azure_managed_identity" => OAuthFlow::AzureManagedIdentity { client_id: py.managed_identity_client_id, }, @@ -42,6 +80,148 @@ impl TryFrom for OAuthConfig { scopes: py.scopes, flow, refresh_buffer_secs: py.refresh_buffer_secs, + token_cache: py.token_cache.map(TokenCacheOptions::from), + }) + } +} + +/// Wrapper around [`lancedb::remote::SessionStatus`] exposing safe metadata. +#[pyclass(name = "SessionStatus", skip_from_py_object)] +#[derive(Clone)] +pub struct PySessionStatus { + inner: SessionStatus, +} + +#[pymethods] +impl PySessionStatus { + /// Whether a cached session exists that can obtain tokens without + /// interactive authentication. + #[getter] + pub fn refreshable(&self) -> bool { + self.inner.refreshable + } + + /// Canonical issuer URL of the cached session. + #[getter] + pub fn issuer_url(&self) -> String { + self.inner.issuer_url.clone() + } + + /// Client ID of the cached session. + #[getter] + pub fn client_id(&self) -> String { + self.inner.client_id.clone() + } + + /// Canonical (sorted, de-duplicated) scopes of the cached session. + #[getter] + pub fn scopes(&self) -> Vec { + self.inner.scopes.clone() + } + + /// Flow that produced the cached session. + #[getter] + pub fn flow(&self) -> String { + self.inner.flow.clone() + } + + /// When the cached session was obtained, as Unix seconds. + #[getter] + pub fn obtained_at(&self) -> Option { + self.inner.obtained_at + } + + pub fn __repr__(&self) -> String { + format!( + "SessionStatus(refreshable={}, issuer_url='{}', client_id='{}', flow='{}')", + self.inner.refreshable, self.inner.issuer_url, self.inner.client_id, self.inner.flow + ) + } +} + +impl From for PySessionStatus { + fn from(inner: SessionStatus) -> Self { + Self { inner } + } +} + +/// Wrapper around [`lancedb::remote::SessionLogout`]. +#[pyclass(name = "SessionLogout", skip_from_py_object)] +#[derive(Clone)] +pub struct PySessionLogout { + inner: SessionLogout, +} + +#[pymethods] +impl PySessionLogout { + /// Whether a cached credential was removed. + #[getter] + pub fn removed(&self) -> bool { + self.inner.removed + } + + pub fn __repr__(&self) -> String { + format!("SessionLogout(removed={})", self.inner.removed) + } +} + +impl From for PySessionLogout { + fn from(inner: SessionLogout) -> Self { + Self { inner } + } +} + +/// Wrapper around [`lancedb::remote::OAuthSession`]. +#[pyclass(name = "OAuthSession", skip_from_py_object)] +#[derive(Clone)] +pub struct PyOAuthSession { + inner: Arc, +} + +#[pymethods] +impl PyOAuthSession { + /// Create a session manager for the given OAuth configuration. + /// + /// The configuration must set ``token_cache`` options and use a flow that + /// supports persistent sessions (authorization code or device code). + #[new] + pub fn new(config: PyOAuthConfig) -> PyResult { + let config: OAuthConfig = config.try_into().infer_error()?; + let inner = OAuthSession::new(config).infer_error()?; + Ok(Self { + inner: Arc::new(inner), + }) + } + + /// Eagerly run the configured authentication flow and store the session. + pub fn login<'py>(&self, py: Python<'py>) -> PyResult> { + let inner = Arc::clone(&self.inner); + future_into_py(py, async move { + inner.login().await.map(PySessionStatus::from).infer_error() + }) + } + + /// Report whether a matching cached session exists, with safe metadata. + pub fn status<'py>(&self, py: Python<'py>) -> PyResult> { + let inner = Arc::clone(&self.inner); + future_into_py(py, async move { + inner + .status() + .await + .map(PySessionStatus::from) + .infer_error() + }) + } + + /// Remove the matching local cached credential. + pub fn logout<'py>(&self, py: Python<'py>) -> PyResult> { + let inner = Arc::clone(&self.inner); + future_into_py(py, async move { + inner + .logout() + .await + .map(PySessionLogout::from) + .infer_error() }) } } @@ -50,16 +230,27 @@ impl TryFrom for OAuthConfig { mod tests { use super::*; - #[test] - fn test_unknown_oauth_flow_returns_invalid_input() { - let config = PyOAuthConfig { + fn base_config() -> PyOAuthConfig { + PyOAuthConfig { issuer_url: "https://issuer.example.com".to_string(), client_id: "client-id".to_string(), scopes: vec!["scope".to_string()], - flow: "typo".to_string(), + flow: "device_code".to_string(), client_secret: None, + redirect_uri: None, + callback_port: None, + use_pkce: true, managed_identity_client_id: None, refresh_buffer_secs: None, + token_cache: None, + } + } + + #[test] + fn test_unknown_oauth_flow_returns_invalid_input() { + let config = PyOAuthConfig { + flow: "typo".to_string(), + ..base_config() }; let err = OAuthConfig::try_from(config).unwrap_err(); @@ -69,4 +260,53 @@ mod tests { if message == "Unknown OAuth flow type: typo" )); } + + #[test] + fn test_authorization_code_conversion_preserves_options() { + let config = PyOAuthConfig { + flow: "authorization_code".to_string(), + client_secret: Some("secret".to_string()), + redirect_uri: Some("http://127.0.0.1:9000/callback".to_string()), + callback_port: Some(9000), + use_pkce: false, + ..base_config() + }; + + let converted = OAuthConfig::try_from(config).unwrap(); + let OAuthFlow::AuthorizationCode(options) = converted.flow else { + panic!("expected authorization code flow"); + }; + assert_eq!( + options.redirect_uri.as_deref(), + Some("http://127.0.0.1:9000/callback") + ); + assert_eq!(options.callback_port, Some(9000)); + assert!(!options.use_pkce); + } + + #[test] + fn test_device_code_conversion() { + let config = base_config(); + let converted = OAuthConfig::try_from(config).unwrap(); + assert!(matches!(converted.flow, OAuthFlow::DeviceCode)); + } + + #[test] + fn test_token_cache_conversion() { + let config = PyOAuthConfig { + token_cache: Some(PyTokenCacheOptions { + cache_dir: Some("/tmp/oauth-cache".to_string()), + lock_timeout_secs: Some(5), + }), + ..base_config() + }; + + let converted = OAuthConfig::try_from(config).unwrap(); + let cache = converted.token_cache.expect("token cache options"); + assert_eq!( + cache.cache_dir.as_deref(), + Some(std::path::Path::new("/tmp/oauth-cache")) + ); + assert_eq!(cache.lock_timeout_secs, Some(5)); + } } diff --git a/python/tests/test_oauth.py b/python/tests/test_oauth.py index 89f5b3f8d..36f687042 100644 --- a/python/tests/test_oauth.py +++ b/python/tests/test_oauth.py @@ -1,10 +1,19 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The LanceDB Authors +import asyncio import importlib.util +import json +import os +import subprocess import sys +import threading +import urllib.parse +from http.server import BaseHTTPRequestHandler, HTTPServer from pathlib import Path +import pytest + def _load_oauth_module(): oauth_path = ( @@ -31,3 +40,289 @@ def test_oauth_config_repr_redacts_client_secret(): rendered = repr(config) assert "super-secret" not in rendered assert "client_secret" not in rendered + + +def test_authorization_code_uses_pkce_by_default(): + oauth = _load_oauth_module() + + config = oauth.OAuthConfig( + issuer_url="https://issuer.example.com", + client_id="client-id", + scopes=["openid"], + flow=oauth.OAuthFlowType.AUTHORIZATION_CODE, + ) + + assert config.use_pkce is True + assert config.redirect_uri is None + assert config.callback_port is None + + +def test_device_code_flow_value(): + oauth = _load_oauth_module() + + assert oauth.OAuthFlowType.DEVICE_CODE.value == "device_code" + + +def test_token_cache_options_default_to_memory_only(): + oauth = _load_oauth_module() + + config = oauth.OAuthConfig( + issuer_url="https://issuer.example.com", + client_id="client-id", + scopes=["openid"], + ) + assert config.token_cache is None + + options = oauth.TokenCacheOptions() + assert options.cache_dir is None + assert options.lock_timeout_secs is None + + +def _remote_oauth(): + pytest.importorskip("lancedb") + from lancedb.remote import oauth as remote_oauth + + return remote_oauth + + +def _device_config(remote_oauth, issuer_url, cache_dir): + return remote_oauth.OAuthConfig( + issuer_url=issuer_url, + client_id="client-id", + scopes=["openid"], + flow=remote_oauth.OAuthFlowType.DEVICE_CODE, + token_cache=remote_oauth.TokenCacheOptions(cache_dir=str(cache_dir)), + ) + + +def test_oauth_session_status_and_logout_without_cache_entry(tmp_path): + remote_oauth = _remote_oauth() + config = _device_config(remote_oauth, "https://issuer.example.com", tmp_path) + + session = remote_oauth.OAuthSession(config) + status = asyncio.run(session.status()) + assert status.refreshable is False + assert status.issuer_url == "https://issuer.example.com" + assert status.client_id == "client-id" + assert status.scopes == ["openid"] + assert status.flow == "device_code" + assert status.obtained_at is None + + logout = asyncio.run(session.logout()) + assert logout.removed is False + + +class _MockIdpState: + def __init__(self, port): + self.port = port + self.lock = threading.Lock() + self.device_authorizations = 0 + self.refresh_grants = 0 + self.invalid_grant_rejections = 0 + self.access_tokens_issued = 0 + self.current_refresh = None + + +class _MockIdpHandler(BaseHTTPRequestHandler): + @property + def state(self) -> _MockIdpState: + return self.server.state + + def log_message(self, fmt, *args): + pass + + def _respond(self, status, payload): + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_GET(self): + if self.path.startswith("/.well-known/openid-configuration"): + base = f"http://127.0.0.1:{self.state.port}" + self._respond( + 200, + { + "token_endpoint": f"{base}/token", + "device_authorization_endpoint": f"{base}/device", + }, + ) + else: + self._respond(404, {}) + + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length).decode() + params = urllib.parse.parse_qs(body) + + if self.path == "/device": + with self.state.lock: + self.state.device_authorizations += 1 + base = f"http://127.0.0.1:{self.state.port}" + self._respond( + 200, + { + "device_code": "device-code", + "user_code": "ABCD-EFGH", + "verification_uri": f"{base}/verify", + "expires_in": 60, + "interval": 1, + }, + ) + return + + if self.path == "/token": + grant_type = params.get("grant_type", [""])[0] + with self.state.lock: + if grant_type == "refresh_token": + self.state.refresh_grants += 1 + offered = params.get("refresh_token", [""])[0] + if offered != self.state.current_refresh: + self.state.invalid_grant_rejections += 1 + self._respond(400, {"error": "invalid_grant"}) + return + elif "device_code" not in grant_type: + self._respond(400, {"error": "unsupported_grant_type"}) + return + self.state.access_tokens_issued += 1 + number = self.state.access_tokens_issued + refresh = f"refresh-{number}" + self.state.current_refresh = refresh + self._respond( + 200, + { + "access_token": f"access-{number}", + "refresh_token": refresh, + "expires_in": 3600, + }, + ) + return + + self._respond(404, {}) + + +def _start_mock_idp() -> tuple[_MockIdpState, HTTPServer]: + server = HTTPServer(("127.0.0.1", 0), _MockIdpHandler) + state = _MockIdpState(server.server_address[1]) + server.state = state + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return state, server + + +def _run_subprocess(script: Path, issuer_url: str, cache_dir: Path): + env = dict(os.environ) + env["LANCEDB_OAUTH_BROWSER"] = "/usr/bin/true" + result = subprocess.run( + [sys.executable, str(script), issuer_url, str(cache_dir)], + capture_output=True, + text=True, + timeout=120, + env=env, + ) + assert result.returncode == 0, ( + f"subprocess failed:\nstdout: {result.stdout}\nstderr: {result.stderr}" + ) + return result + + +LOGIN_SCRIPT = """ +import asyncio +import sys + +from lancedb.remote import OAuthConfig, OAuthFlowType, OAuthSession, TokenCacheOptions + +issuer_url, cache_dir = sys.argv[1], sys.argv[2] +config = OAuthConfig( + issuer_url=issuer_url, + client_id="client-id", + scopes=["openid"], + flow=OAuthFlowType.DEVICE_CODE, + token_cache=TokenCacheOptions(cache_dir=cache_dir), +) +session = OAuthSession(config) +status = asyncio.run(session.login()) +assert status.refreshable, "login must cache a refresh token" +print("LOGIN-OK") +""" + +REUSE_SCRIPT = """ +import asyncio +import sys + +import lancedb +from lancedb.remote import OAuthConfig, OAuthFlowType, OAuthSession, TokenCacheOptions + +issuer_url, cache_dir = sys.argv[1], sys.argv[2] +config = OAuthConfig( + issuer_url=issuer_url, + client_id="client-id", + scopes=["openid"], + flow=OAuthFlowType.DEVICE_CODE, + token_cache=TokenCacheOptions(cache_dir=cache_dir), +) + +session = OAuthSession(config) +status = asyncio.run(session.status()) +assert status.refreshable, "second process must see the cached session" + + +async def main(): + # Point the database endpoint at a dead port. OAuth headers are fetched + # before the request is sent, so a successful refresh proves the second + # process reused the cached session; only the database call fails. + db = await lancedb.connect_async( + "db://e2e", + host_override="http://127.0.0.1:1", + client_config={"retry_config": {"retries": 0}}, + oauth_config=config, + ) + try: + await db.table_names() + except Exception: + print("DATABASE-UNREACHABLE-AS-EXPECTED") + else: + raise AssertionError("expected the database request to fail") + + +asyncio.run(main()) +print("REUSE-OK") +""" + + +def test_cross_process_session_reuse_without_new_prompt(tmp_path): + pytest.importorskip("lancedb") + state, server = _start_mock_idp() + try: + issuer_url = f"http://127.0.0.1:{state.port}" + login_script = tmp_path / "login.py" + login_script.write_text(LOGIN_SCRIPT) + reuse_script = tmp_path / "reuse.py" + reuse_script.write_text(REUSE_SCRIPT) + cache_dir = tmp_path / "oauth-cache" + + result = _run_subprocess(login_script, issuer_url, cache_dir) + assert "LOGIN-OK" in result.stdout + assert state.device_authorizations == 1 + + result = _run_subprocess(reuse_script, issuer_url, cache_dir) + assert "REUSE-OK" in result.stdout + assert "DATABASE-UNREACHABLE-AS-EXPECTED" in result.stdout + + # The second process refreshed exactly once and never started a new + # interactive device flow. + assert state.refresh_grants == 1 + assert state.device_authorizations == 1 + assert state.invalid_grant_rejections == 0 + + logout = asyncio.run( + _remote_oauth() + .OAuthSession(_device_config(_remote_oauth(), issuer_url, cache_dir)) + .logout() + ) + assert logout.removed is True + finally: + server.shutdown() + server.server_close() diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 9bc4c4a8c..1934ff8ce 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -53,7 +53,7 @@ metrics = { workspace = true, optional = true } metrics-util = { workspace = true, optional = true } moka = { workspace = true } pin-project = { workspace = true } -tokio = { workspace = true } +tokio = { workspace = true, features = ["io-util", "net", "time"] } log.workspace = true async-trait = { workspace = true } bytes = { workspace = true } @@ -82,6 +82,9 @@ reqwest = { version = "0.12.0", default-features = false, features = [ tonic = { workspace = true, optional = true } http = { version = "1", optional = true } # Matching what is in reqwest urlencoding = { version = "2", optional = true } +base64 = { version = "0.22", optional = true } +fs4 = { version = "0.13", optional = true } +webbrowser = { version = "1", optional = true } uuid = { workspace = true, features = ["v5"] } polars-arrow = { version = ">=0.37,<0.40.0", optional = true } polars = { version = ">=0.37,<0.40.0", optional = true } @@ -159,6 +162,9 @@ remote = [ "dep:http", "dep:tonic", "dep:urlencoding", + "dep:base64", + "dep:webbrowser", + "dep:fs4", "lance-namespace-impls/rest", "lance-namespace-impls/rest-adapter", ] diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index df7da3d62..b0446919b 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -1546,6 +1546,7 @@ mod tests { scopes: vec!["scope".to_string()], flow: crate::remote::OAuthFlow::ClientCredentials, refresh_buffer_secs: None, + token_cache: None, }; let result = ConnectBuilder::new("db://my-container/my-prefix") @@ -1588,6 +1589,7 @@ mod tests { scopes: vec!["scope".to_string()], flow: crate::remote::OAuthFlow::ClientCredentials, refresh_buffer_secs: None, + token_cache: None, }; let client_config = crate::remote::ClientConfig { header_provider: Some( diff --git a/rust/lancedb/src/remote.rs b/rust/lancedb/src/remote.rs index 6c37ec6a0..db8d98f12 100644 --- a/rust/lancedb/src/remote.rs +++ b/rust/lancedb/src/remote.rs @@ -13,6 +13,7 @@ pub mod oauth; mod retry; pub(crate) mod sql; pub(crate) mod table; +pub(crate) mod token_cache; pub(crate) mod util; const ARROW_STREAM_CONTENT_TYPE: &str = "application/vnd.apache.arrow.stream"; @@ -31,4 +32,5 @@ fn extract_job_id(body: &str) -> Option { pub use client::{ClientConfig, HeaderProvider, RetryConfig, TimeoutConfig, TlsConfig}; pub use db::{RemoteDatabaseOptions, RemoteDatabaseOptionsBuilder}; -pub use oauth::{OAuthConfig, OAuthFlow, OAuthHeaderProvider}; +pub use oauth::{AuthorizationCodeOptions, OAuthConfig, OAuthFlow, OAuthHeaderProvider}; +pub use token_cache::{OAuthSession, SessionLogout, SessionStatus, TokenCacheOptions}; diff --git a/rust/lancedb/src/remote/oauth.rs b/rust/lancedb/src/remote/oauth.rs index 3ebe8f86e..75a8878a8 100644 --- a/rust/lancedb/src/remote/oauth.rs +++ b/rust/lancedb/src/remote/oauth.rs @@ -2,24 +2,144 @@ // SPDX-FileCopyrightText: Copyright The LanceDB Authors use std::collections::HashMap; -use std::net::IpAddr; +use std::net::{IpAddr, SocketAddr}; +use std::process::Command; use std::sync::Arc; use std::time::{Duration, Instant}; use async_trait::async_trait; -use log::debug; +use base64::Engine; +use log::{debug, warn}; +use rand::Rng; use reqwest::Client; use serde::Deserialize; +use sha2::{Digest, Sha256}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; use tokio::sync::RwLock; +use tokio::time::Instant as TokioInstant; +use url::Url; use crate::error::{Error, Result}; use crate::remote::client::HeaderProvider; const DEFAULT_REFRESH_BUFFER_SECS: u64 = 300; const DEFAULT_TOKEN_TTL_SECS: u64 = 3600; +const DEFAULT_CALLBACK_PORT: u16 = 8400; +const AUTHORIZATION_CALLBACK_TIMEOUT_SECS: u64 = 300; const AZURE_IMDS_ENDPOINT: &str = "http://169.254.169.254/metadata/identity/oauth2/token"; const AZURE_IMDS_API_VERSION: &str = "2018-02-01"; +fn oauth_url_uses_secure_transport(url: &Url) -> bool { + url.scheme() == "https" + || (url.scheme() == "http" + && match url.host() { + Some(url::Host::Domain(host)) => host.eq_ignore_ascii_case("localhost"), + Some(url::Host::Ipv4(ip)) => ip.is_loopback(), + Some(url::Host::Ipv6(ip)) => ip.is_loopback(), + None => false, + }) +} + +fn validate_oauth_url(value: &str, name: &str) -> Result { + let url = Url::parse(value).map_err(|e| Error::InvalidInput { + message: format!("Invalid OAuth {name}: {e}"), + })?; + if oauth_url_uses_secure_transport(&url) { + Ok(url) + } else { + Err(Error::InvalidInput { + message: format!("OAuth {name} must use https, except for http on a loopback host"), + }) + } +} + +fn authorization_prompt(url: &Url) -> String { + format!("Open this URL to authenticate with OAuth: {url}") +} + +fn device_prompt(verification_uri: &str, user_code: &str) -> String { + format!("To authenticate with OAuth, visit {verification_uri} and enter code {user_code}") +} + +fn write_oauth_prompt(mut output: impl std::io::Write, prompt: &str) { + let _ = writeln!(output, "{prompt}"); +} + +fn show_oauth_prompt(prompt: &str) { + write_oauth_prompt(std::io::stderr().lock(), prompt); +} + +/// Options for the interactive OAuth Authorization Code flow. +/// +/// The built-in callback server accepts only loopback HTTP redirect URIs. PKCE +/// with the S256 challenge method is enabled by default and should be disabled +/// only for providers that do not support it. +/// +/// # Example +/// +/// ``` +/// use lancedb::remote::{AuthorizationCodeOptions, OAuthFlow}; +/// +/// let flow = OAuthFlow::AuthorizationCode( +/// AuthorizationCodeOptions::new() +/// .callback_port(8400) +/// .use_pkce(true), +/// ); +/// ``` +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct AuthorizationCodeOptions { + /// Redirect URI registered with the identity provider. + /// + /// Defaults to `http://127.0.0.1:{callback_port}/callback`. + pub redirect_uri: Option, + + /// Port for the built-in loopback callback server. + /// + /// Defaults to 8400. When `redirect_uri` contains an explicit port, this + /// option must either be omitted or match that port. + pub callback_port: Option, + + /// Whether to protect the authorization code exchange with S256 PKCE. + pub use_pkce: bool, +} + +impl Default for AuthorizationCodeOptions { + fn default() -> Self { + Self { + redirect_uri: None, + callback_port: None, + use_pkce: true, + } + } +} + +impl AuthorizationCodeOptions { + /// Create authorization-code options with S256 PKCE enabled. + pub fn new() -> Self { + Self::default() + } + + /// Set the loopback redirect URI registered with the identity provider. + pub fn redirect_uri(mut self, redirect_uri: impl Into) -> Self { + self.redirect_uri = Some(redirect_uri.into()); + self + } + + /// Set the port for the built-in loopback callback server. + pub fn callback_port(mut self, callback_port: u16) -> Self { + self.callback_port = Some(callback_port); + self + } + + /// Enable or disable S256 PKCE. + pub fn use_pkce(mut self, use_pkce: bool) -> Self { + self.use_pkce = use_pkce; + self + } +} + /// OAuth authentication flow configuration. #[derive(Debug, Clone)] pub enum OAuthFlow { @@ -27,6 +147,15 @@ pub enum OAuthFlow { /// Requires `client_secret` in [`OAuthConfig`]. ClientCredentials, + /// Authorization Code grant using an interactive browser and a built-in + /// loopback callback server. The authorization URL is also written to + /// stderr so it remains available when the browser cannot be opened. + AuthorizationCode(AuthorizationCodeOptions), + + /// Device Authorization grant for CLI and headless environments. The + /// verification URI and user code are written to stderr. + DeviceCode, + /// Azure Managed Identity via IMDS. /// Works on Azure VMs, AKS, App Service, and Azure Functions. /// IMDS requests bypass proxy settings because the endpoint is link-local. @@ -65,6 +194,14 @@ pub struct OAuthConfig { /// Keep this well below the token TTL; if it is greater than or equal to /// the TTL, each request refreshes the token. pub refresh_buffer_secs: Option, + + /// Opt in to the persistent token cache so short-lived processes can + /// reuse an authenticated session instead of re-prompting. + /// + /// When unset (the default), tokens stay in process memory only. Only + /// refresh tokens are persisted; see + /// [`TokenCacheOptions`](crate::remote::TokenCacheOptions). + pub token_cache: Option, } impl std::fmt::Debug for OAuthConfig { @@ -79,6 +216,7 @@ impl std::fmt::Debug for OAuthConfig { .field("scopes", &self.scopes) .field("flow", &self.flow) .field("refresh_buffer_secs", &self.refresh_buffer_secs) + .field("token_cache", &self.token_cache) .finish() } } @@ -88,26 +226,34 @@ impl std::fmt::Debug for OAuthConfig { #[derive(Clone, Debug, Deserialize)] struct OidcDiscovery { token_endpoint: String, + authorization_endpoint: Option, + device_authorization_endpoint: Option, } // -- Token Response -- #[derive(Deserialize)] -struct TokenResponse { - access_token: String, +pub(crate) struct TokenResponse { + pub(crate) access_token: String, + #[serde(default)] + pub(crate) refresh_token: Option, /// Token lifetime in seconds. /// Some providers (Azure IMDS) return this as a string, so we accept both. #[serde(default, deserialize_with = "deserialize_optional_u64_or_string")] - expires_in: Option, + pub(crate) expires_in: Option, #[serde(default)] #[allow(dead_code)] - token_type: Option, + pub(crate) token_type: Option, } impl std::fmt::Debug for TokenResponse { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("TokenResponse") .field("access_token", &"") + .field( + "refresh_token", + &self.refresh_token.as_ref().map(|_| ""), + ) .field("expires_in", &self.expires_in) .field("token_type", &self.token_type) .finish() @@ -168,6 +314,7 @@ where struct TokenState { access_token: Option, + refresh_token: Option, expires_at: Option, } @@ -175,6 +322,7 @@ impl TokenState { fn new() -> Self { Self { access_token: None, + refresh_token: None, expires_at: None, } } @@ -189,50 +337,72 @@ impl TokenState { fn update(&mut self, resp: &TokenResponse) { self.access_token = Some(resp.access_token.clone()); + if resp.refresh_token.is_some() { + self.refresh_token = resp.refresh_token.clone(); + } let expires_in = resp.expires_in.unwrap_or(DEFAULT_TOKEN_TTL_SECS); self.expires_at = Some(Instant::now() + Duration::from_secs(expires_in)); } } #[async_trait] -trait TokenSource: Send + Sync + std::fmt::Debug { +pub(crate) trait TokenSource: Send + Sync + std::fmt::Debug { async fn fetch_token(&self) -> Result; + + async fn refresh_token(&self, _refresh_token: &str) -> Result { + Ok(RefreshResult::Unsupported) + } } -struct ClientCredentialsSource { +#[derive(Debug)] +pub(crate) enum RefreshResult { + Refreshed(TokenResponse), + Reauthenticate, + Unsupported, +} + +struct OidcClient { issuer_url: String, client_id: String, - client_secret: String, + client_secret: Option, scopes: Vec, http_client: Client, discovery: RwLock>, } -impl std::fmt::Debug for ClientCredentialsSource { +impl std::fmt::Debug for OidcClient { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ClientCredentialsSource") + f.debug_struct("OidcClient") .field("issuer_url", &self.issuer_url) .field("client_id", &self.client_id) - .field("client_secret", &"") + .field( + "client_secret", + &self.client_secret.as_ref().map(|_| ""), + ) .field("scopes", &self.scopes) .finish() } } -impl ClientCredentialsSource { +impl OidcClient { fn new( issuer_url: String, client_id: String, client_secret: Option, scopes: Vec, ) -> Result { - let client_secret = client_secret.ok_or(Error::InvalidInput { - message: "client_secret is required for ClientCredentials flow".to_string(), - })?; Self::validate_issuer_transport(&issuer_url)?; let http_client = Client::builder() .timeout(Duration::from_secs(30)) + .redirect(reqwest::redirect::Policy::custom(|attempt| { + if oauth_url_uses_secure_transport(attempt.url()) { + attempt.follow() + } else { + attempt + .error("OAuth redirects must use https, except for http on a loopback host") + } + })) .build() .map_err(|e| Error::Runtime { message: format!("Failed to create HTTP client for OAuth: {e}"), @@ -249,31 +419,7 @@ impl ClientCredentialsSource { } fn validate_issuer_transport(issuer_url: &str) -> Result<()> { - let issuer = url::Url::parse(issuer_url).map_err(|e| Error::InvalidInput { - message: format!("Invalid OAuth issuer_url: {e}"), - })?; - - match issuer.scheme() { - "https" => Ok(()), - "http" if Self::is_loopback_issuer(&issuer) => Ok(()), - _ => Err(Error::InvalidInput { - message: - "ClientCredentials OAuth issuer_url must use https, except for loopback hosts" - .to_string(), - }), - } - } - - fn is_loopback_issuer(issuer: &url::Url) -> bool { - let Some(host) = issuer.host_str() else { - return false; - }; - - host.eq_ignore_ascii_case("localhost") - || host - .parse::() - .map(|addr| addr.is_loopback()) - .unwrap_or(false) + validate_oauth_url(issuer_url, "issuer_url").map(drop) } async fn get_discovery(&self) -> Result { @@ -319,6 +465,13 @@ impl ClientCredentialsSource { let disc: OidcDiscovery = resp.json().await.map_err(|e| Error::Runtime { message: format!("Failed to parse OIDC discovery document: {e}"), })?; + validate_oauth_url(&disc.token_endpoint, "token_endpoint")?; + if let Some(endpoint) = disc.authorization_endpoint.as_deref() { + validate_oauth_url(endpoint, "authorization_endpoint")?; + } + if let Some(endpoint) = disc.device_authorization_endpoint.as_deref() { + validate_oauth_url(endpoint, "device_authorization_endpoint")?; + } let result = disc.clone(); @@ -337,7 +490,7 @@ impl ClientCredentialsSource { async fn post_token_request( &self, endpoint: &str, - params: &[(&str, &str)], + params: &[(String, String)], ) -> Result { let resp = self .http_client @@ -363,24 +516,690 @@ impl ClientCredentialsSource { message: format!("Failed to parse token response: {e}"), }) } + + async fn refresh_token(&self, refresh_token: &str) -> Result { + let endpoint = self.get_token_endpoint().await?; + let mut params = vec![ + ("grant_type".to_string(), "refresh_token".to_string()), + ("client_id".to_string(), self.client_id.clone()), + ("refresh_token".to_string(), refresh_token.to_string()), + ]; + if let Some(secret) = self.client_secret.as_ref() { + params.push(("client_secret".to_string(), secret.clone())); + } + let response = self + .http_client + .post(&endpoint) + .form(¶ms) + .send() + .await + .map_err(|e| Error::Runtime { + message: format!("Refresh token request to {endpoint} failed: {e}"), + })?; + if response.status().is_success() { + return response + .json() + .await + .map(RefreshResult::Refreshed) + .map_err(|e| Error::Runtime { + message: format!("Failed to parse refresh token response: {e}"), + }); + } + + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + let error_code = serde_json::from_str::(&body) + .ok() + .map(|error| error.error); + if matches!( + error_code.as_deref(), + Some("invalid_grant" | "invalid_token") + ) { + return Ok(RefreshResult::Reauthenticate); + } + Err(Error::Runtime { + message: format!("Refresh token request failed with status {status}: {body}"), + }) + } +} + +struct ClientCredentialsSource { + oidc: OidcClient, +} + +impl std::fmt::Debug for ClientCredentialsSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ClientCredentialsSource") + .field("oidc", &self.oidc) + .finish() + } +} + +impl ClientCredentialsSource { + fn new( + issuer_url: String, + client_id: String, + client_secret: Option, + scopes: Vec, + ) -> Result { + if client_secret.is_none() { + return Err(Error::InvalidInput { + message: "client_secret is required for ClientCredentials flow".to_string(), + }); + } + Ok(Self { + oidc: OidcClient::new(issuer_url, client_id, client_secret, scopes)?, + }) + } } #[async_trait] impl TokenSource for ClientCredentialsSource { async fn fetch_token(&self) -> Result { - let token_endpoint = self.get_token_endpoint().await?; - let scope = self.scopes_string(); + let token_endpoint = self.oidc.get_token_endpoint().await?; let params = [ - ("grant_type", "client_credentials"), - ("client_id", self.client_id.as_str()), - ("client_secret", self.client_secret.as_str()), - ("scope", scope.as_str()), + ("grant_type".to_string(), "client_credentials".to_string()), + ("client_id".to_string(), self.oidc.client_id.clone()), + ( + "client_secret".to_string(), + self.oidc.client_secret.clone().expect("validated in new"), + ), + ("scope".to_string(), self.oidc.scopes_string()), ]; - self.post_token_request(&token_endpoint, ¶ms).await + self.oidc.post_token_request(&token_endpoint, ¶ms).await } } +#[derive(Debug)] +struct ResolvedRedirect { + uri: String, + bind_addr: SocketAddr, + callback_path: String, +} + +impl ResolvedRedirect { + fn new(options: &AuthorizationCodeOptions) -> Result { + let uri = options.redirect_uri.clone().unwrap_or_else(|| { + format!( + "http://127.0.0.1:{}/callback", + options.callback_port.unwrap_or(DEFAULT_CALLBACK_PORT) + ) + }); + let parsed = Url::parse(&uri).map_err(|e| Error::InvalidInput { + message: format!("Invalid OAuth redirect_uri: {e}"), + })?; + + if parsed.scheme() != "http" { + return Err(Error::InvalidInput { + message: "OAuth redirect_uri must use http with a loopback host".to_string(), + }); + } + if parsed.query().is_some() || parsed.fragment().is_some() { + return Err(Error::InvalidInput { + message: "OAuth redirect_uri must not contain a query or fragment".to_string(), + }); + } + + let ip = match parsed.host() { + Some(url::Host::Domain(host)) if host.eq_ignore_ascii_case("localhost") => { + IpAddr::V4(std::net::Ipv4Addr::LOCALHOST) + } + Some(url::Host::Ipv4(ip)) if ip.is_loopback() => IpAddr::V4(ip), + Some(url::Host::Ipv6(ip)) if ip.is_loopback() => IpAddr::V6(ip), + Some(_) => { + return Err(Error::InvalidInput { + message: "OAuth redirect_uri must use a loopback host".to_string(), + }); + } + None => { + return Err(Error::InvalidInput { + message: "OAuth redirect_uri must include a loopback host".to_string(), + }); + } + }; + let port = parsed.port().ok_or(Error::InvalidInput { + message: "OAuth redirect_uri must include a port".to_string(), + })?; + if port == 0 { + return Err(Error::InvalidInput { + message: "OAuth redirect_uri port must be greater than zero".to_string(), + }); + } + if let Some(callback_port) = options.callback_port + && callback_port != port + { + return Err(Error::InvalidInput { + message: format!( + "OAuth callback_port {callback_port} does not match redirect_uri port {port}" + ), + }); + } + + Ok(Self { + uri, + bind_addr: SocketAddr::new(ip, port), + callback_path: parsed.path().to_string(), + }) + } +} + +#[derive(Debug)] +struct AuthorizationRequest { + url: Url, + state: String, + code_verifier: Option, +} + +#[derive(Debug, PartialEq)] +enum AuthorizationCallback { + Code(String), + ProviderError(String), +} + +struct AuthorizationCodeSource { + oidc: OidcClient, + options: AuthorizationCodeOptions, + redirect: ResolvedRedirect, +} + +impl std::fmt::Debug for AuthorizationCodeSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AuthorizationCodeSource") + .field("oidc", &self.oidc) + .field("options", &self.options) + .field("redirect", &self.redirect) + .finish() + } +} + +impl AuthorizationCodeSource { + fn new( + issuer_url: String, + client_id: String, + client_secret: Option, + scopes: Vec, + options: AuthorizationCodeOptions, + ) -> Result { + let redirect = ResolvedRedirect::new(&options)?; + Ok(Self { + oidc: OidcClient::new(issuer_url, client_id, client_secret, scopes)?, + options, + redirect, + }) + } + + async fn build_authorization_request(&self) -> Result { + let endpoint = self + .oidc + .get_discovery() + .await? + .authorization_endpoint + .ok_or(Error::Runtime { + message: "OIDC discovery did not provide authorization_endpoint".to_string(), + })?; + let mut url = validate_oauth_url(&endpoint, "authorization_endpoint")?; + let state = random_urlsafe_string(32); + let code_verifier = self.options.use_pkce.then(|| random_urlsafe_string(64)); + + { + let mut query = url.query_pairs_mut(); + query + .append_pair("response_type", "code") + .append_pair("client_id", &self.oidc.client_id) + .append_pair("redirect_uri", &self.redirect.uri) + .append_pair("scope", &self.oidc.scopes_string()) + .append_pair("state", &state); + if let Some(verifier) = code_verifier.as_ref() { + let challenge = base64::engine::general_purpose::URL_SAFE_NO_PAD + .encode(Sha256::digest(verifier.as_bytes())); + query + .append_pair("code_challenge", &challenge) + .append_pair("code_challenge_method", "S256"); + } + } + + Ok(AuthorizationRequest { + url, + state, + code_verifier, + }) + } + + async fn wait_for_callback( + &self, + listener: &TcpListener, + expected_state: &str, + ) -> Result { + let deadline = + TokioInstant::now() + Duration::from_secs(AUTHORIZATION_CALLBACK_TIMEOUT_SECS); + loop { + let (mut stream, _) = tokio::time::timeout_at(deadline, listener.accept()) + .await + .map_err(|_| Error::Runtime { + message: "Timed out waiting for the OAuth authorization callback".to_string(), + })? + .map_err(|e| Error::Runtime { + message: format!("Failed to accept OAuth callback connection: {e}"), + })?; + match read_authorization_callback( + &mut stream, + &self.redirect.callback_path, + expected_state, + deadline, + ) + .await + { + Ok(AuthorizationCallback::Code(code)) => { + write_callback_response(&mut stream, true).await; + return Ok(code); + } + Ok(AuthorizationCallback::ProviderError(message)) => { + write_callback_response(&mut stream, false).await; + return Err(Error::Runtime { message }); + } + Err(error) => { + if TokioInstant::now() >= deadline { + return Err(Error::Runtime { + message: "Timed out waiting for the OAuth authorization callback" + .to_string(), + }); + } + debug!("Ignoring unrelated OAuth callback connection: {error}"); + write_callback_response(&mut stream, false).await; + } + } + } + } + + async fn exchange_code( + &self, + code: &str, + code_verifier: Option<&str>, + ) -> Result { + let endpoint = self.oidc.get_token_endpoint().await?; + let mut params = vec![ + ("grant_type".to_string(), "authorization_code".to_string()), + ("client_id".to_string(), self.oidc.client_id.clone()), + ("code".to_string(), code.to_string()), + ("redirect_uri".to_string(), self.redirect.uri.clone()), + ]; + if let Some(verifier) = code_verifier { + params.push(("code_verifier".to_string(), verifier.to_string())); + } + if let Some(secret) = self.oidc.client_secret.as_ref() { + params.push(("client_secret".to_string(), secret.clone())); + } + self.oidc.post_token_request(&endpoint, ¶ms).await + } +} + +#[async_trait] +impl TokenSource for AuthorizationCodeSource { + async fn fetch_token(&self) -> Result { + let listener = TcpListener::bind(self.redirect.bind_addr) + .await + .map_err(|e| Error::Runtime { + message: format!( + "Failed to bind OAuth callback server at {}: {e}", + self.redirect.bind_addr + ), + })?; + let request = self.build_authorization_request().await?; + show_oauth_prompt(&authorization_prompt(&request.url)); + launch_browser(request.url.clone()); + let code = self.wait_for_callback(&listener, &request.state).await?; + self.exchange_code(&code, request.code_verifier.as_deref()) + .await + } + + async fn refresh_token(&self, refresh_token: &str) -> Result { + self.oidc.refresh_token(refresh_token).await + } +} + +#[derive(Deserialize)] +struct DeviceAuthorizationResponse { + device_code: String, + user_code: String, + verification_uri: String, + #[serde(default)] + verification_uri_complete: Option, + expires_in: u64, + #[serde(default)] + interval: Option, +} + +#[derive(Debug, Deserialize)] +struct OAuthErrorResponse { + error: String, + #[serde(default)] + error_description: Option, +} + +struct DeviceCodeSource { + oidc: OidcClient, +} + +impl std::fmt::Debug for DeviceCodeSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DeviceCodeSource") + .field("oidc", &self.oidc) + .finish() + } +} + +impl DeviceCodeSource { + fn new( + issuer_url: String, + client_id: String, + client_secret: Option, + scopes: Vec, + ) -> Result { + Ok(Self { + oidc: OidcClient::new(issuer_url, client_id, client_secret, scopes)?, + }) + } + + async fn request_device_authorization(&self) -> Result { + let endpoint = self + .oidc + .get_discovery() + .await? + .device_authorization_endpoint + .ok_or(Error::Runtime { + message: "OIDC discovery did not provide device_authorization_endpoint".to_string(), + })?; + let mut params = vec![ + ("client_id".to_string(), self.oidc.client_id.clone()), + ("scope".to_string(), self.oidc.scopes_string()), + ]; + if let Some(secret) = self.oidc.client_secret.as_ref() { + params.push(("client_secret".to_string(), secret.clone())); + } + let response = self + .oidc + .http_client + .post(&endpoint) + .form(¶ms) + .send() + .await + .map_err(|e| Error::Runtime { + message: format!("Device authorization request to {endpoint} failed: {e}"), + })?; + if !response.status().is_success() { + return Err(Error::Runtime { + message: format!( + "Device authorization request failed with status {}: {}", + response.status(), + response.text().await.unwrap_or_default() + ), + }); + } + let device: DeviceAuthorizationResponse = + response.json().await.map_err(|e| Error::Runtime { + message: format!("Failed to parse device authorization response: {e}"), + })?; + validate_oauth_url(&device.verification_uri, "verification_uri")?; + if let Some(uri) = device.verification_uri_complete.as_deref() { + validate_oauth_url(uri, "verification_uri_complete")?; + } + Ok(device) + } + + async fn poll_for_token(&self, device: &DeviceAuthorizationResponse) -> Result { + let endpoint = self.oidc.get_token_endpoint().await?; + let deadline = TokioInstant::now() + Duration::from_secs(device.expires_in); + let mut interval = Duration::from_secs(device.interval.unwrap_or(5).max(1)); + + loop { + let now = TokioInstant::now(); + if now >= deadline { + return Err(Error::Runtime { + message: "Device authorization expired before authentication completed" + .to_string(), + }); + } + tokio::time::sleep_until(std::cmp::min(now + interval, deadline)).await; + if TokioInstant::now() >= deadline { + return Err(Error::Runtime { + message: "Device authorization expired before authentication completed" + .to_string(), + }); + } + + let mut params = vec![ + ( + "grant_type".to_string(), + "urn:ietf:params:oauth:grant-type:device_code".to_string(), + ), + ("client_id".to_string(), self.oidc.client_id.clone()), + ("device_code".to_string(), device.device_code.clone()), + ]; + if let Some(secret) = self.oidc.client_secret.as_ref() { + params.push(("client_secret".to_string(), secret.clone())); + } + + let response = match self + .oidc + .http_client + .post(&endpoint) + .form(¶ms) + .send() + .await + { + Ok(response) => response, + Err(error) => { + warn!("Device token request to {endpoint} failed; retrying: {error}"); + continue; + } + }; + if response.status().is_success() { + return response.json().await.map_err(|e| Error::Runtime { + message: format!("Failed to parse device token response: {e}"), + }); + } + + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + let oauth_error = serde_json::from_str::(&body).ok(); + match oauth_error.as_ref().map(|error| error.error.as_str()) { + Some("authorization_pending") => continue, + Some("slow_down") => { + interval += Duration::from_secs(5); + continue; + } + Some("temporarily_unavailable") => continue, + Some("access_denied") => { + return Err(Error::Runtime { + message: "Device authorization was denied by the user".to_string(), + }); + } + Some("expired_token") => { + return Err(Error::Runtime { + message: "Device authorization expired before authentication completed" + .to_string(), + }); + } + _ if status == reqwest::StatusCode::TOO_MANY_REQUESTS + || status.is_server_error() => + { + warn!("Device token endpoint returned {status}; retrying"); + continue; + } + _ => { + let detail = oauth_error + .and_then(|error| error.error_description) + .unwrap_or(body); + return Err(Error::Runtime { + message: format!( + "Device token request failed with status {status}: {detail}" + ), + }); + } + } + } + } +} + +#[async_trait] +impl TokenSource for DeviceCodeSource { + async fn fetch_token(&self) -> Result { + let device = self.request_device_authorization().await?; + show_oauth_prompt(&device_prompt(&device.verification_uri, &device.user_code)); + let (browser_url, name) = device + .verification_uri_complete + .as_deref() + .map(|url| (url, "verification_uri_complete")) + .unwrap_or((&device.verification_uri, "verification_uri")); + launch_browser(validate_oauth_url(browser_url, name)?); + self.poll_for_token(&device).await + } + + async fn refresh_token(&self, refresh_token: &str) -> Result { + self.oidc.refresh_token(refresh_token).await + } +} + +fn random_urlsafe_string(length: usize) -> String { + const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~"; + let mut rng = rand::rng(); + (0..length) + .map(|_| CHARSET[rng.random_range(0..CHARSET.len())] as char) + .collect() +} + +fn launch_browser(url: Url) { + drop(tokio::task::spawn_blocking(move || { + if let Some(browser) = std::env::var_os("LANCEDB_OAUTH_BROWSER") { + match Command::new(browser).arg(url.as_str()).status() { + Ok(status) if !status.success() => { + warn!("OAuth browser helper exited with status {status}"); + } + Err(error) => warn!("Could not run the OAuth browser helper: {error}"), + Ok(_) => {} + } + } else if let Err(error) = webbrowser::open(url.as_str()) { + warn!("Could not open an OAuth browser automatically: {error}"); + } + })); +} + +async fn read_authorization_callback( + stream: &mut TcpStream, + expected_path: &str, + expected_state: &str, + overall_deadline: TokioInstant, +) -> Result { + const MAX_CALLBACK_REQUEST_BYTES: usize = 16 * 1024; + let deadline = std::cmp::min( + overall_deadline, + TokioInstant::now() + Duration::from_secs(10), + ); + let mut request = Vec::with_capacity(1024); + loop { + let mut buffer = [0; 1024]; + let count = tokio::time::timeout_at(deadline, stream.read(&mut buffer)) + .await + .map_err(|_| Error::Runtime { + message: "Timed out reading the OAuth authorization callback".to_string(), + })? + .map_err(|e| Error::Runtime { + message: format!("Failed to read OAuth authorization callback: {e}"), + })?; + if count == 0 { + return Err(Error::Runtime { + message: "OAuth authorization callback closed before sending a request".to_string(), + }); + } + request.extend_from_slice(&buffer[..count]); + if request.windows(4).any(|window| window == b"\r\n\r\n") { + break; + } + if request.len() >= MAX_CALLBACK_REQUEST_BYTES { + return Err(Error::Runtime { + message: "OAuth authorization callback request was too large".to_string(), + }); + } + } + let request = std::str::from_utf8(&request).map_err(|e| Error::Runtime { + message: format!("OAuth authorization callback was not valid UTF-8: {e}"), + })?; + parse_authorization_callback(request, expected_path, expected_state) +} + +fn parse_authorization_callback( + request: &str, + expected_path: &str, + expected_state: &str, +) -> Result { + let request_target = request + .lines() + .next() + .and_then(|line| { + let mut parts = line.split_whitespace(); + (parts.next() == Some("GET")) + .then(|| parts.next()) + .flatten() + }) + .ok_or(Error::Runtime { + message: "OAuth authorization callback was not a valid HTTP GET request".to_string(), + })?; + let callback = + Url::parse(&format!("http://loopback{request_target}")).map_err(|e| Error::Runtime { + message: format!("OAuth authorization callback URL was invalid: {e}"), + })?; + if callback.path() != expected_path { + return Err(Error::Runtime { + message: format!( + "OAuth authorization callback used unexpected path {}", + callback.path() + ), + }); + } + let params: HashMap<_, _> = callback.query_pairs().into_owned().collect(); + if params.get("state").map(String::as_str) != Some(expected_state) { + return Err(Error::Runtime { + message: "OAuth authorization callback state did not match".to_string(), + }); + } + if let Some(error) = params.get("error") { + let description = params + .get("error_description") + .map(String::as_str) + .unwrap_or(error); + return Ok(AuthorizationCallback::ProviderError(format!( + "OAuth authorization failed: {description}" + ))); + } + params + .get("code") + .cloned() + .map(AuthorizationCallback::Code) + .ok_or(Error::Runtime { + message: "OAuth authorization callback did not contain a code".to_string(), + }) +} + +async fn write_callback_response(stream: &mut TcpStream, success: bool) { + let (status, body) = if success { + ( + "200 OK", + "

Authentication successful

You can close this window.

", + ) + } else { + ( + "400 Bad Request", + "

Authentication failed

Return to the application for details.

", + ) + }; + let response = format!( + "HTTP/1.1 {status}\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + let _ = stream.write_all(response.as_bytes()).await; +} + struct AzureImdsSource { client_id: Option, resource: String, @@ -463,22 +1282,66 @@ impl TokenSource for AzureImdsSource { } } +/// Build the token source for a configuration. +/// +/// Shared by [`OAuthHeaderProvider`] and +/// [`OAuthSession`](crate::remote::OAuthSession). +pub(crate) fn build_token_source(config: &OAuthConfig) -> Result> { + if config.scopes.is_empty() { + return Err(Error::InvalidInput { + message: "At least one OAuth scope is required".to_string(), + }); + } + Ok(match &config.flow { + OAuthFlow::ClientCredentials => Box::new(ClientCredentialsSource::new( + config.issuer_url.clone(), + config.client_id.clone(), + config.client_secret.clone(), + config.scopes.clone(), + )?), + OAuthFlow::AuthorizationCode(options) => Box::new(AuthorizationCodeSource::new( + config.issuer_url.clone(), + config.client_id.clone(), + config.client_secret.clone(), + config.scopes.clone(), + options.clone(), + )?), + OAuthFlow::DeviceCode => Box::new(DeviceCodeSource::new( + config.issuer_url.clone(), + config.client_id.clone(), + config.client_secret.clone(), + config.scopes.clone(), + )?), + OAuthFlow::AzureManagedIdentity { client_id } => Box::new(AzureImdsSource::new( + config.scopes.clone(), + client_id.clone(), + )?), + }) +} + /// OAuth header provider that manages the full token lifecycle. /// /// Implements [`HeaderProvider`] to inject `Authorization: Bearer ` /// headers into every LanceDB request, with automatic token refresh. It also /// identifies the bearer credential as OIDC so LanceDB's SQL service selects /// OIDC validation instead of API-key validation. +/// +/// When the configuration enables +/// [`token_cache`](OAuthConfig::token_cache), tokens are additionally shared +/// through a hardened on-disk cache so separate processes reuse one session; +/// see [`crate::remote::token_cache`]. pub struct OAuthHeaderProvider { token_source: Box, token_state: Arc>, refresh_buffer: Duration, + token_cache: Option>, } impl std::fmt::Debug for OAuthHeaderProvider { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("OAuthHeaderProvider") .field("token_source", &self.token_source) + .field("token_cache", &self.token_cache) .finish() } } @@ -486,39 +1349,19 @@ impl std::fmt::Debug for OAuthHeaderProvider { impl OAuthHeaderProvider { /// Create a new OAuth header provider from configuration. pub fn new(config: OAuthConfig) -> Result { - let OAuthConfig { - issuer_url, - client_id, - client_secret, - scopes, - flow, - refresh_buffer_secs, - } = config; - - if scopes.is_empty() { - return Err(Error::InvalidInput { - message: "At least one OAuth scope is required".to_string(), - }); - } - - let refresh_buffer = - Duration::from_secs(refresh_buffer_secs.unwrap_or(DEFAULT_REFRESH_BUFFER_SECS)); - let token_source: Box = match flow { - OAuthFlow::ClientCredentials => Box::new(ClientCredentialsSource::new( - issuer_url, - client_id, - client_secret, - scopes, - )?), - OAuthFlow::AzureManagedIdentity { client_id } => { - Box::new(AzureImdsSource::new(scopes, client_id)?) - } - }; + let refresh_buffer = Duration::from_secs( + config + .refresh_buffer_secs + .unwrap_or(DEFAULT_REFRESH_BUFFER_SECS), + ); + let token_source = build_token_source(&config)?; + let token_cache = crate::remote::token_cache::token_cache_for_config(&config)?; Ok(Self { token_source, token_state: Arc::new(RwLock::new(TokenState::new())), refresh_buffer, + token_cache, }) } @@ -544,8 +1387,34 @@ impl OAuthHeaderProvider { return Ok(token.clone()); } - debug!("Acquiring new OAuth token via {:?}", self.token_source); - let resp = self.token_source.fetch_token().await?; + if let Some(cache) = &self.token_cache { + // Cross-process critical section: serialize with other processes, + // reread the durable record, refresh or acquire exactly once, and + // persist the rotated refresh token. + let resp = cache.refresh_or_acquire(self.token_source.as_ref()).await?; + state.update(&resp); + return Ok(resp.access_token); + } + + let refresh_token = state.refresh_token.clone(); + let resp = if let Some(refresh_token) = refresh_token.as_deref() { + debug!("Refreshing OAuth access token via {:?}", self.token_source); + match self.token_source.refresh_token(refresh_token).await? { + RefreshResult::Refreshed(response) => response, + RefreshResult::Unsupported => self.token_source.fetch_token().await?, + RefreshResult::Reauthenticate => { + warn!( + "OAuth refresh token was rejected; acquiring a new token via {:?}", + self.token_source + ); + state.refresh_token = None; + self.token_source.fetch_token().await? + } + } + } else { + debug!("Acquiring new OAuth token via {:?}", self.token_source); + self.token_source.fetch_token().await? + }; state.update(&resp); Ok(resp.access_token) @@ -591,6 +1460,7 @@ mod tests { let mut state = TokenState::new(); let response = TokenResponse { access_token: "tok".to_string(), + refresh_token: None, expires_in: None, token_type: None, }; @@ -601,6 +1471,25 @@ mod tests { assert!(state.is_expired(Duration::from_secs(DEFAULT_TOKEN_TTL_SECS + 1))); } + #[test] + fn test_token_state_retains_refresh_token_when_not_rotated() { + let mut state = TokenState::new(); + state.update(&TokenResponse { + access_token: "token-1".to_string(), + refresh_token: Some("refresh-1".to_string()), + expires_in: Some(60), + token_type: None, + }); + state.update(&TokenResponse { + access_token: "token-2".to_string(), + refresh_token: None, + expires_in: Some(60), + token_type: None, + }); + + assert_eq!(state.refresh_token.as_deref(), Some("refresh-1")); + } + #[test] fn test_token_response_accepts_float_expires_in() { let response: TokenResponse = @@ -622,12 +1511,14 @@ mod tests { fn test_token_response_debug_redacts_access_token() { let response = TokenResponse { access_token: "secret-token".to_string(), + refresh_token: Some("secret-refresh-token".to_string()), expires_in: Some(3600), token_type: Some("Bearer".to_string()), }; let debug = format!("{response:?}"); assert!(!debug.contains("secret-token")); + assert!(!debug.contains("secret-refresh-token")); assert!(debug.contains("access_token: \"\"")); } @@ -641,7 +1532,655 @@ mod tests { ) .unwrap(); - assert_eq!(source.scopes_string(), "scope1 scope2"); + assert_eq!(source.oidc.scopes_string(), "scope1 scope2"); + } + + #[test] + fn test_oauth_transport_requires_https_except_for_loopback() { + assert!(validate_oauth_url("https://idp.example.com/token", "endpoint").is_ok()); + assert!(validate_oauth_url("http://localhost:8080/token", "endpoint").is_ok()); + assert!(validate_oauth_url("http://127.0.0.1:8080/token", "endpoint").is_ok()); + assert!(validate_oauth_url("http://[::1]:8080/token", "endpoint").is_ok()); + + let err = validate_oauth_url("http://idp.example.com/token", "endpoint").unwrap_err(); + assert!(matches!( + err, + Error::InvalidInput { message } + if message == "OAuth endpoint must use https, except for http on a loopback host" + )); + } + + #[test] + fn test_interactive_prompts_use_default_visible_output() { + let authorization_url = Url::parse("https://idp.example.com/authorize?state=abc").unwrap(); + let authorization = authorization_prompt(&authorization_url); + let device = device_prompt("https://idp.example.com/device", "ABCD-EFGH"); + let mut output = Vec::new(); + + write_oauth_prompt(&mut output, &authorization); + write_oauth_prompt(&mut output, &device); + + let output = String::from_utf8(output).unwrap(); + assert!(output.contains(authorization_url.as_str())); + assert!(output.contains("https://idp.example.com/device")); + assert!(output.contains("ABCD-EFGH")); + } + + #[test] + fn test_authorization_code_options_default_to_pkce() { + let options = AuthorizationCodeOptions::new(); + + assert!(options.use_pkce); + assert!(options.redirect_uri.is_none()); + assert!(options.callback_port.is_none()); + } + + #[test] + fn test_authorization_redirect_defaults_to_ipv4_loopback() { + let redirect = ResolvedRedirect::new(&AuthorizationCodeOptions::new()).unwrap(); + + assert_eq!( + redirect.uri, + format!("http://127.0.0.1:{DEFAULT_CALLBACK_PORT}/callback") + ); + assert!(redirect.bind_addr.ip().is_loopback()); + assert_eq!(redirect.bind_addr.port(), DEFAULT_CALLBACK_PORT); + assert_eq!(redirect.callback_path, "/callback"); + } + + #[test] + fn test_authorization_redirect_rejects_non_loopback_host() { + let options = AuthorizationCodeOptions::new() + .redirect_uri("https://client.example.com/oauth/callback"); + + let err = ResolvedRedirect::new(&options).unwrap_err(); + assert!(matches!( + err, + Error::InvalidInput { message } + if message == "OAuth redirect_uri must use http with a loopback host" + )); + } + + #[test] + fn test_authorization_redirect_rejects_mismatched_port() { + let options = AuthorizationCodeOptions::new() + .redirect_uri("http://127.0.0.1:8401/callback") + .callback_port(8400); + + let err = ResolvedRedirect::new(&options).unwrap_err(); + assert!(matches!( + err, + Error::InvalidInput { message } + if message.contains("does not match redirect_uri port") + )); + } + + #[test] + fn test_authorization_redirect_requires_explicit_port() { + let options = AuthorizationCodeOptions::new().redirect_uri("http://127.0.0.1/callback"); + + let err = ResolvedRedirect::new(&options).unwrap_err(); + assert!(matches!( + err, + Error::InvalidInput { message } + if message == "OAuth redirect_uri must include a port" + )); + } + + #[test] + fn test_authorization_redirect_accepts_ipv6_loopback() { + let options = AuthorizationCodeOptions::new().redirect_uri("http://[::1]:8400/callback"); + + let redirect = ResolvedRedirect::new(&options).unwrap(); + assert_eq!( + redirect.bind_addr, + "[::1]:8400".parse::().unwrap() + ); + } + + #[test] + fn test_authorization_callback_parsing() { + let callback = parse_authorization_callback( + "GET /callback?code=abc%20123&state=expected HTTP/1.1\r\nHost: localhost\r\n", + "/callback", + "expected", + ) + .unwrap(); + + assert_eq!(callback, AuthorizationCallback::Code("abc 123".to_string())); + } + + #[test] + fn test_authorization_callback_rejects_state_mismatch() { + let err = parse_authorization_callback( + "GET /callback?code=abc&state=wrong HTTP/1.1\r\nHost: localhost\r\n", + "/callback", + "expected", + ) + .unwrap_err(); + + assert!(matches!( + err, + Error::Runtime { message } + if message == "OAuth authorization callback state did not match" + )); + } + + #[test] + fn test_authorization_callback_reports_provider_error() { + let callback = parse_authorization_callback( + "GET /callback?error=access_denied&error_description=user+cancelled&state=expected HTTP/1.1\r\nHost: localhost\r\n", + "/callback", + "expected", + ) + .unwrap(); + + assert_eq!( + callback, + AuthorizationCallback::ProviderError( + "OAuth authorization failed: user cancelled".to_string() + ) + ); + } + + #[tokio::test] + async fn test_authorization_callback_ignores_unrelated_connection_and_partial_read() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let source = AuthorizationCodeSource::new( + "http://127.0.0.1:1".to_string(), + "client-id".to_string(), + None, + vec!["openid".to_string()], + AuthorizationCodeOptions::new() + .redirect_uri(format!("http://127.0.0.1:{port}/callback")), + ) + .unwrap(); + + let browser = tokio::spawn(async move { + let mut unrelated = TcpStream::connect(("127.0.0.1", port)).await.unwrap(); + unrelated + .write_all(b"GET /favicon.ico HTTP/1.1\r\nHost: localhost\r\n\r\n") + .await + .unwrap(); + let mut response = Vec::new(); + unrelated.read_to_end(&mut response).await.unwrap(); + assert!( + String::from_utf8(response) + .unwrap() + .starts_with("HTTP/1.1 400") + ); + + let mut callback = TcpStream::connect(("127.0.0.1", port)).await.unwrap(); + callback + .write_all(b"GET /callback?code=auth") + .await + .unwrap(); + tokio::task::yield_now().await; + callback + .write_all(b"-code&state=expected HTTP/1.1\r\nHost: localhost\r\n\r\n") + .await + .unwrap(); + }); + + assert_eq!( + source + .wait_for_callback(&listener, "expected") + .await + .unwrap(), + "auth-code" + ); + browser.await.unwrap(); + } + + #[tokio::test] + async fn test_authorization_callback_read_respects_overall_deadline() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let client = tokio::spawn(async move { + let _stream = TcpStream::connect(("127.0.0.1", port)).await.unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + }); + let (mut stream, _) = listener.accept().await.unwrap(); + + let err = read_authorization_callback( + &mut stream, + "/callback", + "expected", + TokioInstant::now() + Duration::from_millis(20), + ) + .await + .unwrap_err(); + + assert!(matches!( + err, + Error::Runtime { message } + if message == "Timed out reading the OAuth authorization callback" + )); + client.abort(); + } + + #[tokio::test] + async fn test_authorization_request_uses_pkce_by_default() { + let (issuer_url, server) = spawn_discovery_server(1).await; + let source = AuthorizationCodeSource::new( + issuer_url, + "client-id".to_string(), + None, + vec!["openid".to_string(), "profile".to_string()], + AuthorizationCodeOptions::new(), + ) + .unwrap(); + + let request = source.build_authorization_request().await.unwrap(); + let params: HashMap<_, _> = request.url.query_pairs().into_owned().collect(); + assert_eq!( + params.get("response_type").map(String::as_str), + Some("code") + ); + assert_eq!( + params.get("scope").map(String::as_str), + Some("openid profile") + ); + assert_eq!( + params.get("code_challenge_method").map(String::as_str), + Some("S256") + ); + assert!(params.contains_key("code_challenge")); + assert!(request.code_verifier.is_some()); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_authorization_request_rejects_plaintext_provider_endpoint() { + let (issuer_url, server) = spawn_insecure_authorization_discovery_server().await; + let source = AuthorizationCodeSource::new( + issuer_url, + "client-id".to_string(), + None, + vec!["openid".to_string()], + AuthorizationCodeOptions::new(), + ) + .unwrap(); + + let err = source.build_authorization_request().await.unwrap_err(); + assert!(matches!( + err, + Error::InvalidInput { message } + if message.contains("authorization_endpoint must use https") + )); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_authorization_request_can_disable_pkce() { + let (issuer_url, server) = spawn_discovery_server(1).await; + let source = AuthorizationCodeSource::new( + issuer_url, + "client-id".to_string(), + Some("secret".to_string()), + vec!["openid".to_string()], + AuthorizationCodeOptions::new().use_pkce(false), + ) + .unwrap(); + + let request = source.build_authorization_request().await.unwrap(); + let params: HashMap<_, _> = request.url.query_pairs().into_owned().collect(); + assert!(!params.contains_key("code_challenge")); + assert!(!params.contains_key("code_challenge_method")); + assert!(request.code_verifier.is_none()); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_authorization_code_exchange_includes_optional_credentials() { + let (issuer_url, request_body, server) = spawn_token_exchange_server().await; + let source = AuthorizationCodeSource::new( + issuer_url, + "client-id".to_string(), + Some("secret".to_string()), + vec!["openid".to_string()], + AuthorizationCodeOptions::new(), + ) + .unwrap(); + + let response = source + .exchange_code("auth-code", Some("verifier")) + .await + .unwrap(); + assert_eq!(response.access_token, "token"); + let body = request_body.lock().unwrap().clone().unwrap(); + assert!(body.contains("grant_type=authorization_code")); + assert!(body.contains("code=auth-code")); + assert!(body.contains("code_verifier=verifier")); + assert!(body.contains("client_secret=secret")); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_refresh_invalid_grant_requires_reauthentication() { + let (issuer_url, server) = + spawn_refresh_error_server("400 Bad Request", r#"{"error":"invalid_grant"}"#).await; + let source = AuthorizationCodeSource::new( + issuer_url, + "client-id".to_string(), + None, + vec!["openid".to_string()], + AuthorizationCodeOptions::new(), + ) + .unwrap(); + + assert!(matches!( + source.oidc.refresh_token("revoked").await.unwrap(), + RefreshResult::Reauthenticate + )); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_refresh_transient_failure_remains_retryable() { + let (issuer_url, server) = spawn_refresh_error_server( + "503 Service Unavailable", + r#"{"error":"temporarily_unavailable"}"#, + ) + .await; + let source = AuthorizationCodeSource::new( + issuer_url, + "client-id".to_string(), + None, + vec!["openid".to_string()], + AuthorizationCodeOptions::new(), + ) + .unwrap(); + + let err = source.oidc.refresh_token("still-valid").await.unwrap_err(); + assert!(matches!( + err, + Error::Runtime { message } + if message.contains("503 Service Unavailable") + )); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_device_authorization_polls_until_success() { + let (issuer_url, token_requests, server) = spawn_device_server().await; + let source = DeviceCodeSource::new( + issuer_url, + "client-id".to_string(), + Some("secret".to_string()), + vec!["openid".to_string()], + ) + .unwrap(); + + let device = source.request_device_authorization().await.unwrap(); + let response = source.poll_for_token(&device).await.unwrap(); + + assert_eq!(response.access_token, "device-token"); + assert_eq!(response.refresh_token.as_deref(), Some("device-refresh")); + assert_eq!(token_requests.load(Ordering::SeqCst), 3); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_device_authorization_rejects_plaintext_verification_uri() { + let (issuer_url, server) = spawn_insecure_device_verification_server().await; + let source = DeviceCodeSource::new( + issuer_url, + "client-id".to_string(), + None, + vec!["openid".to_string()], + ) + .unwrap(); + + let Err(err) = source.request_device_authorization().await else { + panic!("expected insecure verification URI to be rejected"); + }; + assert!(matches!( + err, + Error::InvalidInput { message } + if message.contains("verification_uri must use https") + )); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_device_authorization_retries_transient_failures() { + let (issuer_url, token_requests, server) = spawn_device_transient_server().await; + let source = DeviceCodeSource::new( + issuer_url, + "client-id".to_string(), + None, + vec!["openid".to_string()], + ) + .unwrap(); + let device = test_device_authorization_response(10, 1); + + let response = source.poll_for_token(&device).await.unwrap(); + + assert_eq!(response.access_token, "device-token"); + assert_eq!(token_requests.load(Ordering::SeqCst), 4); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_device_authorization_reports_access_denied() { + let (issuer_url, server) = spawn_device_error_server("access_denied").await; + let source = DeviceCodeSource::new( + issuer_url, + "client-id".to_string(), + None, + vec!["openid".to_string()], + ) + .unwrap(); + let device = test_device_authorization_response(60, 1); + + let err = source.poll_for_token(&device).await.unwrap_err(); + assert!(matches!( + err, + Error::Runtime { message } + if message == "Device authorization was denied by the user" + )); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_device_authorization_reports_provider_expiry() { + let (issuer_url, server) = spawn_device_error_server("expired_token").await; + let source = DeviceCodeSource::new( + issuer_url, + "client-id".to_string(), + None, + vec!["openid".to_string()], + ) + .unwrap(); + let device = test_device_authorization_response(60, 1); + + let err = source.poll_for_token(&device).await.unwrap_err(); + assert!(matches!( + err, + Error::Runtime { message } + if message == "Device authorization expired before authentication completed" + )); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_device_authorization_stops_at_local_deadline() { + let (issuer_url, server) = spawn_discovery_server(1).await; + let source = DeviceCodeSource::new( + issuer_url, + "client-id".to_string(), + None, + vec!["openid".to_string()], + ) + .unwrap(); + let device = test_device_authorization_response(1, 5); + + let err = source.poll_for_token(&device).await.unwrap_err(); + assert!(matches!( + err, + Error::Runtime { message } + if message == "Device authorization expired before authentication completed" + )); + server.await.unwrap(); + } + + #[derive(Debug)] + struct RefreshingTokenSource { + fetches: Arc, + refreshes: Arc, + } + + #[async_trait] + impl TokenSource for RefreshingTokenSource { + async fn fetch_token(&self) -> Result { + self.fetches.fetch_add(1, Ordering::SeqCst); + Ok(TokenResponse { + access_token: "initial".to_string(), + refresh_token: Some("refresh".to_string()), + expires_in: Some(3600), + token_type: Some("Bearer".to_string()), + }) + } + + async fn refresh_token(&self, refresh_token: &str) -> Result { + assert_eq!(refresh_token, "refresh"); + self.refreshes.fetch_add(1, Ordering::SeqCst); + Ok(RefreshResult::Refreshed(TokenResponse { + access_token: "refreshed".to_string(), + refresh_token: None, + expires_in: Some(3600), + token_type: Some("Bearer".to_string()), + })) + } + } + + #[tokio::test] + async fn test_header_provider_uses_and_retains_refresh_token() { + let fetches = Arc::new(AtomicUsize::new(0)); + let refreshes = Arc::new(AtomicUsize::new(0)); + let provider = OAuthHeaderProvider { + token_source: Box::new(RefreshingTokenSource { + fetches: Arc::clone(&fetches), + refreshes: Arc::clone(&refreshes), + }), + token_state: Arc::new(RwLock::new(TokenState::new())), + refresh_buffer: Duration::ZERO, + token_cache: None, + }; + + assert_eq!(provider.get_valid_token().await.unwrap(), "initial"); + provider.token_state.write().await.expires_at = + Some(Instant::now() - Duration::from_secs(1)); + assert_eq!(provider.get_valid_token().await.unwrap(), "refreshed"); + assert_eq!(fetches.load(Ordering::SeqCst), 1); + assert_eq!(refreshes.load(Ordering::SeqCst), 1); + assert_eq!( + provider.token_state.read().await.refresh_token.as_deref(), + Some("refresh") + ); + } + + #[derive(Debug)] + struct FailedRefreshTokenSource { + fetches: Arc, + refreshes: Arc, + } + + #[async_trait] + impl TokenSource for FailedRefreshTokenSource { + async fn fetch_token(&self) -> Result { + self.fetches.fetch_add(1, Ordering::SeqCst); + Ok(TokenResponse { + access_token: "reauthenticated".to_string(), + refresh_token: Some("new-refresh".to_string()), + expires_in: Some(3600), + token_type: Some("Bearer".to_string()), + }) + } + + async fn refresh_token(&self, refresh_token: &str) -> Result { + assert_eq!(refresh_token, "revoked-refresh"); + self.refreshes.fetch_add(1, Ordering::SeqCst); + Ok(RefreshResult::Reauthenticate) + } + } + + #[tokio::test] + async fn test_header_provider_reauthenticates_after_refresh_failure() { + let fetches = Arc::new(AtomicUsize::new(0)); + let refreshes = Arc::new(AtomicUsize::new(0)); + let provider = OAuthHeaderProvider { + token_source: Box::new(FailedRefreshTokenSource { + fetches: Arc::clone(&fetches), + refreshes: Arc::clone(&refreshes), + }), + token_state: Arc::new(RwLock::new(TokenState { + access_token: Some("expired".to_string()), + refresh_token: Some("revoked-refresh".to_string()), + expires_at: Some(Instant::now() - Duration::from_secs(1)), + })), + refresh_buffer: Duration::ZERO, + token_cache: None, + }; + + assert_eq!(provider.get_valid_token().await.unwrap(), "reauthenticated"); + assert_eq!(fetches.load(Ordering::SeqCst), 1); + assert_eq!(refreshes.load(Ordering::SeqCst), 1); + assert_eq!( + provider.token_state.read().await.refresh_token.as_deref(), + Some("new-refresh") + ); + } + + #[derive(Debug)] + struct TransientRefreshFailureSource { + fetches: Arc, + } + + #[async_trait] + impl TokenSource for TransientRefreshFailureSource { + async fn fetch_token(&self) -> Result { + self.fetches.fetch_add(1, Ordering::SeqCst); + unreachable!("a transient refresh failure must not start an interactive flow") + } + + async fn refresh_token(&self, refresh_token: &str) -> Result { + assert_eq!(refresh_token, "valid-refresh"); + Err(Error::Runtime { + message: "token endpoint temporarily unavailable".to_string(), + }) + } + } + + #[tokio::test] + async fn test_header_provider_preserves_refresh_token_after_transient_failure() { + let fetches = Arc::new(AtomicUsize::new(0)); + let provider = OAuthHeaderProvider { + token_source: Box::new(TransientRefreshFailureSource { + fetches: Arc::clone(&fetches), + }), + token_state: Arc::new(RwLock::new(TokenState { + access_token: Some("expired".to_string()), + refresh_token: Some("valid-refresh".to_string()), + expires_at: Some(Instant::now() - Duration::from_secs(1)), + })), + refresh_buffer: Duration::ZERO, + token_cache: None, + }; + + let err = provider.get_valid_token().await.unwrap_err(); + assert!(matches!( + err, + Error::Runtime { message } + if message == "token endpoint temporarily unavailable" + )); + assert_eq!(fetches.load(Ordering::SeqCst), 0); + assert_eq!( + provider.token_state.read().await.refresh_token.as_deref(), + Some("valid-refresh") + ); } #[test] @@ -653,6 +2192,7 @@ mod tests { scopes: vec!["scope".to_string()], flow: OAuthFlow::ClientCredentials, refresh_buffer_secs: None, + token_cache: None, }; let debug = format!("{config:?}"); @@ -669,12 +2209,13 @@ mod tests { scopes: vec!["scope".to_string()], flow: OAuthFlow::ClientCredentials, refresh_buffer_secs: None, + token_cache: None, }; let provider = OAuthHeaderProvider::new(config).unwrap(); let debug = format!("{provider:?}"); assert!(!debug.contains("super-secret")); - assert!(debug.contains("client_secret: \"\"")); + assert!(debug.contains("client_secret: Some(\"\")")); } #[test] @@ -705,6 +2246,7 @@ mod tests { ], flow: OAuthFlow::AzureManagedIdentity { client_id: None }, refresh_buffer_secs: None, + token_cache: None, }; assert!(OAuthHeaderProvider::new(config).is_err()); } @@ -720,7 +2262,7 @@ mod tests { ) .unwrap(); - let err = source.get_token_endpoint().await.unwrap_err(); + let err = source.oidc.get_token_endpoint().await.unwrap_err(); assert!(matches!( err, Error::Runtime { message } @@ -738,6 +2280,7 @@ mod tests { scopes: vec!["scope".to_string()], flow: OAuthFlow::ClientCredentials, refresh_buffer_secs: None, + token_cache: None, }; assert!(OAuthHeaderProvider::new(config).is_err()); } @@ -751,13 +2294,15 @@ mod tests { scopes: vec!["scope".to_string()], flow: OAuthFlow::ClientCredentials, refresh_buffer_secs: None, + token_cache: None, }; let err = OAuthHeaderProvider::new(config).unwrap_err(); assert!(matches!( err, Error::InvalidInput { message } - if message == "ClientCredentials OAuth issuer_url must use https, except for loopback hosts" + if message + == "OAuth issuer_url must use https, except for http on a loopback host" )); } @@ -770,6 +2315,7 @@ mod tests { scopes: vec![], flow: OAuthFlow::AzureManagedIdentity { client_id: None }, refresh_buffer_secs: None, + token_cache: None, }; assert!(OAuthHeaderProvider::new(config).is_err()); } @@ -784,6 +2330,7 @@ mod tests { scopes: vec!["scope".to_string()], flow: OAuthFlow::ClientCredentials, refresh_buffer_secs: Some(0), + token_cache: None, }; let provider = OAuthHeaderProvider::new(config).unwrap(); @@ -805,6 +2352,297 @@ mod tests { server.await.unwrap(); } + async fn spawn_discovery_server(expected_requests: usize) -> (String, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let issuer_url = format!("http://{addr}"); + + let server = tokio::spawn(async move { + for _ in 0..expected_requests { + let (mut stream, _) = listener.accept().await.unwrap(); + let (request_line, _) = read_http_request(&mut stream).await; + assert!(request_line.starts_with("GET /.well-known/openid-configuration ")); + let discovery = format!( + r#"{{"token_endpoint":"http://{addr}/token","authorization_endpoint":"http://{addr}/authorize","device_authorization_endpoint":"http://{addr}/device"}}"# + ); + write_json_response(&mut stream, "200 OK", &discovery).await; + } + }); + + (issuer_url, server) + } + + async fn spawn_insecure_authorization_discovery_server() -> (String, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let issuer_url = format!("http://{addr}"); + let server = tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let (request_line, _) = read_http_request(&mut stream).await; + assert!(request_line.starts_with("GET /.well-known/openid-configuration ")); + write_json_response( + &mut stream, + "200 OK", + r#"{"token_endpoint":"https://idp.example.com/token","authorization_endpoint":"http://idp.example.com/authorize"}"#, + ) + .await; + }); + + (issuer_url, server) + } + + async fn spawn_insecure_device_verification_server() -> (String, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let issuer_url = format!("http://{addr}"); + let server = tokio::spawn(async move { + for _ in 0..2 { + let (mut stream, _) = listener.accept().await.unwrap(); + let (request_line, _) = read_http_request(&mut stream).await; + if request_line.starts_with("GET /.well-known/openid-configuration ") { + let discovery = format!( + r#"{{"token_endpoint":"http://{addr}/token","device_authorization_endpoint":"http://{addr}/device"}}"# + ); + write_json_response(&mut stream, "200 OK", &discovery).await; + } else { + assert!(request_line.starts_with("POST /device ")); + write_json_response( + &mut stream, + "200 OK", + r#"{"device_code":"device-code","user_code":"ABCD-EFGH","verification_uri":"http://idp.example.com/device","expires_in":60}"#, + ) + .await; + } + } + }); + + (issuer_url, server) + } + + async fn spawn_token_exchange_server() -> ( + String, + Arc>>, + JoinHandle<()>, + ) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let issuer_url = format!("http://{addr}"); + let request_body = Arc::new(std::sync::Mutex::new(None)); + let server_request_body = Arc::clone(&request_body); + + let server = tokio::spawn(async move { + for _ in 0..2 { + let (mut stream, _) = listener.accept().await.unwrap(); + let (request_line, body) = read_http_request(&mut stream).await; + if request_line.starts_with("GET /.well-known/openid-configuration ") { + let discovery = format!( + r#"{{"token_endpoint":"http://{addr}/token","authorization_endpoint":"http://{addr}/authorize"}}"# + ); + write_json_response(&mut stream, "200 OK", &discovery).await; + } else if request_line.starts_with("POST /token ") { + *server_request_body.lock().unwrap() = Some(body); + write_json_response( + &mut stream, + "200 OK", + r#"{"access_token":"token","refresh_token":"refresh","expires_in":3600}"#, + ) + .await; + } else { + write_json_response(&mut stream, "404 Not Found", "{}").await; + } + } + }); + + (issuer_url, request_body, server) + } + + async fn spawn_refresh_error_server( + status: &'static str, + response_body: &'static str, + ) -> (String, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let issuer_url = format!("http://{addr}"); + + let server = tokio::spawn(async move { + for _ in 0..2 { + let (mut stream, _) = listener.accept().await.unwrap(); + let (request_line, body) = read_http_request(&mut stream).await; + if request_line.starts_with("GET /.well-known/openid-configuration ") { + let discovery = format!(r#"{{"token_endpoint":"http://{addr}/token"}}"#); + write_json_response(&mut stream, "200 OK", &discovery).await; + } else if request_line.starts_with("POST /token ") { + assert!(body.contains("grant_type=refresh_token")); + assert!(body.contains("refresh_token=")); + write_json_response(&mut stream, status, response_body).await; + } else { + write_json_response(&mut stream, "404 Not Found", "{}").await; + } + } + }); + + (issuer_url, server) + } + + async fn spawn_device_server() -> (String, Arc, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let issuer_url = format!("http://{addr}"); + let token_requests = Arc::new(AtomicUsize::new(0)); + let server_token_requests = Arc::clone(&token_requests); + + let server = tokio::spawn(async move { + for _ in 0..5 { + let (mut stream, _) = listener.accept().await.unwrap(); + let (request_line, body) = read_http_request(&mut stream).await; + if request_line.starts_with("GET /.well-known/openid-configuration ") { + let discovery = format!( + r#"{{"token_endpoint":"http://{addr}/token","device_authorization_endpoint":"http://{addr}/device"}}"# + ); + write_json_response(&mut stream, "200 OK", &discovery).await; + } else if request_line.starts_with("POST /device ") { + assert!(body.contains("client_id=client-id")); + assert!(body.contains("client_secret=secret")); + assert!(body.contains("scope=openid")); + let device = format!( + r#"{{"device_code":"device-code","user_code":"ABCD-EFGH","verification_uri":"http://{addr}/verify","verification_uri_complete":"http://{addr}/verify?user_code=ABCD-EFGH","expires_in":60,"interval":1}}"# + ); + write_json_response(&mut stream, "200 OK", &device).await; + } else if request_line.starts_with("POST /token ") { + assert!(body.contains( + "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Adevice_code" + )); + assert!(body.contains("device_code=device-code")); + assert!(body.contains("client_secret=secret")); + let request = server_token_requests.fetch_add(1, Ordering::SeqCst); + match request { + 0 => { + write_json_response( + &mut stream, + "400 Bad Request", + r#"{"error":"authorization_pending"}"#, + ) + .await; + } + 1 => { + write_json_response( + &mut stream, + "400 Bad Request", + r#"{"error":"slow_down"}"#, + ) + .await; + } + _ => { + write_json_response( + &mut stream, + "200 OK", + r#"{"access_token":"device-token","refresh_token":"device-refresh","expires_in":3600}"#, + ) + .await; + } + } + } else { + write_json_response(&mut stream, "404 Not Found", "{}").await; + } + } + }); + + (issuer_url, token_requests, server) + } + + async fn spawn_device_transient_server() -> (String, Arc, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let issuer_url = format!("http://{addr}"); + let token_requests = Arc::new(AtomicUsize::new(0)); + let server_token_requests = Arc::clone(&token_requests); + + let server = tokio::spawn(async move { + for _ in 0..5 { + let (mut stream, _) = listener.accept().await.unwrap(); + let (request_line, _) = read_http_request(&mut stream).await; + if request_line.starts_with("GET /.well-known/openid-configuration ") { + let discovery = format!(r#"{{"token_endpoint":"http://{addr}/token"}}"#); + write_json_response(&mut stream, "200 OK", &discovery).await; + continue; + } + + assert!(request_line.starts_with("POST /token ")); + match server_token_requests.fetch_add(1, Ordering::SeqCst) { + 0 => drop(stream), + 1 => { + write_json_response( + &mut stream, + "503 Service Unavailable", + r#"{"error":"server_error"}"#, + ) + .await; + } + 2 => { + write_json_response( + &mut stream, + "400 Bad Request", + r#"{"error":"temporarily_unavailable"}"#, + ) + .await; + } + _ => { + write_json_response( + &mut stream, + "200 OK", + r#"{"access_token":"device-token","expires_in":3600}"#, + ) + .await; + } + } + } + }); + + (issuer_url, token_requests, server) + } + + fn test_device_authorization_response( + expires_in: u64, + interval: u64, + ) -> DeviceAuthorizationResponse { + DeviceAuthorizationResponse { + device_code: "device-code".to_string(), + user_code: "ABCD-EFGH".to_string(), + verification_uri: "http://127.0.0.1/verify".to_string(), + verification_uri_complete: None, + expires_in, + interval: Some(interval), + } + } + + async fn spawn_device_error_server(error: &'static str) -> (String, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let issuer_url = format!("http://{addr}"); + + let server = tokio::spawn(async move { + for _ in 0..2 { + let (mut stream, _) = listener.accept().await.unwrap(); + let (request_line, _) = read_http_request(&mut stream).await; + if request_line.starts_with("GET /.well-known/openid-configuration ") { + let discovery = format!(r#"{{"token_endpoint":"http://{addr}/token"}}"#); + write_json_response(&mut stream, "200 OK", &discovery).await; + } else if request_line.starts_with("POST /token ") { + write_json_response( + &mut stream, + "400 Bad Request", + &format!(r#"{{"error":"{error}"}}"#), + ) + .await; + } else { + write_json_response(&mut stream, "404 Not Found", "{}").await; + } + } + }); + + (issuer_url, server) + } + async fn spawn_oauth_server() -> (String, Arc, JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); diff --git a/rust/lancedb/src/remote/token_cache.rs b/rust/lancedb/src/remote/token_cache.rs new file mode 100644 index 000000000..b67ced651 --- /dev/null +++ b/rust/lancedb/src/remote/token_cache.rs @@ -0,0 +1,1753 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Persistent OAuth token cache and session lifecycle APIs. +//! +//! By default, OAuth sessions are kept in process memory only (see +//! [`OAuthHeaderProvider`](crate::remote::OAuthHeaderProvider)). Short-lived +//! processes such as CLI tools, notebooks, or scripts would otherwise have to +//! run a full interactive browser or device flow on every start. Configuring +//! [`TokenCacheOptions`] on an [`OAuthConfig`](crate::remote::OAuthConfig) opts +//! in to an explicit, hardened, on-disk cache that stores only the refresh +//! token plus non-secret metadata, so a second process can silently refresh +//! instead of re-prompting. +//! +//! Security properties: +//! +//! - Opt-in only; callers that do not configure a cache stay memory-only. +//! - Only refresh tokens are persisted. Access tokens never touch disk, so +//! there are no local expiry decisions to get wrong when clocks move. +//! - No client secret is ever stored. +//! - The cache directory is private (`0700`) and each record is `0600`, +//! owner-checked, and symlink-rejected on Unix; records are replaced +//! atomically via `rename` so a crash can never leave a torn file. +//! - Cache filenames are SHA-256 hashes of the canonical issuer, client, +//! scope, flow, and client-auth identity. No secret appears in a filename. +//! - Refresh-token rotation is serialized across processes with a per-key +//! advisory file lock (`flock` on Unix, `LockFileEx` on Windows). The +//! operating system releases these locks when a process dies, so a crash +//! cannot leave a stale lock behind. +//! +//! Use [`OAuthSession`] to explicitly `login`, inspect `status`, or `logout` +//! without issuing a database request. +//! +//! # Example +//! +//! ``` +//! use lancedb::remote::{OAuthConfig, OAuthFlow, TokenCacheOptions}; +//! +//! # async fn example() -> Result<(), Box> { +//! let config = OAuthConfig { +//! issuer_url: "https://issuer.example.com".to_string(), +//! client_id: "my-app".to_string(), +//! client_secret: None, +//! scopes: vec!["openid".to_string()], +//! flow: OAuthFlow::DeviceCode, +//! refresh_buffer_secs: None, +//! token_cache: Some( +//! TokenCacheOptions::new().cache_dir("/tmp/my-app/oauth-cache"), +//! ), +//! }; +//! let session = lancedb::remote::OAuthSession::new(config)?; +//! session.login().await?; +//! let status = session.status().await?; +//! assert!(status.refreshable); +//! # Ok(()) +//! # } +//! ``` + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +use fs4::fs_std::FileExt; +use log::{debug, warn}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +use crate::error::{Error, Result}; +use crate::remote::oauth::{OAuthConfig, OAuthFlow, RefreshResult, TokenResponse, TokenSource}; + +const CACHE_RECORD_VERSION: u32 = 1; +const DEFAULT_LOCK_TIMEOUT_SECS: u64 = 30; +const LOCK_POLL_INTERVAL: Duration = Duration::from_millis(100); + +fn now_unix_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|duration| duration.as_secs()) + .unwrap_or(0) +} + +/// Options for the persistent OAuth token cache. +/// +/// The cache is opt-in: it is only used when set on +/// [`OAuthConfig::token_cache`](crate::remote::OAuthConfig::token_cache). See +/// the [module documentation](self) for the security properties. +#[derive(Clone, Debug, Default)] +pub struct TokenCacheOptions { + /// Directory that holds the cached credentials. + /// + /// Defaults to `$XDG_CACHE_HOME/lancedb/oauth`, `$HOME/.cache/lancedb/oauth` + /// on Unix, or `%LOCALAPPDATA%\lancedb\oauth` on Windows. The directory is + /// created with owner-only permissions (`0700`) when missing. + pub cache_dir: Option, + + /// How long to wait for the cross-process refresh lock before failing. + /// + /// Defaults to 30 seconds. + pub lock_timeout_secs: Option, +} + +impl TokenCacheOptions { + /// Create cache options with all defaults. + pub fn new() -> Self { + Self::default() + } + + /// Set the directory that holds cached credentials. + pub fn cache_dir(mut self, cache_dir: impl Into) -> Self { + self.cache_dir = Some(cache_dir.into()); + self + } + + /// Set the cross-process refresh lock timeout in seconds. + pub fn lock_timeout_secs(mut self, secs: u64) -> Self { + self.lock_timeout_secs = Some(secs); + self + } + + fn resolved_dir(&self) -> Result { + if let Some(dir) = &self.cache_dir { + if dir.as_os_str().is_empty() { + return Err(Error::InvalidInput { + message: "OAuth token cache directory must not be empty".to_string(), + }); + } + return Ok(dir.clone()); + } + default_cache_dir().ok_or_else(|| Error::InvalidInput { + message: "Could not determine a default OAuth token cache directory; \ + set TokenCacheOptions::cache_dir or XDG_CACHE_HOME/HOME" + .to_string(), + }) + } + + fn lock_timeout(&self) -> Duration { + Duration::from_secs(self.lock_timeout_secs.unwrap_or(DEFAULT_LOCK_TIMEOUT_SECS)) + } +} + +#[cfg(unix)] +fn default_cache_dir() -> Option { + let base = std::env::var_os("XDG_CACHE_HOME") + .filter(|value| !value.is_empty()) + .map(PathBuf::from) + .or_else(|| { + std::env::var_os("HOME") + .filter(|value| !value.is_empty()) + .map(|home| { + let mut path = PathBuf::from(home); + path.push(".cache"); + path + }) + })?; + let mut dir = base; + dir.push("lancedb"); + dir.push("oauth"); + Some(dir) +} + +#[cfg(windows)] +fn default_cache_dir() -> Option { + let base = std::env::var_os("LOCALAPPDATA") + .filter(|value| !value.is_empty()) + .map(PathBuf::from)?; + let mut dir = base; + dir.push("lancedb"); + dir.push("oauth"); + Some(dir) +} + +#[cfg(not(any(unix, windows)))] +fn default_cache_dir() -> Option { + None +} + +/// A cached OAuth session record. +/// +/// Only the refresh token is persisted. The metadata mirrors the cache key so +/// `status` can report what a record belongs to without exposing secrets. +#[derive(Serialize, Deserialize)] +struct CachedTokenRecord { + version: u32, + issuer_url: String, + client_id: String, + scopes: Vec, + flow: String, + client_auth: String, + refresh_token: String, + obtained_at: u64, +} + +impl std::fmt::Debug for CachedTokenRecord { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CachedTokenRecord") + .field("version", &self.version) + .field("issuer_url", &self.issuer_url) + .field("client_id", &self.client_id) + .field("scopes", &self.scopes) + .field("flow", &self.flow) + .field("client_auth", &self.client_auth) + .field("refresh_token", &"") + .field("obtained_at", &self.obtained_at) + .finish() + } +} + +fn canonicalize_issuer(issuer_url: &str) -> String { + issuer_url.trim_end_matches('/').to_string() +} + +fn canonicalize_scopes(scopes: &[String]) -> Vec { + let mut canonical: Vec = scopes + .iter() + .map(|scope| scope.trim().to_string()) + .filter(|scope| !scope.is_empty()) + .collect(); + canonical.sort(); + canonical.dedup(); + canonical +} + +/// Returns the cache identity of a flow, or `None` for flows that never +/// persist: client credentials have no refresh token to store, and Azure +/// managed identity is machine identity that must not enter a user token +/// cache (rejected separately with an explicit error). +fn flow_key(flow: &OAuthFlow) -> Option<&'static str> { + match flow { + OAuthFlow::AuthorizationCode(_) => Some("authorization_code"), + OAuthFlow::DeviceCode => Some("device_code"), + OAuthFlow::ClientCredentials | OAuthFlow::AzureManagedIdentity { .. } => None, + } +} + +fn client_auth_key(client_secret: Option<&str>) -> &'static str { + if client_secret.is_some() { + "confidential" + } else { + "public" + } +} + +/// Identity of one cached session: canonical issuer, client, scopes, flow, +/// and client-auth mode, plus the hashed filename derived from it. +#[derive(Clone, Debug)] +struct CacheKey { + issuer_url: String, + client_id: String, + scopes: Vec, + flow: &'static str, + client_auth: &'static str, + file_stem: String, +} + +impl CacheKey { + fn new(config: &OAuthConfig) -> Result { + let flow = flow_key(&config.flow).ok_or_else(|| Error::InvalidInput { + message: format!( + "A persistent OAuth token cache is not supported for the {:?} flow; \ + remove TokenCacheOptions to keep tokens in memory", + config.flow + ), + })?; + let issuer_url = canonicalize_issuer(&config.issuer_url); + let scopes = canonicalize_scopes(&config.scopes); + let client_auth = client_auth_key(config.client_secret.as_deref()); + let identity = format!( + "v1\n{}\n{}\n{}\n{}\n{}", + issuer_url, + config.client_id, + scopes.join(" "), + flow, + client_auth + ); + let file_stem = hex_sha256(identity.as_bytes()); + Ok(Self { + issuer_url, + client_id: config.client_id.clone(), + scopes, + flow, + client_auth, + file_stem, + }) + } +} + +fn hex_sha256(bytes: &[u8]) -> String { + let digest = Sha256::digest(bytes); + let mut hex = String::with_capacity(digest.len() * 2); + for byte in digest { + use std::fmt::Write; + let _ = write!(hex, "{byte:02x}"); + } + hex +} + +/// Guard for the per-key cross-process refresh lock. +/// +/// The lock is an advisory exclusive lock on a per-key file. The operating +/// system releases it when the owning process exits, so crashes cannot strand +/// a stale lock. +struct LockGuard { + #[allow(dead_code)] + file: std::fs::File, +} + +/// The persistent token cache engine for one [`OAuthConfig`]. +pub(crate) struct TokenCache { + dir: PathBuf, + key: CacheKey, + lock_timeout: Duration, + #[cfg(unix)] + dir_owner: u32, +} + +impl std::fmt::Debug for TokenCache { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TokenCache") + .field("dir", &self.dir) + .field("flow", &self.key.flow) + .finish() + } +} + +impl TokenCache { + fn new(config: &OAuthConfig, options: &TokenCacheOptions) -> Result { + let key = CacheKey::new(config)?; + let dir = options.resolved_dir()?; + let lock_timeout = options.lock_timeout(); + prepare_cache_dir(&dir)?; + #[cfg(unix)] + let dir_owner = { + use std::os::unix::fs::MetadataExt; + let metadata = std::fs::metadata(&dir).map_err(|e| Error::Runtime { + message: format!( + "Failed to inspect OAuth token cache directory {}: {e}", + dir.display() + ), + })?; + metadata.uid() + }; + Ok(Self { + dir, + key, + lock_timeout, + #[cfg(unix)] + dir_owner, + }) + } + + fn record_path(&self) -> PathBuf { + let mut path = self.dir.clone(); + path.push(format!("{}.token.json", self.key.file_stem)); + path + } + + fn lock_path(&self) -> PathBuf { + let mut path = self.dir.clone(); + path.push(format!("{}.lock", self.key.file_stem)); + path + } + + /// Load the cached record, if one exists and passes hardening checks. + /// + /// Corrupt, truncated, unknown-version, or permission-invalid records + /// return an actionable error instead of being silently ignored or + /// deleted; the message names the file and how to recover. + async fn load(&self) -> Result> { + let path = self.record_path(); + let dir_owner = self.dir_owner_or_zero(); + tokio::task::spawn_blocking(move || read_record(&path, dir_owner)) + .await + .map_err(|e| Error::Runtime { + message: format!("Failed to join OAuth token cache read: {e}"), + })? + } + + #[cfg(unix)] + fn dir_owner_or_zero(&self) -> u32 { + self.dir_owner + } + + #[cfg(not(unix))] + fn dir_owner_or_zero(&self) -> u32 { + 0 + } + + /// Build a record from a token response, or `None` when the response + /// carries no refresh token (nothing may be persisted). + fn record_from_response(&self, response: &TokenResponse) -> Option { + let refresh_token = response.refresh_token.clone()?; + Some(CachedTokenRecord { + version: CACHE_RECORD_VERSION, + issuer_url: self.key.issuer_url.clone(), + client_id: self.key.client_id.clone(), + scopes: self.key.scopes.clone(), + flow: self.key.flow.to_string(), + client_auth: self.key.client_auth.to_string(), + refresh_token, + obtained_at: now_unix_secs(), + }) + } + + /// Atomically replace the cached record. + async fn store(&self, record: &CachedTokenRecord) -> Result<()> { + let path = self.record_path(); + let dir = self.dir.clone(); + let payload = serde_json::to_vec(record).map_err(|e| Error::Runtime { + message: format!("Failed to serialize OAuth token cache record: {e}"), + })?; + tokio::task::spawn_blocking(move || write_record(&dir, &path, &payload)) + .await + .map_err(|e| Error::Runtime { + message: format!("Failed to join OAuth token cache write: {e}"), + })? + } + + /// Delete the cached record. Returns whether a record was removed. + async fn delete(&self) -> Result { + let path = self.record_path(); + tokio::task::spawn_blocking(move || match std::fs::remove_file(&path) { + Ok(()) => Ok(true), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(Error::Runtime { + message: format!( + "Failed to remove OAuth token cache record {}: {e}", + path.display() + ), + }), + }) + .await + .map_err(|e| Error::Runtime { + message: format!("Failed to join OAuth token cache delete: {e}"), + })? + } + + /// Acquire the per-key cross-process lock, polling until the timeout. + async fn acquire_lock(&self) -> Result { + let path = self.lock_path(); + let timeout = self.lock_timeout; + tokio::task::spawn_blocking(move || { + use std::fs::OpenOptions; + let file = OpenOptions::new() + .create(true) + .read(true) + .write(true) + .truncate(false) + .open(&path) + .map_err(|e| Error::Runtime { + message: format!( + "Failed to open OAuth token cache lock file {}: {e}", + path.display() + ), + })?; + #[cfg(unix)] + set_owner_only_permissions(&file, &path); + let deadline = std::time::Instant::now() + timeout; + loop { + match file.try_lock_exclusive() { + Ok(true) => return Ok(LockGuard { file }), + Ok(false) => { + if std::time::Instant::now() >= deadline { + return Err(Error::Runtime { + message: format!( + "Timed out after {}s waiting for the OAuth token cache \ + lock at {}", + timeout.as_secs(), + path.display() + ), + }); + } + std::thread::sleep(LOCK_POLL_INTERVAL); + } + Err(error) => { + return Err(Error::Runtime { + message: format!( + "Failed to lock the OAuth token cache file {}: {error}", + path.display() + ), + }); + } + } + } + }) + .await + .map_err(|e| Error::Runtime { + message: format!("Failed to join OAuth token cache lock acquisition: {e}"), + })? + } + + /// Run the cross-process refresh critical section. + /// + /// Must be called with the in-process write lock held. Refresh grants are + /// serialized by the per-key cross-process lock; interactive flows (first + /// login or reauthentication) run outside it so a slow human-in-the-loop + /// flow never blocks refreshes in other processes. + pub(crate) async fn refresh_or_acquire( + &self, + source: &dyn TokenSource, + ) -> Result { + // Fast path under the lock: refresh from the durable record. + { + let _guard = self.acquire_lock().await?; + // Reread the record: another process may have rotated the refresh + // token since this process last looked. + if let Some(record) = self.load().await? { + match source.refresh_token(&record.refresh_token).await? { + RefreshResult::Refreshed(response) => { + self.store_if_refreshable(&response).await?; + return Ok(response); + } + RefreshResult::Reauthenticate => { + warn!( + "Cached OAuth refresh token was rejected; removing the cached \ + session before reauthenticating via {:?}", + source + ); + self.delete().await?; + } + RefreshResult::Unsupported => {} + } + } else { + debug!("No cached OAuth session; acquiring one via {:?}", source); + } + } + + // Interactive acquisition happens without the cross-process lock. + // Concurrent logins are independent sessions; the last store wins, + // which is the documented multi-session rule. + let response = source.fetch_token().await?; + let _guard = self.acquire_lock().await?; + self.store_if_refreshable(&response).await?; + Ok(response) + } + + /// Store a fresh login response (used by the eager `login` API). + /// + /// A successful login atomically replaces any prior session for this + /// cache identity: when the provider does not issue a refresh token, the + /// previous record is removed rather than left in place, so logging in + /// can never silently keep an earlier account's credential. + async fn store_login_response(&self, response: &TokenResponse) -> Result<()> { + let _guard = self.acquire_lock().await?; + match self.record_from_response(response) { + Some(record) => self.store(&record).await?, + None => { + self.delete().await?; + } + } + Ok(()) + } + + /// Replace the record when the response carries a refresh token. + /// + /// Called with the cross-process lock already held. Responses without a + /// refresh token are not persistable and are skipped. + async fn store_if_refreshable(&self, response: &TokenResponse) -> Result<()> { + if let Some(record) = self.record_from_response(response) { + self.store(&record).await?; + } else { + debug!( + "OAuth response did not include a refresh token; nothing to cache for {:?}", + self.key.flow + ); + } + Ok(()) + } +} + +fn prepare_cache_dir(dir: &Path) -> Result<()> { + if dir.is_dir() { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let metadata = std::fs::metadata(dir).map_err(|e| Error::Runtime { + message: format!( + "Failed to inspect OAuth token cache directory {}: {e}", + dir.display() + ), + })?; + let mode = metadata.mode(); + if mode & 0o077 != 0 { + return Err(Error::InvalidInput { + message: format!( + "OAuth token cache directory {} must not be accessible by group or \ + other users (mode {:o}); run `chmod 700` on it or choose a \ + private directory", + dir.display(), + mode & 0o777 + ), + }); + } + } + return Ok(()); + } + if dir.exists() { + return Err(Error::InvalidInput { + message: format!( + "OAuth token cache path {} exists and is not a directory", + dir.display() + ), + }); + } + let mut builder = std::fs::DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(dir).map_err(|e| Error::Runtime { + message: format!( + "Failed to create OAuth token cache directory {}: {e}", + dir.display() + ), + }) +} + +fn read_record(path: &Path, dir_owner: u32) -> Result> { + let metadata = match std::fs::symlink_metadata(path) { + Ok(metadata) => metadata, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(e) => { + return Err(Error::Runtime { + message: format!( + "Failed to read OAuth token cache record {}: {e}", + path.display() + ), + }); + } + }; + if !metadata.is_file() { + return Err(Error::InvalidInput { + message: format!( + "OAuth token cache record {} is not a regular file; refusing to use it. \ + Remove the file or call logout to clear it", + path.display() + ), + }); + } + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let mode = metadata.mode(); + if mode & 0o077 != 0 { + return Err(Error::InvalidInput { + message: format!( + "OAuth token cache record {} must not be accessible by group or other \ + users (mode {:o}); run `chmod 600` on it or call logout to clear it", + path.display(), + mode & 0o777 + ), + }); + } + if dir_owner != 0 && metadata.uid() != dir_owner { + return Err(Error::InvalidInput { + message: format!( + "OAuth token cache record {} is owned by a different user; refusing to \ + use it. Remove the file or call logout to clear it", + path.display() + ), + }); + } + } + let payload = std::fs::read_to_string(path).map_err(|e| Error::Runtime { + message: format!( + "Failed to read OAuth token cache record {}: {e}", + path.display() + ), + })?; + let record: CachedTokenRecord = serde_json::from_str(&payload).map_err(|e| Error::Runtime { + message: format!( + "OAuth token cache record {} is corrupt ({e}); remove the file or call \ + logout to clear it", + path.display() + ), + })?; + if record.version != CACHE_RECORD_VERSION { + return Err(Error::Runtime { + message: format!( + "OAuth token cache record {} has unsupported version {}; expected {}. \ + Remove the file or call logout to clear it", + path.display(), + record.version, + CACHE_RECORD_VERSION + ), + }); + } + Ok(Some(record)) +} + +fn write_record(dir: &Path, path: &Path, payload: &[u8]) -> Result<()> { + let random_suffix: String = { + use rand::Rng; + let mut rng = rand::rng(); + (0..8) + .map(|_| format!("{:x}", rng.random_range(0..16u32))) + .collect() + }; + let mut temp_path = dir.to_path_buf(); + temp_path.push(format!( + "{}.tmp.{}.{}", + path.file_name() + .map(|name| name.to_string_lossy().to_string()) + .unwrap_or_default(), + std::process::id(), + random_suffix + )); + let write_attempt = || -> std::io::Result<()> { + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + let file = options.open(&temp_path)?; + #[cfg(unix)] + set_owner_only_permissions(&file, &temp_path); + use std::io::Write; + let mut writer = std::io::BufWriter::new(&file); + writer.write_all(payload)?; + writer.flush()?; + drop(writer); + file.sync_all()?; + std::fs::rename(&temp_path, path)?; + Ok(()) + }; + write_attempt().map_err(|e| { + let _ = std::fs::remove_file(&temp_path); + Error::Runtime { + message: format!( + "Failed to write OAuth token cache record {}: {e}", + path.display() + ), + } + }) +} + +#[cfg(unix)] +fn set_owner_only_permissions(file: &std::fs::File, path: &Path) { + use std::os::unix::fs::PermissionsExt; + if let Err(e) = file.set_permissions(std::fs::Permissions::from_mode(0o600)) { + debug!("Could not restrict permissions on {}: {e}", path.display()); + } +} + +/// Safe, non-secret view of a cached OAuth session, returned by +/// [`OAuthSession::status`] and [`OAuthSession::login`]. +#[derive(Clone, Debug, PartialEq)] +pub struct SessionStatus { + /// Whether a cached session exists that can obtain tokens without + /// interactive authentication. + /// + /// Because access tokens are not persisted, this is `true` exactly when a + /// refresh token is cached; the next connection refreshes with it rather + /// than opening a browser or device prompt. + pub refreshable: bool, + + /// Canonical issuer URL of the cached session. + pub issuer_url: String, + + /// Client ID of the cached session. + pub client_id: String, + + /// Canonical (sorted, de-duplicated) scope set of the cached session. + pub scopes: Vec, + + /// Flow that produced the cached session. + pub flow: String, + + /// When the cached session was obtained, as Unix seconds. + pub obtained_at: Option, +} + +/// Result of [`OAuthSession::logout`]. +#[derive(Clone, Debug, PartialEq)] +pub struct SessionLogout { + /// Whether a cached credential was removed. `false` means no matching + /// session was cached; logout is idempotent. + pub removed: bool, +} + +/// Explicit OAuth session lifecycle: eager `login`, non-secret `status`, and +/// local `logout` for the persistent token cache. +/// +/// A session is built from the same [`OAuthConfig`](crate::remote::OAuthConfig) +/// used to connect (including its +/// [`token_cache`](crate::remote::OAuthConfig::token_cache) options). A +/// connection created with the same configuration shares the cache, so logging +/// in here prepares tokens for later processes without any database request. +/// +/// `login` always runs the configured interactive flow and replaces the +/// cached session (the most recent login wins; see the module documentation +/// about multiple accounts). `logout` removes only the local credential; it +/// does not revoke anything with the provider and does not terminate a +/// browser SSO session. +/// +/// # Example +/// +/// ``` +/// # use lancedb::remote::{OAuthConfig, OAuthFlow, OAuthSession, TokenCacheOptions}; +/// # fn example() -> lancedb::error::Result<()> { +/// let config = OAuthConfig { +/// issuer_url: "https://issuer.example.com".to_string(), +/// client_id: "my-app".to_string(), +/// client_secret: None, +/// scopes: vec!["openid".to_string()], +/// flow: OAuthFlow::DeviceCode, +/// refresh_buffer_secs: None, +/// token_cache: Some(TokenCacheOptions::new()), +/// }; +/// let session = OAuthSession::new(config)?; +/// # Ok(()) +/// # } +/// ``` +pub struct OAuthSession { + token_source: Box, + cache: TokenCache, +} + +impl std::fmt::Debug for OAuthSession { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OAuthSession") + .field("cache", &self.cache) + .finish() + } +} + +impl OAuthSession { + /// Create a session manager for the given configuration. + /// + /// The configuration must enable [`TokenCacheOptions`] and use a flow + /// that supports persistent sessions (authorization code or device + /// authorization). + pub fn new(config: OAuthConfig) -> Result { + let cache_options = config + .token_cache + .clone() + .ok_or_else(|| Error::InvalidInput { + message: "OAuthSession requires OAuthConfig.token_cache to be set".to_string(), + })?; + if config.scopes.is_empty() { + return Err(Error::InvalidInput { + message: "At least one OAuth scope is required".to_string(), + }); + } + let token_source = crate::remote::oauth::build_token_source(&config)?; + let cache = TokenCache::new(&config, &cache_options)?; + Ok(Self { + token_source, + cache, + }) + } + + /// Eagerly run the configured authentication flow and store the session. + /// + /// Returns the resulting [`SessionStatus`]. A successful login always + /// replaces any prior cached session for this identity; if the provider + /// does not issue a refresh token (for example without `offline_access`), + /// the previous record is removed and the status reports + /// `refreshable == false`. + pub async fn login(&self) -> Result { + let response = self.token_source.fetch_token().await?; + self.cache.store_login_response(&response).await?; + self.status().await + } + + /// Report whether a matching cached session exists, with safe metadata. + /// + /// This never contacts the identity provider and never exposes token + /// values. + pub async fn status(&self) -> Result { + let record = self.cache.load().await?; + Ok(match record { + Some(record) => SessionStatus { + refreshable: true, + issuer_url: record.issuer_url, + client_id: record.client_id, + scopes: record.scopes, + flow: record.flow, + obtained_at: Some(record.obtained_at), + }, + None => SessionStatus { + refreshable: false, + issuer_url: self.cache.key.issuer_url.clone(), + client_id: self.cache.key.client_id.clone(), + scopes: self.cache.key.scopes.clone(), + flow: self.cache.key.flow.to_string(), + obtained_at: None, + }, + }) + } + + /// Remove the matching local cached credential. + /// + /// This only deletes the local cache entry. It does not revoke the + /// refresh token with the provider and does not sign out of a browser + /// SSO session. Repeated calls succeed; `removed` reports whether a + /// credential existed. + pub async fn logout(&self) -> Result { + let removed = self.cache.delete().await?; + Ok(SessionLogout { removed }) + } +} + +/// Resolve the persistent cache for a configuration, if one should exist. +/// +/// Returns `Ok(None)` for configurations without cache options and for the +/// client-credentials flow, which has no refresh token to persist (a debug +/// note is logged). The Azure managed-identity flow is rejected because +/// machine identity must not enter a user token cache. +pub(crate) fn token_cache_for_config(config: &OAuthConfig) -> Result>> { + let Some(options) = &config.token_cache else { + return Ok(None); + }; + if matches!(config.flow, OAuthFlow::AzureManagedIdentity { .. }) { + return Err(Error::InvalidInput { + message: "A persistent OAuth token cache cannot be used with the \ + AzureManagedIdentity flow; remove TokenCacheOptions to keep the \ + machine identity token in memory" + .to_string(), + }); + } + if matches!(config.flow, OAuthFlow::ClientCredentials) { + debug!( + "The client-credentials flow has no refresh token to persist; the OAuth \ + token cache is not used" + ); + return Ok(None); + } + Ok(Some(Arc::new(TokenCache::new(config, options)?))) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::{TcpListener, TcpStream}; + + use crate::remote::HeaderProvider; + use crate::remote::oauth::OAuthHeaderProvider; + use serial_test::serial; + + /// Temp directory that satisfies the cache hardening checks. CI runners + /// can create temp directories with group/other bits set, which the + /// private-directory validation correctly rejects. + fn cache_tempdir() -> tempfile::TempDir { + let dir = tempfile::tempdir().unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + } + dir + } + + fn device_config(cache_dir: &Path) -> OAuthConfig { + OAuthConfig { + issuer_url: "https://issuer.example.com".to_string(), + client_id: "client-id".to_string(), + client_secret: None, + scopes: vec!["openid".to_string()], + flow: OAuthFlow::DeviceCode, + refresh_buffer_secs: None, + token_cache: Some(TokenCacheOptions::new().cache_dir(cache_dir)), + } + } + + /// Stateful mock IdP covering discovery, device authorization, device + /// polling, client credentials, and refresh with strict rotation: a + /// refresh token that is not the currently issued one is rejected with + /// `invalid_grant`, which is exactly what real providers do on rotation. + struct MockIdp { + issuer_url: String, + device_authorizations: Arc, + refresh_attempts: Arc, + invalid_grant_rejections: Arc, + access_tokens_issued: Arc, + current_refresh: Arc>>, + fail_refreshes: Arc, + issue_refresh_tokens: Arc, + } + + impl MockIdp { + async fn start() -> Self { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let issuer_url = format!("http://{addr}"); + let server = Self { + issuer_url: issuer_url.clone(), + device_authorizations: Arc::new(AtomicUsize::new(0)), + refresh_attempts: Arc::new(AtomicUsize::new(0)), + invalid_grant_rejections: Arc::new(AtomicUsize::new(0)), + access_tokens_issued: Arc::new(AtomicUsize::new(0)), + current_refresh: Arc::new(std::sync::Mutex::new(None)), + fail_refreshes: Arc::new(AtomicBool::new(false)), + issue_refresh_tokens: Arc::new(AtomicBool::new(true)), + }; + let device_authorizations = Arc::clone(&server.device_authorizations); + let refresh_attempts = Arc::clone(&server.refresh_attempts); + let invalid_grant_rejections = Arc::clone(&server.invalid_grant_rejections); + let access_tokens_issued = Arc::clone(&server.access_tokens_issued); + let current_refresh = Arc::clone(&server.current_refresh); + let fail_refreshes = Arc::clone(&server.fail_refreshes); + let issue_refresh_tokens = Arc::clone(&server.issue_refresh_tokens); + + tokio::spawn(async move { + loop { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + let device_authorizations = Arc::clone(&device_authorizations); + let refresh_attempts = Arc::clone(&refresh_attempts); + let invalid_grant_rejections = Arc::clone(&invalid_grant_rejections); + let access_tokens_issued = Arc::clone(&access_tokens_issued); + let current_refresh = Arc::clone(¤t_refresh); + let fail_refreshes = Arc::clone(&fail_refreshes); + let issue_refresh_tokens = Arc::clone(&issue_refresh_tokens); + tokio::spawn(async move { + let (request_line, body) = read_http_request(&mut stream).await; + if request_line.starts_with("GET /.well-known/openid-configuration ") { + let discovery = format!( + r#"{{"token_endpoint":"http://{addr}/token","device_authorization_endpoint":"http://{addr}/device"}}"# + ); + write_json_response(&mut stream, "200 OK", &discovery).await; + } else if request_line.starts_with("POST /device ") { + device_authorizations.fetch_add(1, Ordering::SeqCst); + let device = format!( + r#"{{"device_code":"device-code","user_code":"ABCD-EFGH","verification_uri":"http://{addr}/verify","expires_in":60,"interval":1}}"# + ); + write_json_response(&mut stream, "200 OK", &device).await; + } else if request_line.starts_with("POST /token ") { + if body.contains("grant_type=refresh_token") { + refresh_attempts.fetch_add(1, Ordering::SeqCst); + if fail_refreshes.load(Ordering::SeqCst) { + write_json_response( + &mut stream, + "503 Service Unavailable", + r#"{"error":"temporarily_unavailable"}"#, + ) + .await; + return; + } + let expected = current_refresh.lock().unwrap().clone(); + let matched = body + .split('&') + .find_map(|pair| pair.strip_prefix("refresh_token=")) + .map(|token| token.to_string()) + .zip(expected) + .is_some_and(|(offered, expected)| offered == expected); + if !matched { + invalid_grant_rejections.fetch_add(1, Ordering::SeqCst); + write_json_response( + &mut stream, + "400 Bad Request", + r#"{"error":"invalid_grant"}"#, + ) + .await; + return; + } + issue_access_token( + &mut stream, + &access_tokens_issued, + ¤t_refresh, + issue_refresh_tokens.load(Ordering::SeqCst), + ) + .await; + } else { + // Device polling or client credentials: issue + // a token and, for interactive grants, a fresh + // refresh token with strict rotation. + let grant_device = + body.contains("grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Adevice_code"); + if grant_device { + issue_access_token( + &mut stream, + &access_tokens_issued, + ¤t_refresh, + issue_refresh_tokens.load(Ordering::SeqCst), + ) + .await; + } else { + let token = format!( + r#"{{"access_token":"access-{}","expires_in":3600}}"#, + access_tokens_issued.fetch_add(1, Ordering::SeqCst) + 1 + ); + write_json_response(&mut stream, "200 OK", &token).await; + } + } + } else { + write_json_response(&mut stream, "404 Not Found", "{}").await; + } + }); + } + }); + server + } + + fn config(&self, cache_dir: &Path) -> OAuthConfig { + let mut config = device_config(cache_dir); + config.issuer_url = self.issuer_url.clone(); + config + } + } + + async fn issue_access_token( + stream: &mut TcpStream, + access_tokens_issued: &AtomicUsize, + current_refresh: &std::sync::Mutex>, + with_refresh: bool, + ) { + let number = access_tokens_issued.fetch_add(1, Ordering::SeqCst) + 1; + let token = if with_refresh { + *current_refresh.lock().unwrap() = Some(format!("refresh-{number}")); + format!( + r#"{{"access_token":"access-{number}","refresh_token":"refresh-{number}","expires_in":3600}}"# + ) + } else { + format!(r#"{{"access_token":"access-{number}","expires_in":3600}}"#) + }; + write_json_response(stream, "200 OK", &token).await; + } + + async fn read_http_request(stream: &mut TcpStream) -> (String, String) { + let mut buffer = Vec::new(); + let mut header_end = None; + while header_end.is_none() { + let mut chunk = [0; 1024]; + let read = stream.read(&mut chunk).await.unwrap(); + assert_ne!(read, 0, "connection closed before request headers"); + buffer.extend_from_slice(&chunk[..read]); + header_end = find_subsequence(&buffer, b"\r\n\r\n").map(|pos| pos + 4); + } + let header_end = header_end.unwrap(); + let headers = String::from_utf8_lossy(&buffer[..header_end]).to_string(); + let request_line = headers.lines().next().unwrap_or_default().to_string(); + let content_length = headers + .lines() + .find_map(|line| { + let (name, value) = line.split_once(':')?; + name.eq_ignore_ascii_case("content-length") + .then(|| value.trim().parse::().ok()) + .flatten() + }) + .unwrap_or(0); + while buffer.len() < header_end + content_length { + let mut chunk = [0; 1024]; + let read = stream.read(&mut chunk).await.unwrap(); + assert_ne!(read, 0, "connection closed before request body"); + buffer.extend_from_slice(&chunk[..read]); + } + let body = + String::from_utf8_lossy(&buffer[header_end..header_end + content_length]).to_string(); + (request_line, body) + } + + fn find_subsequence(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) + } + + async fn write_json_response(stream: &mut TcpStream, status: &str, body: &str) { + let response = format!( + "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", + body.len() + ); + stream.write_all(response.as_bytes()).await.unwrap(); + } + + fn suppress_browser() { + // Point the OAuth browser helper at a no-op so device-flow tests never + // open a real browser window. + unsafe { std::env::set_var("LANCEDB_OAUTH_BROWSER", "/usr/bin/true") }; + } + + #[tokio::test] + #[serial] + async fn test_provider_reuses_cached_session_across_instances() { + suppress_browser(); + let dir = cache_tempdir(); + let idp = MockIdp::start().await; + + let first = OAuthHeaderProvider::new(idp.config(dir.path())).unwrap(); + let headers = first.get_headers().await.unwrap(); + assert_eq!(headers.get("authorization").unwrap(), "Bearer access-1"); + + // A second, independent provider (simulating a second process) must + // refresh silently instead of starting another device flow. + let second = OAuthHeaderProvider::new(idp.config(dir.path())).unwrap(); + let headers = second.get_headers().await.unwrap(); + assert_eq!(headers.get("authorization").unwrap(), "Bearer access-2"); + + assert_eq!(idp.device_authorizations.load(Ordering::SeqCst), 1); + assert_eq!(idp.refresh_attempts.load(Ordering::SeqCst), 1); + + let cache = crate::remote::token_cache::TokenCache::new( + &idp.config(dir.path()), + &TokenCacheOptions::new().cache_dir(dir.path()), + ) + .unwrap(); + let record = cache.load().await.unwrap().unwrap(); + assert_eq!(record.refresh_token, "refresh-2"); + } + + #[tokio::test] + #[serial] + async fn test_concurrent_providers_serialize_rotation() { + suppress_browser(); + let dir = cache_tempdir(); + let idp = MockIdp::start().await; + + let priming = OAuthHeaderProvider::new(idp.config(dir.path())).unwrap(); + priming.get_headers().await.unwrap(); + assert_eq!(idp.device_authorizations.load(Ordering::SeqCst), 1); + + let provider_a = OAuthHeaderProvider::new(idp.config(dir.path())).unwrap(); + let provider_b = OAuthHeaderProvider::new(idp.config(dir.path())).unwrap(); + let (headers_a, headers_b) = + tokio::join!(provider_a.get_headers(), provider_b.get_headers()); + let token_a = headers_a.unwrap().remove("authorization").unwrap(); + let token_b = headers_b.unwrap().remove("authorization").unwrap(); + assert!( + { + let mut tokens = [token_a, token_b]; + tokens.sort(); + tokens + } == ["Bearer access-2".to_string(), "Bearer access-3".to_string()], + "each provider must observe its own refreshed token" + ); + + // Rotation raced would produce an invalid_grant and a second device + // flow; the lock prevents both. + assert_eq!(idp.invalid_grant_rejections.load(Ordering::SeqCst), 0); + assert_eq!(idp.device_authorizations.load(Ordering::SeqCst), 1); + assert_eq!(idp.refresh_attempts.load(Ordering::SeqCst), 2); + + let cache = crate::remote::token_cache::TokenCache::new( + &idp.config(dir.path()), + &TokenCacheOptions::new().cache_dir(dir.path()), + ) + .unwrap(); + let record = cache.load().await.unwrap().unwrap(); + assert_eq!(record.refresh_token, "refresh-3"); + } + + #[tokio::test] + #[serial] + async fn test_transient_refresh_failure_retains_record() { + suppress_browser(); + let dir = cache_tempdir(); + let idp = MockIdp::start().await; + + let priming = OAuthHeaderProvider::new(idp.config(dir.path())).unwrap(); + priming.get_headers().await.unwrap(); + + idp.fail_refreshes.store(true, Ordering::SeqCst); + let second = OAuthHeaderProvider::new(idp.config(dir.path())).unwrap(); + let error = second.get_headers().await.unwrap_err(); + assert!(error.to_string().contains("503")); + + let session = OAuthSession::new(idp.config(dir.path())).unwrap(); + let status = session.status().await.unwrap(); + assert!( + status.refreshable, + "transient failures must keep the record" + ); + } + + #[tokio::test] + #[serial] + async fn test_invalid_grant_deletes_record_and_reauthenticates() { + suppress_browser(); + let dir = cache_tempdir(); + let idp = MockIdp::start().await; + + let priming = OAuthHeaderProvider::new(idp.config(dir.path())).unwrap(); + priming.get_headers().await.unwrap(); + + // Simulate a revoked refresh token by replacing the record with one + // the provider never issued. + let cache = crate::remote::token_cache::TokenCache::new( + &idp.config(dir.path()), + &TokenCacheOptions::new().cache_dir(dir.path()), + ) + .unwrap(); + let mut record = cache.load().await.unwrap().unwrap(); + record.refresh_token = "revoked-refresh".to_string(); + cache.store(&record).await.unwrap(); + + let second = OAuthHeaderProvider::new(idp.config(dir.path())).unwrap(); + let headers = second.get_headers().await.unwrap(); + assert_eq!(headers.get("authorization").unwrap(), "Bearer access-2"); + + assert_eq!(idp.invalid_grant_rejections.load(Ordering::SeqCst), 1); + assert_eq!(idp.device_authorizations.load(Ordering::SeqCst), 2); + + let record = cache.load().await.unwrap().unwrap(); + assert_eq!(record.refresh_token, "refresh-2"); + } + + #[tokio::test] + #[serial] + async fn test_session_login_status_logout_lifecycle() { + suppress_browser(); + let dir = cache_tempdir(); + let idp = MockIdp::start().await; + + let session = OAuthSession::new(idp.config(dir.path())).unwrap(); + let status = session.status().await.unwrap(); + assert!(!status.refreshable); + + let status = session.login().await.unwrap(); + assert!(status.refreshable); + assert_eq!(status.issuer_url, idp.issuer_url); + assert_eq!(status.client_id, "client-id"); + assert_eq!(status.scopes, vec!["openid".to_string()]); + assert_eq!(status.flow, "device_code"); + assert!(status.obtained_at.is_some()); + + // An independent session manager sees the same cached login. + let other = OAuthSession::new(idp.config(dir.path())).unwrap(); + assert!(other.status().await.unwrap().refreshable); + + assert!(session.logout().await.unwrap().removed); + assert!(!session.logout().await.unwrap().removed); + assert!(!other.status().await.unwrap().refreshable); + assert_eq!(idp.device_authorizations.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + #[serial] + async fn test_login_without_refresh_token_clears_prior_record() { + suppress_browser(); + let dir = cache_tempdir(); + let idp = MockIdp::start().await; + + let session = OAuthSession::new(idp.config(dir.path())).unwrap(); + session.login().await.unwrap(); + assert!(session.status().await.unwrap().refreshable); + + // A provider that stops issuing refresh tokens (for example a login + // without offline_access) must not leave the earlier account behind. + idp.issue_refresh_tokens.store(false, Ordering::SeqCst); + let status = session.login().await.unwrap(); + assert!(!status.refreshable); + assert!(!session.status().await.unwrap().refreshable); + } + + #[tokio::test] + async fn test_client_credentials_with_cache_stays_memory_only() { + let dir = cache_tempdir(); + let idp = MockIdp::start().await; + + let mut config = idp.config(dir.path()); + config.flow = OAuthFlow::ClientCredentials; + config.client_secret = Some("secret".to_string()); + + let provider = OAuthHeaderProvider::new(config).unwrap(); + let headers = provider.get_headers().await.unwrap(); + assert_eq!(headers.get("authorization").unwrap(), "Bearer access-1"); + assert_eq!(dir.path().read_dir().unwrap().count(), 0); + } + + #[test] + fn test_managed_identity_with_cache_is_rejected() { + let dir = cache_tempdir(); + let mut config = device_config(dir.path()); + config.flow = OAuthFlow::AzureManagedIdentity { client_id: None }; + + let err = OAuthHeaderProvider::new(config).unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { message } if message.contains("AzureManagedIdentity")) + ); + } + + #[tokio::test] + #[serial] + async fn test_provider_debug_and_status_reveal_no_secrets() { + suppress_browser(); + let dir = cache_tempdir(); + let idp = MockIdp::start().await; + + let provider = OAuthHeaderProvider::new(idp.config(dir.path())).unwrap(); + let headers = provider.get_headers().await.unwrap(); + assert_eq!(headers.get("authorization").unwrap(), "Bearer access-1"); + let debug = format!("{provider:?}"); + assert!(!debug.contains("access-1")); + assert!(!debug.contains("refresh-1")); + + let session = OAuthSession::new(idp.config(dir.path())).unwrap(); + let status = session.status().await.unwrap(); + assert!(!format!("{status:?}").contains("refresh-")); + } + + #[test] + fn test_cache_key_canonicalizes_scopes_and_issuer() { + let mut config = device_config(Path::new("/tmp/cache")); + config.issuer_url = "https://issuer.example.com/".to_string(); + config.scopes = vec!["b".to_string(), " a ".to_string(), "a".to_string()]; + let key = CacheKey::new(&config).unwrap(); + assert_eq!(key.issuer_url, "https://issuer.example.com"); + assert_eq!(key.scopes, vec!["a".to_string(), "b".to_string()]); + + config.issuer_url = "https://issuer.example.com".to_string(); + let canonical = CacheKey::new(&config).unwrap(); + assert_eq!(canonical.file_stem, key.file_stem); + } + + #[test] + fn test_cache_key_separates_identity_dimensions() { + let base = device_config(Path::new("/tmp/cache")); + let base_key = CacheKey::new(&base).unwrap(); + + let mut other = base.clone(); + other.client_id = "other-client".to_string(); + assert_ne!(CacheKey::new(&other).unwrap().file_stem, base_key.file_stem); + + let mut other = base.clone(); + other.issuer_url = "https://other.example.com".to_string(); + assert_ne!(CacheKey::new(&other).unwrap().file_stem, base_key.file_stem); + + let mut other = base.clone(); + other.scopes = vec!["profile".to_string()]; + assert_ne!(CacheKey::new(&other).unwrap().file_stem, base_key.file_stem); + + let mut other = base.clone(); + other.flow = OAuthFlow::AuthorizationCode(Default::default()); + assert_ne!(CacheKey::new(&other).unwrap().file_stem, base_key.file_stem); + + let mut other = base.clone(); + other.client_secret = Some("secret".to_string()); + assert_ne!(CacheKey::new(&other).unwrap().file_stem, base_key.file_stem); + } + + #[test] + fn test_cache_key_contains_no_secret_material() { + let mut config = device_config(Path::new("/tmp/cache")); + config.client_secret = Some("super-secret-value".to_string()); + let key = CacheKey::new(&config).unwrap(); + assert!(!key.file_stem.contains("super-secret-value")); + assert_eq!(key.file_stem.len(), 64); + } + + #[test] + fn test_flow_key_rejects_non_persistent_flows() { + let mut config = device_config(Path::new("/tmp/cache")); + config.flow = OAuthFlow::ClientCredentials; + let err = CacheKey::new(&config).unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { message } if message.contains("not supported")) + ); + + let mut config = device_config(Path::new("/tmp/cache")); + config.flow = OAuthFlow::AzureManagedIdentity { client_id: None }; + assert!(CacheKey::new(&config).is_err()); + } + + #[test] + fn test_token_cache_options_defaults() { + let options = TokenCacheOptions::new(); + assert!(options.cache_dir.is_none()); + assert!(options.lock_timeout_secs.is_none()); + assert_eq!(options.lock_timeout(), Duration::from_secs(30)); + + let options = options.cache_dir("/tmp/x").lock_timeout_secs(5); + assert_eq!(options.cache_dir.as_deref(), Some(Path::new("/tmp/x"))); + assert_eq!(options.lock_timeout(), Duration::from_secs(5)); + } + + #[test] + fn test_token_cache_options_rejects_empty_dir() { + let options = TokenCacheOptions::new().cache_dir(""); + assert!(matches!( + options.resolved_dir(), + Err(Error::InvalidInput { message }) if message.contains("must not be empty") + )); + } + + #[tokio::test] + async fn test_session_lifecycle_without_cache_entry() { + let dir = cache_tempdir(); + let session = OAuthSession::new(device_config(dir.path())).unwrap(); + + let status = session.status().await.unwrap(); + assert!(!status.refreshable); + assert_eq!(status.issuer_url, "https://issuer.example.com"); + assert_eq!(status.client_id, "client-id"); + assert_eq!(status.scopes, vec!["openid".to_string()]); + assert_eq!(status.flow, "device_code"); + assert_eq!(status.obtained_at, None); + + let logout = session.logout().await.unwrap(); + assert!(!logout.removed); + } + + #[tokio::test] + async fn test_record_round_trip_and_redaction() { + let dir = cache_tempdir(); + let cache = TokenCache::new( + &device_config(dir.path()), + &TokenCacheOptions::new().cache_dir(dir.path()), + ) + .unwrap(); + let response = TokenResponse { + access_token: "access-token".to_string(), + refresh_token: Some("refresh-token".to_string()), + expires_in: Some(3600), + token_type: Some("Bearer".to_string()), + }; + let record = cache.record_from_response(&response).unwrap(); + let debug = format!("{record:?}"); + assert!(!debug.contains("refresh-token")); + assert!(debug.contains("")); + // No access-token material is persisted. + let json = serde_json::to_string(&record).unwrap(); + assert!(!json.contains("access-token")); + + cache.store(&record).await.unwrap(); + let loaded = cache.load().await.unwrap().unwrap(); + assert_eq!(loaded.refresh_token, "refresh-token"); + assert_eq!(loaded.version, CACHE_RECORD_VERSION); + + // The on-disk file must not be group/other readable and must not be a symlink target. + let path = cache.record_path(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(&path).unwrap().permissions().mode(); + assert_eq!(mode & 0o077, 0); + } + assert!(std::fs::symlink_metadata(&path).unwrap().is_file()); + + assert!(cache.delete().await.unwrap()); + assert!(cache.load().await.unwrap().is_none()); + assert!(!cache.delete().await.unwrap()); + } + + #[cfg(unix)] + #[tokio::test] + async fn test_record_rejects_symlink() { + let dir = cache_tempdir(); + let cache = TokenCache::new( + &device_config(dir.path()), + &TokenCacheOptions::new().cache_dir(dir.path()), + ) + .unwrap(); + let record = cache + .record_from_response(&TokenResponse { + access_token: "a".to_string(), + refresh_token: Some("r".to_string()), + expires_in: None, + token_type: None, + }) + .unwrap(); + cache.store(&record).await.unwrap(); + let path = cache.record_path(); + let target = dir.path().join("evil.json"); + std::fs::write(&target, "{}").unwrap(); + std::fs::remove_file(&path).unwrap(); + std::os::unix::fs::symlink(&target, &path).unwrap(); + + let err = cache.load().await.unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { message } if message.contains("not a regular file")) + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn test_record_rejects_world_readable_file() { + let dir = cache_tempdir(); + let cache = TokenCache::new( + &device_config(dir.path()), + &TokenCacheOptions::new().cache_dir(dir.path()), + ) + .unwrap(); + let path = cache.record_path(); + std::fs::write(&path, "{}").unwrap(); + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + + let err = cache.load().await.unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { message } if message.contains("group or other")) + ); + } + + /// Write a record file that passes the permission hardening checks. + fn write_record_file(path: &Path, contents: &str) { + std::fs::write(path, contents).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + } + + #[tokio::test] + async fn test_record_rejects_unknown_version_and_corruption() { + let dir = cache_tempdir(); + let cache = TokenCache::new( + &device_config(dir.path()), + &TokenCacheOptions::new().cache_dir(dir.path()), + ) + .unwrap(); + let path = cache.record_path(); + + // A complete, well-formed record with an unknown schema version. + let record = cache + .record_from_response(&TokenResponse { + access_token: "a".to_string(), + refresh_token: Some("r".to_string()), + expires_in: None, + token_type: None, + }) + .unwrap(); + let mut json = serde_json::to_value(&record).unwrap(); + json["version"] = serde_json::json!(99); + write_record_file(&path, &json.to_string()); + let err = cache.load().await.unwrap_err(); + assert!( + matches!(err, Error::Runtime { message } if message.contains("unsupported version")) + ); + + write_record_file(&path, r#"{"version":1,"issuer_url":"x""#); + let err = cache.load().await.unwrap_err(); + assert!(matches!(err, Error::Runtime { message } if message.contains("corrupt"))); + + write_record_file(&path, ""); + assert!( + cache + .load() + .await + .unwrap_err() + .to_string() + .contains("corrupt") + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn test_cache_dir_rejects_open_permissions() { + let dir = cache_tempdir(); + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o755)).unwrap(); + let err = TokenCache::new( + &device_config(dir.path()), + &TokenCacheOptions::new().cache_dir(dir.path()), + ) + .unwrap_err(); + assert!(matches!(err, Error::InvalidInput { message } if message.contains("chmod 700"))); + } + + #[tokio::test] + async fn test_lock_serializes_and_releases() { + let dir = cache_tempdir(); + let cache = TokenCache::new( + &device_config(dir.path()), + &TokenCacheOptions::new().cache_dir(dir.path()), + ) + .unwrap(); + let guard = cache.acquire_lock().await.unwrap(); + + let contender = { + let dir = dir.path().to_path_buf(); + let cache2 = TokenCache::new( + &device_config(&dir), + &TokenCacheOptions::new() + .cache_dir(&dir) + .lock_timeout_secs(1), + ) + .unwrap(); + tokio::time::timeout(Duration::from_millis(300), cache2.acquire_lock()).await + }; + assert!(contender.is_err(), "second acquire must block while held"); + drop(guard); + + let cache3 = TokenCache::new( + &device_config(dir.path()), + &TokenCacheOptions::new().cache_dir(dir.path()), + ) + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), cache3.acquire_lock()) + .await + .expect("lock re-acquirable after release") + .unwrap(); + } + + #[test] + fn test_token_cache_for_config_gating() { + let dir = cache_tempdir(); + assert!( + token_cache_for_config(&device_config(dir.path())) + .unwrap() + .is_some() + ); + + let mut config = device_config(dir.path()); + config.token_cache = None; + assert!(token_cache_for_config(&config).unwrap().is_none()); + + let mut config = device_config(dir.path()); + config.flow = OAuthFlow::ClientCredentials; + assert!(token_cache_for_config(&config).unwrap().is_none()); + + let mut config = device_config(dir.path()); + config.flow = OAuthFlow::AzureManagedIdentity { client_id: None }; + let err = token_cache_for_config(&config).unwrap_err(); + assert!( + matches!(err, Error::InvalidInput { message } if message.contains("AzureManagedIdentity")) + ); + } + + #[test] + fn test_oauth_session_requires_cache_options() { + let mut config = device_config(Path::new("/tmp/cache")); + config.token_cache = None; + let err = OAuthSession::new(config).unwrap_err(); + assert!(matches!(err, Error::InvalidInput { message } if message.contains("token_cache"))); + } + + #[tokio::test] + async fn test_store_skips_responses_without_refresh_token() { + let dir = cache_tempdir(); + let cache = TokenCache::new( + &device_config(dir.path()), + &TokenCacheOptions::new().cache_dir(dir.path()), + ) + .unwrap(); + let response = TokenResponse { + access_token: "access-token".to_string(), + refresh_token: None, + expires_in: Some(3600), + token_type: None, + }; + assert!(cache.record_from_response(&response).is_none()); + cache.store_if_refreshable(&response).await.unwrap(); + assert!(cache.load().await.unwrap().is_none()); + } + + #[test] + fn test_session_status_debug_has_no_secrets() { + let status = SessionStatus { + refreshable: true, + issuer_url: "https://issuer.example.com".to_string(), + client_id: "client-id".to_string(), + scopes: vec!["openid".to_string()], + flow: "device_code".to_string(), + obtained_at: Some(100), + }; + let debug = format!("{status:?}"); + assert!(!debug.contains("refresh-token")); + } +} From 2d4622491e924b753da4cf129a30bc0eab9c1507 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Tue, 15 Sep 2026 11:47:39 -0700 Subject: [PATCH 13/43] fix: keep computed-column freshness across branches and clones (#4188) A branch or shallow clone recomputed every computed column on its first refresh: the input signature carried each data file's raw base id, which the clone commit rewrites, and the signature sidecar was looked up under the branch's own tree, where nothing was ever written. A file is now identified by where its store resolves it, so the source's own root and a clone's registered reference to that root sign the same while different bases stay distinct; compaction products are matched the same way. Sidecars live in one `_computed/` at the table root shared by main and its branches; a shallow clone reads through the base it was cloned from and keeps a copy. Pruning collects references across every branch, and the staleness walk treats the versions a branch does not hold as unknown. Stamps written before this change no longer match, so the first refresh after upgrading recomputes once; a deep clone, which copies no sidecar, still starts over. --- rust/lancedb/src/table/freshness.rs | 414 +++++++++++++++++++++++----- rust/lancedb/src/table/refresh.rs | 71 ++++- 2 files changed, 413 insertions(+), 72 deletions(-) diff --git a/rust/lancedb/src/table/freshness.rs b/rust/lancedb/src/table/freshness.rs index e7a1841e6..b2c933cd8 100644 --- a/rust/lancedb/src/table/freshness.rs +++ b/rust/lancedb/src/table/freshness.rs @@ -17,16 +17,17 @@ //! only the reference. Sidecars no retained version references are removed //! by [`prune_sidecars`]. -use std::collections::{BTreeMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::ops::Range; use arrow_array::{Array, UInt64Array}; use futures::TryStreamExt; use lance::Dataset; +use lance::dataset::refs::Ref; use lance::dataset::transaction::Operation; use lance_core::ROW_ADDR; use lance_core::datatypes::{Field as LanceField, Schema as LanceSchema}; -use lance_io::object_store::ObjectStore; +use lance_io::object_store::{ObjectStore, uri_to_url}; use lance_table::format::{DataFile, Fragment}; use object_store::path::Path; use roaring::RoaringBitmap; @@ -108,36 +109,88 @@ pub fn fields_for_paths(schema: &LanceSchema, paths: &[String]) -> Result, String, i32)>, - overlays: Vec<(Option, String, i32, RoaringBitmap, u64)>, + files: Vec<(String, String, i32)>, + overlays: Vec<(String, String, i32, RoaringBitmap, u64)>, } -pub fn input_basis(metadata: &Fragment, ids: &[i32]) -> Result { +/// Where each storage base's data lives, as the store resolves it. A file +/// signs the same on the table that wrote it and on a branch or shallow +/// clone reading it through a base, while files under different bases +/// stay distinct. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Bases { + own: String, + registered: HashMap, +} + +impl Bases { + pub fn of(dataset: &Dataset) -> Result { + let registered = dataset + .manifest() + .base_paths + .values() + .map(|base| Ok((base.id, location(&base.path, base.is_dataset_root)?))) + .collect::>()?; + Ok(Self { + own: location(dataset.uri(), true)?, + registered, + }) + } + + fn location(&self, base_id: Option) -> Result<&str> { + match base_id { + None => Ok(&self.own), + Some(id) => self + .registered + .get(&id) + .map(String::as_str) + .ok_or_else(|| invalid(format!("base path id {id} not found"))), + } + } +} + +/// `uri` as its store resolves it, with the data directory under a dataset +/// root. The query part only selects a commit handler, so it is left out. +fn location(uri: &str, is_dataset_root: bool) -> Result { + let url = uri_to_url(uri)?; + let mut location = url[..url::Position::BeforeQuery] + .trim_end_matches('/') + .to_string(); + if is_dataset_root { + location.push_str("/data"); + } + Ok(location) +} + +pub fn input_basis(bases: &Bases, metadata: &Fragment, ids: &[i32]) -> Result { let column_of = |file: &DataFile| { file.fields .iter() .position(|id| ids.contains(id)) .map(|pos| (pos, file.column_indices.get(pos).copied().unwrap_or(-1))) }; - let files = metadata - .files - .iter() - .filter_map(|file| { - column_of(file).map(|(_, column)| (file.base_id, file.path.clone(), column)) - }) - .collect(); + let mut files = Vec::new(); + for file in &metadata.files { + if let Some((_, column)) = column_of(file) { + files.push(( + bases.location(file.base_id)?.to_string(), + file.path.clone(), + column, + )); + } + } let mut overlays = Vec::new(); for overlay in &metadata.overlays { let Some((pos, column)) = column_of(&overlay.data_file) else { continue; }; overlays.push(( - overlay.data_file.base_id, + bases.location(overlay.data_file.base_id)?.to_string(), overlay.data_file.path.clone(), column, overlay.coverage_for_field(pos)?.as_ref().clone(), @@ -152,10 +205,14 @@ pub fn input_basis(metadata: &Fragment, ids: &[i32]) -> Result { /// computed, and the rows that stay keep their values. Physical identity, /// not content, so a rewrite that preserves values still reads as a change; /// compaction is followed separately. -pub fn fragment_input_signature(fragment: &Fragment, inputs: &InputFields) -> Result { +pub fn fragment_input_signature( + bases: &Bases, + fragment: &Fragment, + inputs: &InputFields, +) -> Result { let mut parts = Vec::new(); for (path, ids) in inputs { - let basis = input_basis(fragment, ids)?; + let basis = input_basis(bases, fragment, ids)?; parts.push(format!("{}={basis:?}", path.join("."))); } Ok(short_hash(&parts.join("|"))) @@ -166,9 +223,10 @@ fn signature_of( fragment_id: u32, inputs: &InputFields, ) -> Result> { + let bases = Bases::of(dataset)?; dataset .get_fragment(fragment_id as usize) - .map(|fragment| fragment_input_signature(fragment.metadata(), inputs)) + .map(|fragment| fragment_input_signature(&bases, fragment.metadata(), inputs)) .transpose() } @@ -179,6 +237,7 @@ pub fn signatures_for( inputs: &InputFields, ) -> Result { let wanted: HashSet = fragment_ids.iter().copied().collect(); + let bases = Bases::of(dataset)?; dataset .get_fragments() .iter() @@ -186,7 +245,7 @@ pub fn signatures_for( .map(|fragment| { Ok(( fragment.id() as u32, - fragment_input_signature(fragment.metadata(), inputs)?, + fragment_input_signature(&bases, fragment.metadata(), inputs)?, )) }) .collect() @@ -222,19 +281,16 @@ const SIDECAR_FORMAT: u8 = 1; /// version history a cleanup keeps. const SIDECAR_UNVERIFIED_THRESHOLD_DAYS: i64 = 7; -/// The dataset's root directory: the parent of its versions directory. -/// Rebuilt from the raw parts, since re-encoding them would escape a -/// Windows drive letter's colon. -fn dataset_root(dataset: &Dataset) -> Path { - let versions = dataset.versions_dir(); - let count = versions.parts().count(); - Path::from_iter(versions.parts().take(count.saturating_sub(1))) +/// The table's root: one `_computed/` per table, shared by main and its +/// branches, so a branch reads the stamps it was taken with. +fn dataset_root(dataset: &Dataset) -> Result { + Ok(dataset.branch_location().find_main()?.path) } -fn sidecar_path(dataset: &Dataset, digest: &str) -> Path { - dataset_root(dataset) +fn sidecar_path(dataset: &Dataset, digest: &str) -> Result { + Ok(dataset_root(dataset)? .join(SIDECAR_DIR) - .join(format!("{digest}.sig")) + .join(format!("{digest}.sig"))) } /// `CSIG`, format byte, entry count, then one fragment id and 8-byte @@ -299,16 +355,31 @@ async fn write_sidecar(dataset: &Dataset, map: &SignatureMap) -> Result let digest = digest_of(&bytes); store(dataset) .await? - .put(&sidecar_path(dataset, &digest), &bytes) + .put(&sidecar_path(dataset, &digest)?, &bytes) .await?; Ok(format!("{SIDECAR_REF}{digest}")) } +/// A map is read from the table's own directory, then through the bases a +/// shallow clone reads its data from, where the source stamped it. A copy +/// found through a base is put in the table's own directory so it outlives +/// the source's pruning; a failed copy only costs the next lookup. async fn read_sidecar(dataset: &Dataset, digest: &str) -> Result { - let bytes = store(dataset) - .await? - .read_one_all(&sidecar_path(dataset, digest)) - .await?; + let own = sidecar_path(dataset, digest)?; + let store = store(dataset).await?; + let bytes = match store.read_one_all(&own).await { + Ok(bytes) => bytes, + Err(error) if error.is_not_found() => { + let bytes = read_sidecar_through_bases(dataset, digest).await?; + if let Err(error) = store.put(&own, &bytes).await { + log::warn!( + "computed column signature sidecar {digest} could not be copied to {own}: {error}" + ); + } + bytes + } + Err(error) => return Err(error.into()), + }; if digest_of(&bytes) != digest { return Err(invalid(format!( "signature sidecar {digest} does not match its digest" @@ -317,6 +388,38 @@ async fn read_sidecar(dataset: &Dataset, digest: &str) -> Result { decode_sidecar(&bytes) } +async fn read_sidecar_through_bases(dataset: &Dataset, digest: &str) -> Result { + let registry = dataset.session().store_registry(); + for base in dataset.manifest().base_paths.values() { + if !base.is_dataset_root { + continue; + } + let path = base.extract_path(registry.clone())?; + // A base cloned from a branch is named after it and sits at + // `/tree/`; the stamps are at that root. + let below_root = base + .name + .as_deref() + .map_or(0, |branch| 1 + branch.split('/').count()); + let count = path.parts().count(); + let root = Path::from_iter(path.parts().take(count.saturating_sub(below_root))); + let location = root.join(SIDECAR_DIR).join(format!("{digest}.sig")); + match dataset + .object_store(Some(base.id)) + .await? + .read_one_all(&location) + .await + { + Ok(bytes) => return Ok(bytes), + Err(error) if error.is_not_found() => continue, + Err(error) => return Err(error.into()), + } + } + Err(invalid(format!( + "signature sidecar {digest} is in neither the table's directory nor a base's" + ))) +} + /// Remove signature sidecars that no version still present references, /// the counterpart of lance's version cleanup for `_computed/`, with the /// same protection for objects still being published: a sidecar is put @@ -324,8 +427,17 @@ async fn read_sidecar(dataset: &Dataset, digest: &str) -> Result { /// [`SIDECAR_UNVERIFIED_THRESHOLD_DAYS`] is left alone unless /// `delete_unverified`. Returns how many were removed. pub async fn prune_sidecars(dataset: &Dataset, delete_unverified: bool) -> Result { + // The directory serves main and every branch, so the references are + // collected across all of them whichever handle prunes. + let main; + let dataset = if dataset.manifest().branch.is_some() { + main = dataset.checkout_version(Ref::Version(None, None)).await?; + &main + } else { + dataset + }; let store = store(dataset).await?; - let dir = dataset_root(dataset).join(SIDECAR_DIR); + let dir = dataset_root(dataset)?.join(SIDECAR_DIR); let unmodified_since = (!delete_unverified) .then(|| chrono::Utc::now() - chrono::Duration::days(SIDECAR_UNVERIFIED_THRESHOLD_DAYS)); let present: Vec = match store @@ -349,6 +461,22 @@ pub async fn prune_sidecars(dataset: &Dataset, delete_unverified: bool) -> Resul return Ok(0); } let mut referenced = HashSet::new(); + referenced_digests(dataset, &mut referenced).await?; + for branch in dataset.list_branches().await?.keys() { + referenced_digests(&dataset.checkout_branch(branch).await?, &mut referenced).await?; + } + let mut removed = 0; + for digest in present { + if !referenced.contains(&digest) { + store.delete(&sidecar_path(dataset, &digest)?).await?; + removed += 1; + } + } + Ok(removed) +} + +/// The sidecars every retained version of `dataset` references. +async fn referenced_digests(dataset: &Dataset, into: &mut HashSet) -> Result<()> { for version in dataset.versions().await? { let at = dataset.checkout_version(version.version).await?; for field in at.schema().fields_pre_order() { @@ -357,18 +485,11 @@ pub async fn prune_sidecars(dataset: &Dataset, delete_unverified: bool) -> Resul .get(SOURCE_SIGNATURE_META_KEY) .and_then(|value| value.strip_prefix(SIDECAR_REF)) { - referenced.insert(digest.to_string()); + into.insert(digest.to_string()); } } } - let mut removed = 0; - for digest in present { - if !referenced.contains(&digest) { - store.delete(&sidecar_path(dataset, &digest)).await?; - removed += 1; - } - } - Ok(removed) + Ok(()) } /// Read the column's stored map. An unreadable map is a state, not an error: @@ -427,8 +548,12 @@ enum Step { /// skipped: a row-moving update rewrites the rows it moves, so it does not /// carry their inputs unchanged. async fn step_at(dataset: &Dataset, version: u64) -> Result> { - let Some(transaction) = dataset.read_transaction_by_version(version).await? else { - return Ok(None); + let transaction = match dataset.read_transaction_by_version(version).await { + Ok(Some(transaction)) => transaction, + // A branch or clone holds no manifest before the version it was + // taken from; what happened there is unknown, and unknown is stale. + Ok(None) | Err(lance_core::Error::DatasetNotFound { .. }) => return Ok(None), + Err(error) => return Err(error.into()), }; let groups = match &transaction.operation { Operation::Append { .. } => return Ok(Some(Step::Append)), @@ -436,18 +561,16 @@ async fn step_at(dataset: &Dataset, version: u64) -> Result> { _ => return Ok(None), }; let at = dataset.checkout_version(version).await?; - let by_file: BTreeMap<(Option, String), u32> = at - .get_fragments() - .iter() - .flat_map(|fragment| { - let id = fragment.id() as u32; - fragment - .metadata() - .files - .iter() - .map(move |file| ((file.base_id, file.path.clone()), id)) - }) - .collect(); + let bases = Bases::of(&at)?; + let identity = |file: &DataFile| -> Result<(String, String)> { + Ok((bases.location(file.base_id)?.to_string(), file.path.clone())) + }; + let mut by_file = BTreeMap::new(); + for fragment in at.get_fragments() { + for file in &fragment.metadata().files { + by_file.insert(identity(file)?, fragment.id() as u32); + } + } let compactions = groups .iter() .map(|group| { @@ -456,11 +579,14 @@ async fn step_at(dataset: &Dataset, version: u64) -> Result> { .new_fragments .iter() .map(|fragment| { - fragment - .files - .iter() - .find_map(|file| by_file.get(&(file.base_id, file.path.clone())).copied()) - .ok_or_else(|| { + let mut found = None; + for file in &fragment.files { + if let Some(id) = by_file.get(&identity(file)?) { + found = Some(*id); + break; + } + } + found.ok_or_else(|| { invalid(format!( "a fragment added in version {version} is not in that version's manifest" )) @@ -711,10 +837,11 @@ pub async fn staleness_against( }; let mut dirty = HashSet::new(); let mut live = HashSet::new(); + let bases = Bases::of(dataset)?; for fragment in dataset.get_fragments() { let id = fragment.id() as u32; live.insert(id); - let current = fragment_input_signature(fragment.metadata(), inputs)?; + let current = fragment_input_signature(&bases, fragment.metadata(), inputs)?; if stored.get(&id).or_else(|| inherited.get(&id)) != Some(¤t) { dirty.insert(id); } @@ -1043,8 +1170,50 @@ mod tests { None, None, )); - let basis = input_basis(&fragment, word_count).unwrap(); - assert_eq!(basis.files, vec![(None, "packed.lance".to_string(), 3)]); + let basis = input_basis(&bases("memory://t"), &fragment, word_count).unwrap(); + assert_eq!( + basis.files, + vec![("memory://t/data".to_string(), "packed.lance".to_string(), 3)] + ); + } + + fn bases(root: &str) -> Bases { + Bases { + own: location(root, true).unwrap(), + registered: HashMap::new(), + } + } + + /// A file is identified by where its store resolves it: the table that + /// wrote it and a clone reading it through a base agree, however the + /// root was spelled, and a different root is a different file. + #[test] + fn a_file_signs_by_its_resolved_location() { + let file = |base_id| { + let mut fragment = Fragment::new(0); + fragment.files.push(DataFile::new( + "a.lance", + vec![7], + vec![0], + ConcreteFileVersion::V2_2, + None, + base_id, + )); + fragment + }; + let source = bases("/t/source/"); + let mut clone = bases("/t/clone"); + clone + .registered + .insert(1, location("/t/source", true).unwrap()); + let written = input_basis(&source, &file(None), &[7]).unwrap(); + assert_eq!(written, input_basis(&clone, &file(Some(1)), &[7]).unwrap()); + assert_ne!(written, input_basis(&clone, &file(None), &[7]).unwrap()); + assert_ne!( + written, + input_basis(&bases("/t/other"), &file(None), &[7]).unwrap() + ); + assert!(input_basis(&source, &file(Some(1)), &[7]).is_err()); } /// An overlay that stores the input in another physical column of the @@ -1065,7 +1234,7 @@ mod tests { coverage: OverlayCoverage::dense(RoaringBitmap::from_iter([0u32])), committed_version: 2, }); - input_basis(&fragment, &[7]).unwrap() + input_basis(&bases("memory://t"), &fragment, &[7]).unwrap() }; assert_ne!(overlay(0), overlay(1)); assert_eq!(overlay(0), overlay(0)); @@ -1075,7 +1244,7 @@ mod tests { let mut names = store(dataset) .await .unwrap() - .read_dir(dataset_root(dataset).join(SIDECAR_DIR)) + .read_dir(dataset_root(dataset).unwrap().join(SIDECAR_DIR)) .await .unwrap_or_default(); names.sort(); @@ -1143,7 +1312,7 @@ mod tests { .strip_prefix(SIDECAR_REF) .unwrap() .to_string(); - let path = sidecar_path(&dataset, &digest); + let path = sidecar_path(&dataset, &digest).unwrap(); let object_store = store(&dataset).await.unwrap(); object_store.put(&path, b"CSIG garbage").await.unwrap(); assert!(plan(&dataset).await.recompute_all); @@ -1198,7 +1367,7 @@ mod tests { store(&dataset) .await .unwrap() - .put(&sidecar_path(&dataset, "orphan"), b"CSIG") + .put(&sidecar_path(&dataset, "orphan").unwrap(), b"CSIG") .await .unwrap(); assert_eq!(prune_sidecars(&dataset, true).await.unwrap(), 1); @@ -1533,4 +1702,107 @@ mod tests { stamp_all(&mut dataset).await; assert_eq!(plan(&dataset).await, StalenessPlan::default()); } + async fn names_under(dataset: &Dataset, dir: Path) -> Vec { + let mut names = store(dataset) + .await + .unwrap() + .read_dir(dir.join(SIDECAR_DIR)) + .await + .unwrap_or_default(); + names.sort(); + names + } + + /// A branch starts with main's freshness, and its stamps join the + /// table's one sidecar directory rather than the branch's tree. + #[tokio::test] + async fn a_branch_starts_with_mains_freshness() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let mut branch = dataset + .create_branch("exp", dataset.version().version, None) + .await + .unwrap(); + assert_eq!(plan(&branch).await, StalenessPlan::default()); + rewrite_value(&mut branch, 3).await; + assert_eq!(plan(&branch).await.dirty, HashSet::from([0])); + stamp_all(&mut branch).await; + assert_eq!(plan(&branch).await, StalenessPlan::default()); + assert_eq!(sidecar_names(&dataset).await.len(), 2); + assert!( + names_under(&branch, branch.branch_location().path) + .await + .is_empty() + ); + assert_eq!(plan(&dataset).await, StalenessPlan::default()); + } + + /// A branch taken after an append main never stamped: the versions + /// before the branch are not in its tree, so the walk skips them and + /// the appended fragment is simply stale. + #[tokio::test] + async fn a_branch_walks_only_the_history_it_holds() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let appended = append_rows(&mut dataset, 100, 10).await; + let mut branch = dataset + .create_branch("exp", dataset.version().version, None) + .await + .unwrap(); + let staleness = plan(&branch).await; + assert_eq!(staleness.dirty, HashSet::from([appended])); + assert!(staleness.inherited.is_empty()); + let product = compact(&mut branch).await; + assert_eq!(plan(&branch).await.dirty, HashSet::from([product])); + } + + /// A shallow clone reads the map through the base it was cloned from + /// and keeps its own copy from then on. + #[tokio::test] + async fn a_shallow_clone_starts_with_its_sources_freshness() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().join("source").to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let target = dir.path().join("clone"); + let mut clone = dataset + .shallow_clone(target.to_str().unwrap(), dataset.version().version, None) + .await + .unwrap(); + assert_eq!(plan(&clone).await, StalenessPlan::default()); + assert_eq!(sidecar_names(&clone).await, sidecar_names(&dataset).await); + rewrite_value(&mut clone, 3).await; + assert_eq!(plan(&clone).await.dirty, HashSet::from([0])); + stamp_all(&mut clone).await; + assert_eq!(plan(&clone).await, StalenessPlan::default()); + assert_eq!(plan(&dataset).await, StalenessPlan::default()); + } + + /// A sidecar only a branch references survives a prune from either + /// handle until the branch is gone. + #[tokio::test] + async fn pruning_keeps_what_a_branch_references() { + let dir = tempfile::tempdir().unwrap(); + let mut dataset = table(dir.path().to_str().unwrap()).await; + stamp_all(&mut dataset).await; + let mains = signature_ref(&dataset, COLUMN); + let mut branch = dataset + .create_branch("exp", dataset.version().version, None) + .await + .unwrap(); + rewrite_value(&mut branch, 3).await; + stamp_all(&mut branch).await; + assert_ne!(signature_ref(&branch, COLUMN), mains); + assert_eq!(sidecar_names(&dataset).await.len(), 2); + assert_eq!(prune_sidecars(&dataset, true).await.unwrap(), 0); + assert_eq!(prune_sidecars(&branch, true).await.unwrap(), 0); + assert_eq!(plan(&branch).await, StalenessPlan::default()); + dataset.delete_branch("exp").await.unwrap(); + assert_eq!(prune_sidecars(&dataset, true).await.unwrap(), 1); + assert_eq!( + sidecar_names(&dataset).await, + vec![format!("{}.sig", mains.strip_prefix(SIDECAR_REF).unwrap())] + ); + } } diff --git a/rust/lancedb/src/table/refresh.rs b/rust/lancedb/src/table/refresh.rs index 996f2e383..0119ba36d 100644 --- a/rust/lancedb/src/table/refresh.rs +++ b/rust/lancedb/src/table/refresh.rs @@ -116,6 +116,7 @@ async fn execute_refresh_column_with_source( // since they were computed, or the definition did. Decided once, from // the manifest the values are read from. let inputs = freshness::fields_for_paths(dataset.schema(), &bound.inputs)?; + let bases = freshness::Bases::of(&dataset)?; let definition = freshness::definition_version(&expression); let staleness = freshness::staleness_against(&dataset, column, &definition, &inputs).await?; @@ -143,7 +144,7 @@ async fn execute_refresh_column_with_source( if whole { computed.insert( fragment_id, - freshness::fragment_input_signature(fragment.metadata(), &inputs)?, + freshness::fragment_input_signature(&bases, fragment.metadata(), &inputs)?, ); } let gained = Arc::new(AtomicU64::new(0)); @@ -880,6 +881,74 @@ mod tests { ); } + /// A branch inherits the freshness main recorded: its first refresh + /// fills nothing, and only its own appends after that. + #[tokio::test] + async fn test_a_branch_inherits_freshness() { + let dir = tempfile::tempdir().unwrap(); + let conn = connect(dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let batch = record_batch!(("x", Int32, vec![1, 2, 3])).unwrap(); + let table = conn.create_table("t", batch).execute().await.unwrap(); + declare_doubled(&table).await.unwrap(); + table.refresh_column("doubled").await.unwrap(); + let branch = table + .create_branch("exp", table.version().await.unwrap()) + .await + .unwrap(); + assert_eq!( + branch.refresh_column("doubled").await.unwrap().rows_filled, + 0 + ); + append(&branch, vec![4]).await; + assert_eq!( + branch.refresh_column("doubled").await.unwrap().rows_filled, + 1 + ); + assert_eq!( + table.refresh_column("doubled").await.unwrap().rows_filled, + 0 + ); + assert_eq!( + read(&branch, "doubled").await, + vec![Some(2), Some(4), Some(6), Some(8)] + ); + } + + /// A shallow clone inherits the freshness its source recorded. + #[tokio::test] + async fn test_a_shallow_clone_inherits_freshness() { + let dir = tempfile::tempdir().unwrap(); + let conn = connect(dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let batch = record_batch!(("x", Int32, vec![1, 2, 3])).unwrap(); + let table = conn.create_table("t", batch).execute().await.unwrap(); + declare_doubled(&table).await.unwrap(); + table.refresh_column("doubled").await.unwrap(); + let clone = conn + .clone_table("copy", dir.path().join("t.lance").to_str().unwrap()) + .execute() + .await + .unwrap(); + assert_eq!( + clone.refresh_column("doubled").await.unwrap().rows_filled, + 0 + ); + append(&clone, vec![4]).await; + assert_eq!( + clone.refresh_column("doubled").await.unwrap().rows_filled, + 1 + ); + assert_eq!( + table.refresh_column("doubled").await.unwrap().rows_filled, + 0 + ); + } + /// A row is filled only by gaining a value, so an expression yielding null /// settles at once instead of re-selecting the same rows forever: the /// second refresh finds the fragment signed and moves nothing. From f12996557f866e0ec8e4fc887e2f3298f3a169a7 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Tue, 15 Sep 2026 12:13:39 -0700 Subject: [PATCH 14/43] feat(remote): support materialized view APIs (#4180) ## Summary Align the experimental materialized-view HTTP transport with the equivalent Table API shape and add remote materialized-view support across Rust, Python, and TypeScript. This is an intentional breaking change to the experimental materialized-view surface. Materialized-view creation performs an initial refresh by default. The create endpoint returns `202 Accepted` with `{ "job_id": "..." }`; blocking SDK creation waits for that job before returning a populated view. `with_no_data` / `withNoData` explicitly creates only the definition and empty backing table. ## Route comparison | Operation | Materialized-view API | Equivalent Table API | | --- | --- | --- | | Create | `POST /v1/materialized_view/{id}/create` | `POST /v1/table/{id}/create` | | Describe/open | `POST /v1/materialized_view/{id}/describe` | `POST /v1/table/{id}/describe` | | List | `GET /v1/namespace/{id}/materialized_view/list` | `GET /v1/namespace/{id}/table/list` | | Refresh | `POST /v1/materialized_view/{id}/refresh` | asynchronous Table mutation pattern | | Drop | `POST /v1/materialized_view/{id}/drop` | `POST /v1/table/{id}/drop` | Create, describe, refresh, and drop identify the target in the singular item path instead of duplicating it in the request body. Create and drop require `202 Accepted` with a valid job ID. List is a namespace-scoped GET with opaque pagination tokens. The Rust list API now returns view names, matching Table listing and the existing Python and TypeScript APIs. ## Python API changes | Operation | Synchronous API | Asynchronous API | Table/job pattern | | --- | --- | --- | --- | | Create and wait | `DBConnection.create_materialized_view(...)` | `await AsyncConnection.create_materialized_view(...)` | Returns a materialized-view handle after its initial-population job finishes | | Submit create | `DBConnection.create_materialized_view_async(...) -> Job[None]` | `await AsyncConnection.create_materialized_view_async(...) -> AsyncJob[None]` | Matches job-returning Table mutations such as `create_index_async` | | Open | `DBConnection.open_materialized_view(...)` | `await AsyncConnection.open_materialized_view(...)` | Opens the backing Table plus its definition | | List | `DBConnection.list_materialized_views()` | `await AsyncConnection.list_materialized_views()` | Returns names like Table listing | | Refresh and wait | `MaterializedView.refresh(...)` | `await AsyncMaterializedView.refresh(...)` | Returns the typed refresh result after the job finishes | | Submit refresh | `MaterializedView.refresh_async(...) -> Job[RefreshMaterializedViewResult]` | `await AsyncMaterializedView.refresh_async(...) -> AsyncJob[RefreshMaterializedViewResult]` | Matches `Table.refresh_column_async`; remote job handles expose the server job ID | | Drop | `DBConnection.drop_materialized_view(...)` | `await AsyncConnection.drop_materialized_view(...)` | Matches blocking `drop_table` | | Submit drop | `DBConnection.drop_materialized_view_async(...) -> Job[None]` | `await AsyncConnection.drop_materialized_view_async(...) -> AsyncJob[None]` | Matches `drop_table_async`; remote handles expose the server cleanup job ID | The materialized-view handle exposes its backing Table through `.table`, so normal Table query, search, and index APIs apply. Definition lookup and refresh are backend-aware rather than depending on local schema metadata. TypeScript exposes the equivalent blocking/job drop pair as `dropMaterializedView` and `dropMaterializedViewAsync`. --- docs/src/js/classes/Connection.md | 60 ++- docs/src/js/classes/MaterializedView.md | 2 +- nodejs/__test__/materialized_view.test.ts | 26 +- nodejs/__test__/remote.test.ts | 18 +- nodejs/lancedb/connection.ts | 53 ++- nodejs/lancedb/materialized_view.ts | 22 +- nodejs/lancedb/table.ts | 9 +- nodejs/src/connection.rs | 34 +- nodejs/src/table.rs | 13 + python/python/lancedb/_lancedb.pyi | 22 + python/python/lancedb/db.py | 176 ++++++- python/python/lancedb/materialized_view.py | 56 ++- python/python/lancedb/namespace.py | 101 +++- python/python/lancedb/remote/db.py | 72 ++- .../python/tests/test_materialized_views.py | 264 +++++++++- python/src/connection.rs | 70 ++- python/src/error.rs | 1 + python/src/table.rs | 45 ++ rust/lancedb/src/database.rs | 71 ++- rust/lancedb/src/materialized_view.rs | 449 +++++++++++++----- rust/lancedb/src/materialized_view/refresh.rs | 20 + rust/lancedb/src/remote/db.rs | 221 +++++++++ rust/lancedb/src/remote/table.rs | 224 ++++++++- rust/lancedb/src/table.rs | 23 + 24 files changed, 1820 insertions(+), 232 deletions(-) diff --git a/docs/src/js/classes/Connection.md b/docs/src/js/classes/Connection.md index 18c1deb3b..d46d414d3 100644 --- a/docs/src/js/classes/Connection.md +++ b/docs/src/js/classes/Connection.md @@ -180,13 +180,13 @@ abstract createMaterializedView( Define a materialized view named `name` over the table `source`. -The view is created empty, with the query recorded in its schema -metadata; `view.refresh()` computes the rows. The view is a normal -table: it can be queried, indexed and searched, and it appears in -`tableNames`. The source table must have stable row ids (create it with +The view is populated before creation returns. Set `withNoData` to create +only its definition and empty backing table. The view is a normal table: +it can be queried, indexed and searched, and it appears in `tableNames`. +The source table must have stable row ids (create it with the `newTableEnableStableRowIds` storage option); they keep the view's provenance valid across source compactions and cannot be enabled after -a table exists. Local databases only. +a table exists. #### Parameters @@ -202,6 +202,8 @@ a table exists. Local databases only. * **options.where?**: `string` +* **options.withNoData?**: `boolean` + #### Returns `Promise`<[`MaterializedView`](MaterializedView.md)> @@ -373,6 +375,54 @@ Drop all tables in the database. *** +### dropMaterializedView() + +```ts +abstract dropMaterializedView(name, namespacePath?): Promise +``` + +Drop the materialized view named `name`. + +The view may become unavailable before physical cleanup finishes. Use +[dropMaterializedViewAsync](Connection.md#dropmaterializedviewasync) to retain and wait for the cleanup job. + +Rejects a table that exists but is not a materialized view. + +#### Parameters + +* **name**: `string` + +* **namespacePath?**: `string`[] + +#### Returns + +`Promise`<`void`> + +*** + +### dropMaterializedViewAsync() + +```ts +abstract dropMaterializedViewAsync(name, namespacePath?): Promise +``` + +Start dropping the materialized view named `name` and return its cleanup +job without waiting for completion. + +Rejects a table that exists but is not a materialized view. + +#### Parameters + +* **name**: `string` + +* **namespacePath?**: `string`[] + +#### Returns + +`Promise`<[`Job`](Job.md)> + +*** + ### dropNamespace() ```ts diff --git a/docs/src/js/classes/MaterializedView.md b/docs/src/js/classes/MaterializedView.md index e6ff66142..58eaa3c13 100644 --- a/docs/src/js/classes/MaterializedView.md +++ b/docs/src/js/classes/MaterializedView.md @@ -49,7 +49,7 @@ get name(): string definition(): Promise ``` -The query that defines the view, read from its stored schema. +The query that defines the view. #### Returns diff --git a/nodejs/__test__/materialized_view.test.ts b/nodejs/__test__/materialized_view.test.ts index b9d8ec911..7d0e5ebb3 100644 --- a/nodejs/__test__/materialized_view.test.ts +++ b/nodejs/__test__/materialized_view.test.ts @@ -76,11 +76,7 @@ describe("materialized views", () => { where: "age >= 18", }); expect(view.name).toBe("adults"); - expect(await view.table().countRows()).toBe(0); - - const result = await view.refresh(); - expect(result.mode).toBe("rebuild"); - expect(Number(result.rowsWritten)).toBe(2); + expect(await view.table().countRows()).toBe(2); const rows = await view.table().query().toArray(); expect(rows.map((r) => r.shout).sort()).toEqual(["ADA", "GRACE"]); @@ -102,7 +98,9 @@ describe("materialized views", () => { }); it("refreshes incrementally after an append", async () => { - const view = await db.createMaterializedView("copy", "people"); + const view = await db.createMaterializedView("copy", "people", { + withNoData: true, + }); await view.refresh(); const people = await db.openTable("people"); @@ -123,6 +121,21 @@ describe("materialized views", () => { await expect(db.openMaterializedView("people")).rejects.toThrow( "not a materialized view", ); + await expect(db.dropMaterializedView("people")).rejects.toThrow( + "not a materialized view", + ); + + await db.dropMaterializedView("adults"); + expect(await db.listMaterializedViews()).toEqual([]); + }); + + it("returns a job when dropping a view asynchronously", async () => { + await db.createMaterializedView("adults", "people"); + + const job = await db.dropMaterializedViewAsync("adults"); + expect(job.id).toBeNull(); + await job.wait(); + expect(await db.listMaterializedViews()).toEqual([]); }); it("rejects an invalid expression at create time", async () => { @@ -155,6 +168,7 @@ describe("materialized views", () => { }); const view = await db.createMaterializedView("quoted", "odd_names", { select: ["order item"], + withNoData: true, }); const result = await view.refresh(); expect(Number(result.rowsWritten)).toBe(1); diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index 519f0eb5f..5da0bf724 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -82,21 +82,17 @@ async function withMockDatabase( } describe("remote connection", () => { - it("refuses materialized views before issuing any request", async () => { - const paths: string[] = []; + it("lists materialized views through the namespace route", async () => { await withMockDatabase( (req, res) => { - paths.push(req.url ?? ""); - res.writeHead(404).end(); + expect(req.method).toBe("GET"); + expect(req.url).toBe("/v1/namespace/$/materialized_view/list"); + res + .writeHead(200, { "content-type": "application/json" }) + .end(JSON.stringify({ views: ["daily_sales"] })); }, async (db) => { - await expect(db.openMaterializedView("secret_table")).rejects.toThrow( - /only on local databases/, - ); - await expect(db.listMaterializedViews()).rejects.toThrow( - /only on local databases/, - ); - expect(paths).toEqual([]); + expect(await db.listMaterializedViews()).toEqual(["daily_sales"]); }, ); }); diff --git a/nodejs/lancedb/connection.ts b/nodejs/lancedb/connection.ts index 094819ec1..f5a6679cf 100644 --- a/nodejs/lancedb/connection.ts +++ b/nodejs/lancedb/connection.ts @@ -320,13 +320,13 @@ export abstract class Connection { /** * Define a materialized view named `name` over the table `source`. * - * The view is created empty, with the query recorded in its schema - * metadata; `view.refresh()` computes the rows. The view is a normal - * table: it can be queried, indexed and searched, and it appears in - * `tableNames`. The source table must have stable row ids (create it with + * The view is populated before creation returns. Set `withNoData` to create + * only its definition and empty backing table. The view is a normal table: + * it can be queried, indexed and searched, and it appears in `tableNames`. + * The source table must have stable row ids (create it with * the `newTableEnableStableRowIds` storage option); they keep the view's * provenance valid across source compactions and cannot be enabled after - * a table exists. Local databases only. + * a table exists. */ abstract createMaterializedView( name: string, @@ -335,6 +335,7 @@ export abstract class Connection { select?: MaterializedViewSelect; where?: string; limit?: number; + withNoData?: boolean; }, ): Promise; @@ -352,6 +353,30 @@ export abstract class Connection { */ abstract listMaterializedViews(): Promise; + /** + * Drop the materialized view named `name`. + * + * The view may become unavailable before physical cleanup finishes. Use + * {@link dropMaterializedViewAsync} to retain and wait for the cleanup job. + * + * Rejects a table that exists but is not a materialized view. + */ + abstract dropMaterializedView( + name: string, + namespacePath?: string[], + ): Promise; + + /** + * Start dropping the materialized view named `name` and return its cleanup + * job without waiting for completion. + * + * Rejects a table that exists but is not a materialized view. + */ + abstract dropMaterializedViewAsync( + name: string, + namespacePath?: string[], + ): Promise; + abstract openTable( name: string, namespacePath?: string[], @@ -631,6 +656,7 @@ export class LocalConnection extends Connection { select?: MaterializedViewSelect; where?: string; limit?: number; + withNoData?: boolean; }, ): Promise { validateNonNegativeInteger(options?.limit, "limit"); @@ -640,6 +666,7 @@ export class LocalConnection extends Connection { normalizeSelect(options?.select), options?.where, options?.limit, + options?.withNoData ?? false, ); return new MaterializedView(new LocalTable(innerTable)); } @@ -653,6 +680,22 @@ export class LocalConnection extends Connection { return await this.inner.listMaterializedViews(); } + async dropMaterializedView( + name: string, + namespacePath?: string[], + ): Promise { + return this.inner.dropMaterializedView(name, namespacePath ?? []); + } + + async dropMaterializedViewAsync( + name: string, + namespacePath?: string[], + ): Promise { + return new Job( + await this.inner.dropMaterializedViewAsync(name, namespacePath ?? []), + ); + } + async listTables( namespacePathOrOptions?: string[] | Partial, options?: Partial, diff --git a/nodejs/lancedb/materialized_view.ts b/nodejs/lancedb/materialized_view.ts index 1d47b640a..e729c960b 100644 --- a/nodejs/lancedb/materialized_view.ts +++ b/nodejs/lancedb/materialized_view.ts @@ -78,10 +78,22 @@ export function definitionFromMetadata( if (raw === undefined) { throw new Error(`Table '${name}' is not a materialized view`); } + return definitionFromJson(raw, name); +} + +/** @internal Parse the backend-independent definition returned by native code. */ +export function definitionFromJson( + raw: string, + name: string, +): MaterializedViewDefinition { // biome-ignore lint/suspicious/noExplicitAny: raw JSON const value: any = JSON.parse(raw); // "namespaced_select" keeps older readers from resolving the source at root. - if (value.kind !== "select" && value.kind !== "namespaced_select") { + if ( + value.kind !== undefined && + value.kind !== "select" && + value.kind !== "namespaced_select" + ) { throw new Error( `materialized view '${name}' is defined by '${value.kind}', which this ` + "version of lancedb cannot refresh", @@ -134,10 +146,12 @@ export class MaterializedView { return this.inner; } - /** The query that defines the view, read from its stored schema. */ + /** The query that defines the view. */ async definition(): Promise { - const schema = await this.inner.schema(); - return definitionFromMetadata(schema.metadata, this.name); + return definitionFromJson( + await this.inner.materializedViewDefinition(), + this.name, + ); } /** diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index a70243402..0280ac7f8 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -640,7 +640,7 @@ export abstract class Table { * Recompute this table's contents from its materialized-view definition. * * Plumbing for {@link MaterializedView.refresh}, which is the way to call - * it: rejects tables that carry no view definition. Local tables only. + * it: rejects tables that carry no view definition. * @ignore */ abstract refreshMaterializedView( @@ -648,6 +648,9 @@ export abstract class Table { sourceVersion?: number, ): Promise; + /** @ignore */ + abstract materializedViewDefinition(): Promise; + /** * Alter the name or nullability of columns. * @param {ColumnAlteration[]} columnAlterations One or more alterations to @@ -1367,6 +1370,10 @@ export class LocalTable extends Table { return await this.inner.refreshMaterializedView(full, sourceVersion); } + async materializedViewDefinition(): Promise { + return await this.inner.materializedViewDefinition(); + } + async alterColumns( columnAlterations: ColumnAlteration[], ): Promise { diff --git a/nodejs/src/connection.rs b/nodejs/src/connection.rs index 586238359..0c07ffd50 100644 --- a/nodejs/src/connection.rs +++ b/nodejs/src/connection.rs @@ -308,6 +308,7 @@ impl Connection { projections: Option>>, filter: Option, limit: Option, + with_no_data: bool, ) -> napi::Result
{ let mut builder = self.get_inner()?.create_materialized_view(name, source); if let Some(projections) = projections { @@ -328,6 +329,7 @@ impl Connection { .map_err(|_| napi::Error::from_reason("limit must be a non-negative integer"))?; builder = builder.limit(limit); } + builder = builder.with_no_data(with_no_data); let view = builder.execute().await.default_error()?; Ok(Table::new(view.table().clone())) } @@ -349,7 +351,37 @@ impl Connection { .list_materialized_views() .await .default_error()?; - Ok(views.into_iter().map(|v| v.name).collect()) + Ok(views) + } + + /// Drop a materialized view. + #[napi(catch_unwind)] + pub async fn drop_materialized_view( + &self, + name: String, + namespace_path: Option>, + ) -> napi::Result<()> { + let ns = namespace_path.unwrap_or_default(); + self.get_inner()? + .drop_materialized_view(&name, &ns) + .await + .default_error() + } + + /// Start dropping a materialized view and return its cleanup job. + #[napi(catch_unwind)] + pub async fn drop_materialized_view_async( + &self, + name: String, + namespace_path: Option>, + ) -> napi::Result { + let ns = namespace_path.unwrap_or_default(); + let job = self + .get_inner()? + .drop_materialized_view_async(&name, &ns) + .await + .default_error()?; + Ok(crate::job::Job::new(job)) } #[napi(catch_unwind)] diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index cf7ec4020..344017d08 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -447,6 +447,19 @@ impl Table { Ok(result.into()) } + #[napi(catch_unwind)] + pub async fn materialized_view_definition(&self) -> napi::Result { + let inner = self.inner_ref()?.clone(); + let view = lancedb::MaterializedView::from_table(inner) + .await + .default_error()?; + serde_json::to_string(view.definition()).map_err(|err| { + napi::Error::from_reason(format!( + "failed to serialize materialized-view definition: {err}" + )) + }) + } + #[napi(catch_unwind)] pub async fn add_columns_with_schema( &self, diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 08d81bfd3..32159dd5a 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -211,8 +211,24 @@ class Connection(object): projections: Optional[List[Tuple[str, str]]] = None, filter: Optional[str] = None, limit: Optional[int] = None, + with_no_data: bool = False, ) -> Table: ... + async def create_materialized_view_async( + self, + name: str, + source: str, + projections: Optional[List[Tuple[str, str]]] = None, + filter: Optional[str] = None, + limit: Optional[int] = None, + with_no_data: bool = False, + ) -> Job: ... async def list_materialized_views(self) -> List[str]: ... + async def drop_materialized_view( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> None: ... + async def drop_materialized_view_async( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> Job: ... async def drop_table( self, name: str, namespace_path: Optional[List[str]] = None ) -> None: ... @@ -429,6 +445,10 @@ class Table: async def refresh_materialized_view( self, full: bool = False, source_version: Optional[int] = None ) -> RefreshMaterializedViewResult: ... + async def refresh_materialized_view_async( + self, full: bool = False, source_version: Optional[int] = None + ) -> Job: ... + async def materialized_view_definition(self) -> str: ... async def add_columns_with_schema(self, schema: pa.Schema) -> AddColumnsResult: ... async def alter_columns( self, columns: list[dict[str, Any]] @@ -782,6 +802,8 @@ class RefreshColumnResult: version: int class RefreshMaterializedViewResult: + @staticmethod + def from_json(value: str) -> RefreshMaterializedViewResult: ... mode: str rows_written: int source_version: int diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index 88718e968..d09db80c3 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -529,13 +529,14 @@ class DBConnection(EnforceOverrides): select: SelectArg = None, where: Optional[str] = None, limit: Optional[int] = None, + with_no_data: bool = False, ) -> MaterializedView: """Define a materialized view named ``name`` over the table ``source``. - The view is created empty, with the query recorded in its schema - metadata; ``view.refresh()`` computes the rows. The view is a normal - table: it can be queried, indexed and searched, and it appears in - ``table_names``. Local databases only. + The view is populated before creation returns. Pass + ``with_no_data=True`` to create only its definition and empty backing + table. The view is a normal table: it can be queried, indexed and + searched, and it appears in ``table_names``. The source table must have stable row ids (create it with the ``new_table_enable_stable_row_ids`` storage option): they keep the @@ -556,6 +557,8 @@ class DBConnection(EnforceOverrides): SQL predicate; only matching source rows appear in the view. limit: int, optional Cap the view at this many rows, in materialization order. + with_no_data: bool, default False + Skip the initial refresh and leave the backing table empty. Returns ------- @@ -565,6 +568,27 @@ class DBConnection(EnforceOverrides): "materialized views are not supported on this connection type" ) + def create_materialized_view_async( + self, + name: str, + source: str, + *, + select: SelectArg = None, + where: Optional[str] = None, + limit: Optional[int] = None, + with_no_data: bool = False, + ) -> Job[None]: + """Submit materialized-view creation and return its job. + + The job may already be complete for a local database. On LanceDB + Cloud and Enterprise, its ``id`` is the server job identifier from + the ``202 Accepted`` create response. Wait for the job before opening + or querying the view. + """ + raise NotImplementedError( + "materialized views are not supported on this connection type" + ) + def open_materialized_view(self, name: str) -> MaterializedView: """Open the materialized view named ``name``. @@ -585,6 +609,32 @@ class DBConnection(EnforceOverrides): "materialized views are not supported on this connection type" ) + def drop_materialized_view( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> None: + """Drop a materialized view. + + The view may become unavailable before physical cleanup finishes. Use + :meth:`drop_materialized_view_async` to retain and wait for the cleanup + job. + """ + raise NotImplementedError( + "materialized views are not supported on this connection type" + ) + + def drop_materialized_view_async( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> Job[None]: + """Start dropping a materialized view and return its cleanup job. + + The job may already be complete for a local database. On LanceDB Cloud + and Enterprise, its ``id`` is the server job identifier from the + ``202 Accepted`` drop response. + """ + raise NotImplementedError( + "materialized views are not supported on this connection type" + ) + def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): """Drop a table from the database. @@ -1270,6 +1320,7 @@ class LanceDBConnection(DBConnection): select: SelectArg = None, where: Optional[str] = None, limit: Optional[int] = None, + with_no_data: bool = False, ) -> MaterializedView: """Define a materialized view named ``name`` over the table ``source``. See @@ -1290,17 +1341,44 @@ class LanceDBConnection(DBConnection): ... select=["name", ("shout", "upper(name)")], ... where="age >= 18", ... ) - >>> result = view.refresh() - >>> result.rows_written + >>> view.table.count_rows() 1 """ LOOP.run( self._conn.create_materialized_view( - name, source, select=select, where=where, limit=limit + name, + source, + select=select, + where=where, + limit=limit, + with_no_data=with_no_data, ) ) return MaterializedView(self.open_table(name)) + @override + def create_materialized_view_async( + self, + name: str, + source: str, + *, + select: SelectArg = None, + where: Optional[str] = None, + limit: Optional[int] = None, + with_no_data: bool = False, + ) -> Job[None]: + job = LOOP.run( + self._conn.create_materialized_view_async( + name, + source, + select=select, + where=where, + limit=limit, + with_no_data=with_no_data, + ) + ) + return Job(job) + @override def open_materialized_view(self, name: str) -> MaterializedView: """Open the materialized view named ``name``.""" @@ -1313,6 +1391,25 @@ class LanceDBConnection(DBConnection): """The names of the materialized views in this database.""" return LOOP.run(self._conn.list_materialized_views()) + @override + def drop_materialized_view( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> None: + if namespace_path is None: + namespace_path = [] + LOOP.run(self._conn.drop_materialized_view(name, namespace_path=namespace_path)) + + @override + def drop_materialized_view_async( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> Job[None]: + if namespace_path is None: + namespace_path = [] + job = LOOP.run( + self._conn.drop_materialized_view_async(name, namespace_path=namespace_path) + ) + return Job(job) + def clone_table( self, target_table_name: str, @@ -2080,6 +2177,7 @@ class AsyncConnection(object): select: SelectArg = None, where: Optional[str] = None, limit: Optional[int] = None, + with_no_data: bool = False, ) -> AsyncMaterializedView: """Define a materialized view named ``name`` over the table ``source``. See @@ -2091,19 +2189,40 @@ class AsyncConnection(object): projections=normalize_select(select), filter=where, limit=limit, + with_no_data=with_no_data, ) return AsyncMaterializedView(AsyncTable(inner)) + async def create_materialized_view_async( + self, + name: str, + source: str, + *, + select: SelectArg = None, + where: Optional[str] = None, + limit: Optional[int] = None, + with_no_data: bool = False, + ) -> AsyncJob[None]: + """Submit materialized-view creation and return its job. + + Wait for the returned job before opening or querying the view. + """ + inner = await self._inner.create_materialized_view_async( + name, + source, + projections=normalize_select(select), + filter=where, + limit=limit, + with_no_data=with_no_data, + ) + return AsyncJob(inner) + async def open_materialized_view(self, name: str) -> AsyncMaterializedView: """Open the materialized view named ``name``. Raises ``ValueError`` if the table exists but is not a materialized view. """ - if self.uri.startswith("db://"): - raise NotImplementedError( - "materialized views are supported only on local databases" - ) view = AsyncMaterializedView(await self.open_table(name)) await view.definition() return view @@ -2116,6 +2235,41 @@ class AsyncConnection(object): """ return await self._inner.list_materialized_views() + async def drop_materialized_view( + self, + name: str, + *, + namespace_path: Optional[List[str]] = None, + ) -> None: + """Drop a materialized view. + + The view may become unavailable before physical cleanup finishes. Use + :meth:`drop_materialized_view_async` to retain and wait for the cleanup + job. + """ + if namespace_path is None: + namespace_path = [] + await self._inner.drop_materialized_view(name, namespace_path=namespace_path) + + async def drop_materialized_view_async( + self, + name: str, + *, + namespace_path: Optional[List[str]] = None, + ) -> AsyncJob[None]: + """Start dropping a materialized view and return its cleanup job. + + Await :meth:`AsyncJob.wait` before assuming physical cleanup has + finished. + """ + if namespace_path is None: + namespace_path = [] + return AsyncJob( + await self._inner.drop_materialized_view_async( + name, namespace_path=namespace_path + ) + ) + async def clone_table( self, target_table_name: str, diff --git a/python/python/lancedb/materialized_view.py b/python/python/lancedb/materialized_view.py index c52a2d7a9..384683c9f 100644 --- a/python/python/lancedb/materialized_view.py +++ b/python/python/lancedb/materialized_view.py @@ -11,6 +11,7 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Tuple, Union from .background_loop import LOOP +from .job import AsyncJob, Job, _typed_job if TYPE_CHECKING: import pyarrow as pa @@ -73,6 +74,20 @@ def _definition_from_schema( ) +def _definition_from_json(raw: str) -> MaterializedViewDefinition: + value = json.loads(raw) + return MaterializedViewDefinition( + source_table=value["source_table"], + projections=[ + (p["output"], p["expression"]) for p in value.get("projections", []) + ], + filter=value.get("filter"), + limit=value.get("limit"), + inputs=value.get("inputs", []), + source_namespace=value.get("source_namespace", []), + ) + + def _quote_identifier(name: str) -> str: """Quote a column name as a Lance SQL identifier (backticks).""" escaped = name.replace("`", "``") @@ -126,8 +141,9 @@ class AsyncMaterializedView: return self._table async def definition(self) -> MaterializedViewDefinition: - """The query that defines the view, read from its stored schema.""" - return _definition_from_schema(await self._table.schema(), self.name) + """The query that defines the view.""" + raw = await self._table._inner.materialized_view_definition() + return _definition_from_json(raw) async def refresh( self, *, full: bool = False, source_version: Optional[int] = None @@ -148,6 +164,24 @@ class AsyncMaterializedView: full=full, source_version=source_version ) + async def refresh_async( + self, *, full: bool = False, source_version: Optional[int] = None + ) -> "AsyncJob[RefreshMaterializedViewResult]": + """Submit a refresh and return its job without waiting. + + The job may already be complete for a local view. On LanceDB Cloud + and Enterprise, its ``id`` is the server job identifier returned by + the refresh endpoint. + """ + from ._lancedb import RefreshMaterializedViewResult + + return _typed_job( + await self._table._inner.refresh_materialized_view_async( + full=full, source_version=source_version + ), + RefreshMaterializedViewResult.from_json, + ) + class MaterializedView: """Synchronous variant of @@ -171,8 +205,8 @@ class MaterializedView: @property def definition(self) -> MaterializedViewDefinition: - """The query that defines the view, read from its stored schema.""" - return _definition_from_schema(self._table.schema, self.name) + """The query that defines the view.""" + return LOOP.run(self._async.definition()) def refresh( self, *, full: bool = False, source_version: Optional[int] = None @@ -180,3 +214,17 @@ class MaterializedView: """Recompute the view from its source. See [AsyncMaterializedView.refresh][lancedb.materialized_view.AsyncMaterializedView.refresh].""" return LOOP.run(self._async.refresh(full=full, source_version=source_version)) + + def refresh_async( + self, *, full: bool = False, source_version: Optional[int] = None + ) -> "Job[RefreshMaterializedViewResult]": + """Submit a refresh and return its job without waiting. + + See + [AsyncMaterializedView.refresh_async][lancedb.materialized_view.AsyncMaterializedView.refresh_async]. + """ + return Job( + LOOP.run( + self._async.refresh_async(full=full, source_version=source_version) + ) + ) diff --git a/python/python/lancedb/namespace.py b/python/python/lancedb/namespace.py index 61d2122f3..fe38a47c7 100644 --- a/python/python/lancedb/namespace.py +++ b/python/python/lancedb/namespace.py @@ -637,6 +637,7 @@ class LanceNamespaceDBConnection(DBConnection): select: "SelectArg" = None, where: Optional[str] = None, limit: Optional[int] = None, + with_no_data: bool = False, ) -> "MaterializedView": """Define a materialized view over a table in the root namespace. See @@ -646,12 +647,40 @@ class LanceNamespaceDBConnection(DBConnection): self.open_table( LOOP.run( self._inner.create_materialized_view( - name, source, select=select, where=where, limit=limit + name, + source, + select=select, + where=where, + limit=limit, + with_no_data=with_no_data, ) ).name ) ) + @override + def create_materialized_view_async( + self, + name: str, + source: str, + *, + select: "SelectArg" = None, + where: Optional[str] = None, + limit: Optional[int] = None, + with_no_data: bool = False, + ) -> Job[None]: + job = LOOP.run( + self._inner.create_materialized_view_async( + name, + source, + select=select, + where=where, + limit=limit, + with_no_data=with_no_data, + ) + ) + return Job(job) + @override def open_materialized_view(self, name: str) -> "MaterializedView": """Open the materialized view named ``name``.""" @@ -664,6 +693,30 @@ class LanceNamespaceDBConnection(DBConnection): """The names of the materialized views in the root namespace.""" return LOOP.run(self._inner.list_materialized_views()) + @override + def drop_materialized_view( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> None: + if namespace_path is None: + namespace_path = [] + LOOP.run( + self._inner.drop_materialized_view(name, namespace_path=namespace_path) + ) + + @override + def drop_materialized_view_async( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> Job[None]: + if namespace_path is None: + namespace_path = [] + return Job( + LOOP.run( + self._inner.drop_materialized_view_async( + name, namespace_path=namespace_path + ) + ) + ) + @override def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): if namespace_path is None: @@ -1194,15 +1247,41 @@ class AsyncLanceNamespaceDBConnection: select: "SelectArg" = None, where: Optional[str] = None, limit: Optional[int] = None, + with_no_data: bool = False, ) -> "AsyncMaterializedView": """Define a materialized view over a table in the root namespace.""" view = await self._inner.create_materialized_view( - name, source, select=select, where=where, limit=limit + name, + source, + select=select, + where=where, + limit=limit, + with_no_data=with_no_data, ) # Reopen through the namespace so the view's table carries the # namespace client and pushdown configuration a bare inner table lacks. return AsyncMaterializedView(await self.open_table(view.name)) + async def create_materialized_view_async( + self, + name: str, + source: str, + *, + select: "SelectArg" = None, + where: Optional[str] = None, + limit: Optional[int] = None, + with_no_data: bool = False, + ) -> AsyncJob[None]: + """Submit materialized-view creation and return its job.""" + return await self._inner.create_materialized_view_async( + name, + source, + select=select, + where=where, + limit=limit, + with_no_data=with_no_data, + ) + async def open_materialized_view(self, name: str) -> "AsyncMaterializedView": """Open the materialized view named ``name``.""" view = AsyncMaterializedView(await self.open_table(name)) @@ -1213,6 +1292,24 @@ class AsyncLanceNamespaceDBConnection: """The names of the materialized views in the root namespace.""" return await self._inner.list_materialized_views() + async def drop_materialized_view( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> None: + """Drop a materialized view from the namespace.""" + if namespace_path is None: + namespace_path = [] + await self._inner.drop_materialized_view(name, namespace_path=namespace_path) + + async def drop_materialized_view_async( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> AsyncJob[None]: + """Start dropping a materialized view and return its cleanup job.""" + if namespace_path is None: + namespace_path = [] + return await self._inner.drop_materialized_view_async( + name, namespace_path=namespace_path + ) + async def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): """Drop a table from the namespace.""" if namespace_path is None: diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index 5b6828b04..fb4e30fdf 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -665,22 +665,80 @@ class RemoteDBConnection(DBConnection): select: SelectArg = None, where: Optional[str] = None, limit: Optional[int] = None, + with_no_data: bool = False, ) -> MaterializedView: - raise NotImplementedError( - "materialized views are supported only on local databases" + from .table import RemoteTable + + view = LOOP.run( + self._conn.create_materialized_view( + name, + source, + select=select, + where=where, + limit=limit, + with_no_data=with_no_data, + ) ) + return MaterializedView( + RemoteTable( + view.table, + self.db_name, + connection_state=self.serialize, + namespace_path=[], + ) + ) + + @override + def create_materialized_view_async( + self, + name: str, + source: str, + *, + select: SelectArg = None, + where: Optional[str] = None, + limit: Optional[int] = None, + with_no_data: bool = False, + ) -> Job[None]: + job = LOOP.run( + self._conn.create_materialized_view_async( + name, + source, + select=select, + where=where, + limit=limit, + with_no_data=with_no_data, + ) + ) + return Job(job) @override def open_materialized_view(self, name: str) -> MaterializedView: - raise NotImplementedError( - "materialized views are supported only on local databases" - ) + view = MaterializedView(self.open_table(name)) + view.definition + return view @override def list_materialized_views(self) -> List[str]: - raise NotImplementedError( - "materialized views are supported only on local databases" + return LOOP.run(self._conn.list_materialized_views()) + + @override + def drop_materialized_view( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> None: + if namespace_path is None: + namespace_path = [] + LOOP.run(self._conn.drop_materialized_view(name, namespace_path=namespace_path)) + + @override + def drop_materialized_view_async( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> Job[None]: + if namespace_path is None: + namespace_path = [] + job = LOOP.run( + self._conn.drop_materialized_view_async(name, namespace_path=namespace_path) ) + return Job(job) @override def drop_table(self, name: str, namespace_path: Optional[List[str]] = None): diff --git a/python/python/tests/test_materialized_views.py b/python/python/tests/test_materialized_views.py index 5cd7b23d6..695ef142a 100644 --- a/python/python/tests/test_materialized_views.py +++ b/python/python/tests/test_materialized_views.py @@ -1,9 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The LanceDB Authors +import contextlib +import http.server +import json +import threading + import lancedb import pytest from lancedb.materialized_view import MaterializedViewDefinition +from lancedb.remote.db import RemoteDBConnection STABLE_ROW_IDS = {"new_table_enable_stable_row_ids": "true"} @@ -22,6 +28,152 @@ def make_db(tmp_path): return db +@contextlib.contextmanager +def mock_remote_materialized_views(): + requests = [] + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_GET(self): + requests.append(self.path) + encoded = json.dumps({"views": ["daily_sales"]}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + with http.server.HTTPServer(("localhost", 0), Handler) as server: + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + yield f"http://localhost:{server.server_address[1]}", requests + finally: + server.shutdown() + thread.join() + + +@contextlib.contextmanager +def mock_remote_materialized_view_create(): + requests = [] + + class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + length = int(self.headers.get("Content-Length", 0)) + body = json.loads(self.rfile.read(length) or b"{}") + requests.append((self.path, body)) + job_id = "mv-drop-123" if self.path.endswith("/drop") else "mv-create-123" + encoded = json.dumps({"job_id": job_id}).encode() + self.send_response(202) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + with http.server.HTTPServer(("localhost", 0), Handler) as server: + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + yield f"http://localhost:{server.server_address[1]}", requests + finally: + server.shutdown() + thread.join() + + +def test_remote_list_uses_namespace_route(): + with mock_remote_materialized_views() as (host, requests): + db = lancedb.connect( + "db://dev", + api_key="fake", + host_override=host, + client_config={"retry_config": {"retries": 0}}, + ) + assert db.list_materialized_views() == ["daily_sales"] + assert requests == ["/v1/namespace/$/materialized_view/list"] + + +def test_remote_create_async_returns_server_job(): + with mock_remote_materialized_view_create() as (host, requests): + db = lancedb.connect( + "db://dev", + api_key="fake", + host_override=host, + client_config={"retry_config": {"retries": 0}}, + ) + job = db.create_materialized_view_async("adults", "people", where="age >= 18") + assert job.id == "mv-create-123" + assert requests == [ + ( + "/v1/materialized_view/adults/create", + {"query": 'SELECT * FROM "people" WHERE age >= 18', "with_no_data": False}, + ) + ] + + +def test_remote_drop_async_returns_server_job(): + with mock_remote_materialized_view_create() as (host, requests): + db = lancedb.connect( + "db://dev", + api_key="fake", + host_override=host, + client_config={"retry_config": {"retries": 0}}, + ) + job = db.drop_materialized_view_async("adults") + assert job.id == "mv-drop-123" + assert requests == [("/v1/materialized_view/adults/drop", {})] + + +def test_sync_remote_create_uses_public_async_connection(): + calls = [] + + class StubAsyncTable: + name = "adults" + + class StubAsyncMaterializedView: + table = StubAsyncTable() + + class StrictAsyncConnection: + async def create_materialized_view( + self, + name, + source, + *, + select=None, + where=None, + limit=None, + with_no_data=False, + ): + calls.append((name, source, select, where, limit, with_no_data)) + return StubAsyncMaterializedView() + + async def drop_materialized_view(self, name, *, namespace_path=None): + calls.append(("drop", name, namespace_path)) + + db = RemoteDBConnection.__new__(RemoteDBConnection) + db._conn = StrictAsyncConnection() + db.db_name = "example" + db.serialize = lambda: "{}" + + view = db.create_materialized_view( + "adults", + "people", + select=["name"], + where="age >= 18", + limit=10, + with_no_data=True, + ) + assert view.name == "adults" + assert calls == [("adults", "people", ["name"], "age >= 18", 10, True)] + + db.drop_materialized_view("adults", namespace_path=["analytics"]) + assert calls[-1] == ("drop", "adults", ["analytics"]) + + def test_create_refresh_and_query(tmp_path): db = make_db(tmp_path) view = db.create_materialized_view( @@ -31,16 +183,34 @@ def test_create_refresh_and_query(tmp_path): where="age >= 18", ) assert view.name == "adults" - assert view.table.count_rows() == 0 - - result = view.refresh() - assert result.mode == "rebuild" - assert result.rows_written == 2 + assert view.table.count_rows() == 2 rows = view.table.search().to_list() assert sorted(row["shout"] for row in rows) == ["ADA", "GRACE"] +def test_create_and_refresh_jobs(tmp_path): + db = make_db(tmp_path) + create_job = db.create_materialized_view_async( + "adults", "people", where="age >= 18", with_no_data=True + ) + assert create_job.id is None + assert create_job.wait() is None + + view = db.open_materialized_view("adults") + refresh_job = view.refresh_async() + assert refresh_job.id is None + result = refresh_job.wait() + assert result.mode == "rebuild" + assert result.rows_written == 2 + assert view.table.count_rows() == 2 + + drop_job = db.drop_materialized_view_async("adults") + assert drop_job.id is None + assert drop_job.wait() is None + assert "adults" not in db.list_materialized_views() + + def test_definition_round_trips(tmp_path): db = make_db(tmp_path) db.create_materialized_view("adults", "people", where="age >= 18") @@ -56,7 +226,7 @@ def test_definition_round_trips(tmp_path): def test_incremental_refresh_after_append(tmp_path): db = make_db(tmp_path) - view = db.create_materialized_view("copy", "people") + view = db.create_materialized_view("copy", "people", with_no_data=True) view.refresh() db.open_table("people").add([{"name": "alan", "age": 41}]) @@ -70,7 +240,7 @@ def test_incremental_refresh_after_append(tmp_path): def test_incremental_refresh_after_update(tmp_path): db = make_db(tmp_path) - view = db.create_materialized_view("copy", "people") + view = db.create_materialized_view("copy", "people", with_no_data=True) view.refresh() db.open_table("people").update(where="name = 'kid'", values={"age": 8}) @@ -87,7 +257,7 @@ def test_legacy_storage_source_update_rebuilds(tmp_path): storage_options={**STABLE_ROW_IDS, "new_table_data_storage_version": "legacy"}, ) db.create_table("people", [{"name": "ada", "age": 36}, {"name": "kid", "age": 7}]) - view = db.create_materialized_view("copy", "people") + view = db.create_materialized_view("copy", "people", with_no_data=True) view.refresh() db.open_table("people").update(where="name = 'kid'", values={"age": 8}) @@ -104,6 +274,11 @@ def test_list_and_not_a_view(tmp_path): assert db.list_materialized_views() == ["adults"] with pytest.raises(ValueError, match="not a materialized view"): db.open_materialized_view("people") + with pytest.raises(ValueError, match="not a materialized view"): + db.drop_materialized_view("people") + + db.drop_materialized_view("adults") + assert db.list_materialized_views() == [] def test_invalid_expression_fails_at_create(tmp_path): @@ -119,7 +294,10 @@ async def test_async_create_refresh_and_open(tmp_path): await db.create_table("people", [{"name": "ada", "age": 36}]) view = await db.create_materialized_view( - "shouts", "people", select=[("shout", "upper(name)")] + "shouts", + "people", + select=[("shout", "upper(name)")], + with_no_data=True, ) result = await view.refresh() assert result.mode == "rebuild" @@ -131,11 +309,35 @@ async def test_async_create_refresh_and_open(tmp_path): assert await db.list_materialized_views() == ["shouts"] +@pytest.mark.asyncio +async def test_async_create_and_refresh_jobs(tmp_path): + db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) + await db.create_table("people", [{"name": "ada", "age": 36}]) + + create_job = await db.create_materialized_view_async( + "adults", "people", with_no_data=True + ) + assert create_job.id is None + assert await create_job.wait() is None + + view = await db.open_materialized_view("adults") + refresh_job = await view.refresh_async() + assert refresh_job.id is None + result = await refresh_job.wait() + assert result.mode == "rebuild" + assert result.rows_written == 1 + + drop_job = await db.drop_materialized_view_async("adults") + assert drop_job.id is None + assert await drop_job.wait() is None + assert "adults" not in await db.list_materialized_views() + + @pytest.mark.asyncio async def test_async_incremental(tmp_path): db = await lancedb.connect_async(tmp_path, storage_options=STABLE_ROW_IDS) await db.create_table("people", [{"name": "ada", "age": 36}]) - view = await db.create_materialized_view("copy", "people") + view = await db.create_materialized_view("copy", "people", with_no_data=True) await view.refresh() table = await db.open_table("people") @@ -157,7 +359,10 @@ def test_bare_select_names_are_quoted(tmp_path): db.create_table("odd_names", [{"order item": "widget", "select": 2}]) view = db.create_materialized_view( - "quoted", "odd_names", select=["order item", "select"] + "quoted", + "odd_names", + select=["order item", "select"], + with_no_data=True, ) result = view.refresh() assert result.rows_written == 1 @@ -166,19 +371,6 @@ def test_bare_select_names_are_quoted(tmp_path): assert rows[0]["select"] == 2 -@pytest.mark.asyncio -async def test_async_remote_is_refused_without_network(): - db = await lancedb.connect_async( - "db://nowhere", api_key="sk_test", region="us-east-1" - ) - with pytest.raises(NotImplementedError, match="local"): - await db.create_materialized_view("v", "src") - with pytest.raises(NotImplementedError, match="local"): - await db.open_materialized_view("v") - with pytest.raises(NotImplementedError, match="local"): - await db.list_materialized_views() - - def test_scalar_select_is_one_column(tmp_path): db = make_db(tmp_path) view = db.create_materialized_view("just_name", "people", select="name") @@ -235,6 +427,17 @@ def test_namespace_connection_materialized_views(tmp_path): with pytest.raises(ValueError, match="not a materialized view"): db.open_materialized_view("people") + create_job = db.create_materialized_view_async( + "job_view", "people", with_no_data=True + ) + assert create_job.wait() is None + refresh_job = db.open_materialized_view("job_view").refresh_async() + assert refresh_job.wait().rows_written == 2 + + assert db.drop_materialized_view_async("job_view").wait() is None + db.drop_materialized_view("adults") + assert db.list_materialized_views() == [] + @pytest.mark.asyncio async def test_async_namespace_connection_materialized_views(tmp_path): @@ -267,6 +470,19 @@ async def test_async_namespace_connection_materialized_views(tmp_path): ) assert handle._namespace_path == through_namespace._namespace_path + create_job = await db.create_materialized_view_async( + "job_view", "people", with_no_data=True + ) + assert await create_job.wait() is None + job_view = await db.open_materialized_view("job_view") + refresh_job = await job_view.refresh_async() + assert (await refresh_job.wait()).rows_written == 2 + + drop_job = await db.drop_materialized_view_async("job_view") + assert await drop_job.wait() is None + await db.drop_materialized_view("adults") + assert await db.list_materialized_views() == [] + def test_namespaced_select_kind_is_read_and_unknown_kinds_are_refused(): import json diff --git a/python/src/connection.rs b/python/src/connection.rs index 2d613966a..65d4c98e8 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -381,7 +381,7 @@ impl Connection { }) } - #[pyo3(signature = (name, source, projections=None, filter=None, limit=None))] + #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))] pub fn create_materialized_view( self_: PyRef<'_, Self>, name: String, @@ -389,6 +389,7 @@ impl Connection { projections: Option>, filter: Option, limit: Option, + with_no_data: bool, ) -> PyResult> { let inner = self_.get_inner()?.clone(); future_into_py(self_.py(), async move { @@ -402,16 +403,79 @@ impl Connection { if let Some(limit) = limit { builder = builder.limit(limit); } - let view = builder.execute().await.infer_error()?; + builder = builder.with_no_data(with_no_data); + let view = Box::pin(builder.execute()).await.infer_error()?; Ok(Table::new(view.table().clone())) }) } + #[pyo3(signature = (name, source, projections=None, filter=None, limit=None, with_no_data=false))] + pub fn create_materialized_view_async( + self_: PyRef<'_, Self>, + name: String, + source: String, + projections: Option>, + filter: Option, + limit: Option, + with_no_data: bool, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + future_into_py(self_.py(), async move { + let mut builder = inner.create_materialized_view(name, source); + if let Some(projections) = projections { + builder = builder.select(projections); + } + if let Some(filter) = filter { + builder = builder.only_if(filter); + } + if let Some(limit) = limit { + builder = builder.limit(limit); + } + let job = Box::pin(builder.with_no_data(with_no_data).execute_async()) + .await + .infer_error()?; + Ok(crate::job::Job::new(job)) + }) + } + pub fn list_materialized_views(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.get_inner()?.clone(); future_into_py(self_.py(), async move { let views = inner.list_materialized_views().await.infer_error()?; - Ok(views.into_iter().map(|view| view.name).collect::>()) + Ok(views) + }) + } + + #[pyo3(signature = (name, namespace_path=None))] + pub fn drop_materialized_view( + self_: PyRef<'_, Self>, + name: String, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + inner + .drop_materialized_view(name, &namespace_path) + .await + .infer_error() + }) + } + + #[pyo3(signature = (name, namespace_path=None))] + pub fn drop_materialized_view_async( + self_: PyRef<'_, Self>, + name: String, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + inner + .drop_materialized_view_async(name, &namespace_path) + .await + .infer_error() + .map(crate::job::Job::new) }) } diff --git a/python/src/error.rs b/python/src/error.rs index aa13a8e87..b46fa6c83 100644 --- a/python/src/error.rs +++ b/python/src/error.rs @@ -29,6 +29,7 @@ impl PythonErrorExt for std::result::Result { LanceError::InvalidInput { .. } | LanceError::InvalidTableName { .. } | LanceError::TableNotFound { .. } + | LanceError::NotAMaterializedView { .. } | LanceError::Schema { .. } | LanceError::TableAlreadyExists { .. } => self.value_error(), LanceError::CreateDir { .. } => self.os_error(), diff --git a/python/src/table.rs b/python/src/table.rs index 784d29136..4cea61543 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -452,6 +452,17 @@ pub struct RefreshMaterializedViewResult { #[pymethods] impl RefreshMaterializedViewResult { + #[staticmethod] + pub fn from_json(value: &str) -> PyResult { + let result: lancedb::RefreshMaterializedViewResult = + serde_json::from_str(value).map_err(|err| { + PyValueError::new_err(format!( + "failed to decode materialized-view refresh result: {err}" + )) + })?; + Ok(Self::from(result)) + } + pub fn __repr__(&self) -> String { format!( "RefreshMaterializedViewResult(mode={}, rows_written={}, source_version={}, version={})", @@ -1647,6 +1658,40 @@ impl Table { }) } + #[pyo3(signature = (full=false, source_version=None))] + pub fn refresh_materialized_view_async( + self_: PyRef<'_, Self>, + full: bool, + source_version: Option, + ) -> PyResult> { + let inner = self_.inner_ref()?.clone(); + future_into_py(self_.py(), async move { + let view = lancedb::MaterializedView::from_table(inner) + .await + .infer_error()?; + let mut builder = view.refresh().full(full); + if let Some(version) = source_version { + builder = builder.source_version(version); + } + let job = builder.execute_async().await.infer_error()?; + Ok(crate::job::Job::new_typed(job)) + }) + } + + pub fn materialized_view_definition(self_: PyRef<'_, Self>) -> PyResult> { + let inner = self_.inner_ref()?.clone(); + future_into_py(self_.py(), async move { + let view = lancedb::MaterializedView::from_table(inner) + .await + .infer_error()?; + serde_json::to_string(view.definition()).map_err(|err| { + PyRuntimeError::new_err(format!( + "failed to serialize materialized-view definition: {err}" + )) + }) + }) + } + pub fn add_columns_with_schema( self_: PyRef<'_, Self>, schema: PyArrowType, diff --git a/rust/lancedb/src/database.rs b/rust/lancedb/src/database.rs index 843170030..b8d19443b 100644 --- a/rust/lancedb/src/database.rs +++ b/rust/lancedb/src/database.rs @@ -14,7 +14,7 @@ //! * Tables may be managed by a database system (e.g. Postgres) //! * A custom table implementation (e.g. remote table, etc.) may be used -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use std::time::Duration; @@ -28,6 +28,8 @@ use lance_namespace::models::{ use crate::data::scannable::Scannable; use crate::error::Result; +use crate::job::Job; +use crate::materialized_view::CreateMaterializedViewRequest; use crate::table::{BaseTable, WriteOptions}; pub mod listing; @@ -301,6 +303,73 @@ pub trait Database: ) -> Result> { function_catalog_not_supported() } + /// Create a materialized view through a remote catalog and return its + /// initial-population job. Local connections use the native declaration + /// path directly. + #[doc(hidden)] + async fn create_materialized_view_async( + &self, + _request: CreateMaterializedViewRequest, + ) -> Result { + job_op_not_supported("remote materialized-view creation") + } + /// Drop a materialized view through its resource endpoint and return its + /// cleanup job. Local connections validate the view and use table drop. + #[doc(hidden)] + async fn drop_materialized_view_async( + &self, + _name: &str, + _namespace_path: &[String], + ) -> Result { + job_op_not_supported("remote materialized-view drop") + } + /// List materialized-view names in a namespace. + #[doc(hidden)] + async fn list_materialized_views(&self, namespace_path: &[String]) -> Result> { + let mut names = Vec::new(); + let mut page_token = None; + let mut seen_page_tokens = HashSet::new(); + loop { + let response = self + .list_tables(ListTablesRequest { + id: Some(namespace_path.to_vec()), + page_token: page_token.clone(), + ..Default::default() + }) + .await?; + for name in response.tables { + let Ok(table) = self + .open_table(OpenTableRequest { + name: name.clone(), + namespace_path: namespace_path.to_vec(), + index_cache_size: None, + lance_read_params: None, + location: None, + namespace_client: None, + managed_versioning: None, + }) + .await + else { + continue; + }; + let schema = table.schema().await?; + if crate::materialized_view::materialized_view_kind(schema.metadata())?.is_some() { + names.push(name); + } + } + let Some(next_page_token) = response.page_token.filter(|token| !token.is_empty()) + else { + break; + }; + if !seen_page_tokens.insert(next_page_token.clone()) { + return Err(crate::Error::Runtime { + message: "materialized-view listing repeated a page token".into(), + }); + } + page_token = Some(next_page_token); + } + Ok(names) + } /// Look up one exact immutable Function version. async fn get_function( &self, diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index 98f1338d0..342e89bfa 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -5,9 +5,10 @@ //! //! A materialized view is a table whose contents are defined by a query over //! one source table and maintained by refresh rather than by writes. Creation -//! commits an empty table carrying the kind-tagged definition in schema -//! metadata; a kind added later reads back as unrefreshable, not as a plain -//! table. Queries, indexes and search work on the view unchanged. +//! records a kind-tagged definition in schema metadata and populates the view +//! unless creation explicitly requests no data. A kind added later reads back +//! as unrefreshable, not as a plain table. Queries, indexes and search work on +//! the view unchanged. pub mod refresh; @@ -28,6 +29,7 @@ use crate::database::listing::OPT_NEW_TABLE_ENABLE_STABLE_ROW_IDS; use crate::database::{CreateTableRequest, Database, OpenTableRequest}; use crate::embeddings::EmbeddingDefinition; use crate::function::FunctionBinding; +use crate::job::Job; use crate::table::Table; use crate::table::computed_columns::{ FUNCTION_BINDINGS_META_KEY, computed_column_from_field, computed_columns, @@ -124,6 +126,30 @@ pub struct MaterializedViewDefinition { pub inputs: Vec, } +/// The backend-independent metadata needed to open a materialized view. +#[doc(hidden)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct MaterializedViewInfo { + /// The parsed view definition. + pub definition: MaterializedViewDefinition, + /// The current physical incarnation, when one has been minted. + pub incarnation: Option, +} + +/// The backend-independent request used to create a remote materialized view. +#[doc(hidden)] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CreateMaterializedViewRequest { + /// Name of the new view. + pub name: String, + /// Namespace in which to create the view. + pub namespace_path: Vec, + /// Defining SELECT query. + pub query: String, + /// Whether to skip the initial population job. + pub with_no_data: bool, +} + /// Prefix of the internal columns holding source columns a computed column /// reads without the view projecting them; see /// [`PreparedDeclaration::input_column`]. @@ -199,6 +225,28 @@ pub fn materialized_view_kind( Ok(Some(MaterializedViewKind::Select(definition))) } +pub(crate) fn materialized_view_info_from_metadata( + name: &str, + metadata: &HashMap, +) -> Result { + let incarnation = metadata.get(INCARNATION_META_KEY).cloned(); + match materialized_view_kind(metadata)? { + Some(MaterializedViewKind::Select(definition)) => Ok(MaterializedViewInfo { + definition, + incarnation, + }), + Some(MaterializedViewKind::Unrecognized { kind }) => Err(Error::NotSupported { + message: format!( + "materialized view '{name}' is defined by '{kind}', which this version of \ + lancedb cannot refresh" + ), + }), + None => Err(Error::NotAMaterializedView { + name: name.to_string(), + }), + } +} + /// Resolve a definition against the source schema into the view's projected /// fields, with `inputs` filled in. Everything statically checkable is /// checked here rather than at refresh time. Empty `projections` selects @@ -1114,27 +1162,6 @@ pub async fn prepare_declaration( }) } -/// One row of [`Connection::list_materialized_views`]: a view's name and its -/// definition kind, which may be one this version cannot refresh. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct MaterializedViewEntry { - /// Name of the view's table. - pub name: String, - /// The view's definition as stored. - pub kind: MaterializedViewKind, -} - -/// Materialized views are local-only; refuse a remote connection before any -/// request is made. -fn ensure_local(connection: &Connection) -> Result<()> { - if connection.uri().starts_with("db://") { - return Err(Error::NotSupported { - message: "materialized views are supported only on local databases".into(), - }); - } - Ok(()) -} - /// Builds a materialized view. Created by /// [`Connection::create_materialized_view`]. pub struct CreateMaterializedViewBuilder { @@ -1146,6 +1173,7 @@ pub struct CreateMaterializedViewBuilder { projections: Vec<(String, String)>, filter: Option, limit: Option, + with_no_data: bool, } impl CreateMaterializedViewBuilder { @@ -1159,6 +1187,7 @@ impl CreateMaterializedViewBuilder { projections: Vec::new(), filter: None, limit: None, + with_no_data: false, } } @@ -1200,11 +1229,84 @@ impl CreateMaterializedViewBuilder { self } - /// Create the view: an empty table carrying the definition; refresh - /// computes the rows. The source must keep stable row ids -- they hold - /// provenance across compaction, and cannot be enabled later. + /// Create only the definition and empty backing table. By default create + /// also waits for the initial refresh so the returned view is populated. + pub fn with_no_data(mut self, with_no_data: bool) -> Self { + self.with_no_data = with_no_data; + self + } + + fn query(&self) -> String { + fn quote(name: &str) -> String { + format!("\"{}\"", name.replace('"', "\"\"")) + } + + let projection = if self.projections.is_empty() { + "*".to_string() + } else { + self.projections + .iter() + .map(|(output, expression)| format!("{expression} AS {}", quote(output))) + .collect::>() + .join(", ") + }; + let source = self + .source_namespace + .iter() + .chain(std::iter::once(&self.source)) + .map(|part| quote(part)) + .collect::>() + .join("."); + let mut query = format!("SELECT {projection} FROM {source}"); + if let Some(filter) = &self.filter { + query.push_str(" WHERE "); + query.push_str(filter); + } + if let Some(limit) = self.limit { + query.push_str(&format!(" LIMIT {limit}")); + } + query + } + + /// Submit creation and initial population, returning a [`Job`] that + /// settles when the view is ready. The source must keep stable row ids -- + /// they hold provenance across compaction, and cannot be enabled later. + pub async fn execute_async(self) -> Result { + if self.connection.uri().starts_with("db://") { + return self + .connection + .database() + .create_materialized_view_async(CreateMaterializedViewRequest { + name: self.name.clone(), + namespace_path: self.namespace.clone(), + query: self.query(), + with_no_data: self.with_no_data, + }) + .await; + } + Ok(Job::spawned(tokio::spawn(async move { + self.execute_native().await.map(|_| ()) + }))) + } + + /// Create and populate the view, waiting until it is ready. pub async fn execute(self) -> Result { - ensure_local(&self.connection)?; + if !self.connection.uri().starts_with("db://") { + return self.execute_native().await; + } + let connection = self.connection.clone(); + let name = self.name.clone(); + let namespace = self.namespace.clone(); + self.execute_async().await?.wait().await?; + let table = connection + .open_table(name) + .namespace(namespace) + .execute() + .await?; + MaterializedView::from_table(table).await + } + + async fn execute_native(self) -> Result { let source = self .connection .open_table(&self.source) @@ -1218,7 +1320,11 @@ impl CreateMaterializedViewBuilder { self.limit, ) .await?; - prepared.create_in(&self.namespace, &self.name).await + let view = prepared.create_in(&self.namespace, &self.name).await?; + if !self.with_no_data { + view.refresh().execute().await?; + } + Ok(view) } } @@ -1235,32 +1341,12 @@ impl MaterializedView { /// for a plain table, [`Error::NotSupported`] for a kind this version /// cannot refresh. pub async fn from_table(table: Table) -> Result { - // Same local-only boundary the connection-level entry points hold, - // applied before the schema read so a remote table costs no request. - if table.as_native().is_none() { - return Err(Error::NotSupported { - message: "materialized views are supported only on local databases".into(), - }); - } - let schema = table.schema().await?; - let incarnation = schema.metadata().get(INCARNATION_META_KEY).cloned(); - match materialized_view_kind(schema.metadata())? { - Some(MaterializedViewKind::Select(definition)) => Ok(Self { - table, - definition, - incarnation, - }), - Some(MaterializedViewKind::Unrecognized { kind }) => Err(Error::NotSupported { - message: format!( - "materialized view '{}' is defined by '{kind}', which this version of \ - lancedb cannot refresh", - table.name() - ), - }), - None => Err(Error::NotAMaterializedView { - name: table.name().to_string(), - }), - } + let info = table.base_table().materialized_view_info().await?; + Ok(Self { + table, + definition: info.definition, + incarnation: info.incarnation, + }) } /// The view, as the table it is. Queries, indexes and search all apply. @@ -1345,22 +1431,52 @@ impl RefreshMaterializedViewBuilder { self } + /// Submit the refresh and return a job that settles with its result. + pub async fn execute_async(self) -> Result> { + if self.view.table.as_native().is_none() { + return self + .view + .table + .base_table() + .refresh_materialized_view_async( + self.full, + self.source_version, + self.expected_incarnation.as_deref(), + ) + .await; + } + Ok(Job::spawned(tokio::spawn(async move { + refresh::execute_refresh( + &self.view.table, + self.full, + self.source_version, + self.expected_incarnation.as_deref(), + ) + .await + }))) + } + + /// Refresh the view, waiting for the job to finish. pub async fn execute(self) -> Result { - refresh::execute_refresh( - &self.view.table, - self.full, - self.source_version, - self.expected_incarnation.as_deref(), - ) - .await + if self.view.table.as_native().is_some() { + return refresh::execute_refresh( + &self.view.table, + self.full, + self.source_version, + self.expected_incarnation.as_deref(), + ) + .await; + } + self.execute_async().await?.wait().await } } impl Connection { /// Define a materialized view named `name` over `source`. /// - /// The view is created empty, with the definition recorded in its schema - /// metadata; refresh computes the rows. Local databases only. + /// The definition is recorded in schema metadata and the initial refresh + /// is completed before this method returns. Use + /// [`CreateMaterializedViewBuilder::with_no_data`] to skip population. /// /// ```no_run /// # #![recursion_limit = "256"] @@ -1372,7 +1488,7 @@ impl Connection { /// .only_if("age >= 18") /// .execute() /// .await?; - /// view.refresh().execute().await?; + /// assert_eq!(view.table().count_rows(None).await?, 1); /// # Ok(()) /// # } /// ``` @@ -1389,28 +1505,75 @@ impl Connection { &self, name: impl Into, ) -> Result { - ensure_local(self)?; let table = self.open_table(name).execute().await?; MaterializedView::from_table(table).await } - /// The materialized views in this database, unrefreshable kinds included. - /// Costs a table open per table; one that cannot be opened is skipped - /// rather than failing the listing. - pub async fn list_materialized_views(&self) -> Result> { - ensure_local(self)?; - let names = self.table_names().execute().await?; - let mut views = Vec::new(); - for name in names { - let Ok(table) = self.open_table(&name).execute().await else { - continue; - }; - let schema = table.schema().await?; - if let Some(kind) = materialized_view_kind(schema.metadata())? { - views.push(MaterializedViewEntry { name, kind }); - } + /// The names of materialized views in the root namespace. + pub async fn list_materialized_views(&self) -> Result> { + self.database().list_materialized_views(&[]).await + } + + /// Drop a materialized view. + /// + /// The view may become unavailable before its physical data is removed. + /// Use [`Connection::drop_materialized_view_async`] to retain the cleanup + /// job and wait for it explicitly. + pub async fn drop_materialized_view( + &self, + name: impl AsRef, + namespace_path: &[String], + ) -> Result<()> { + let name = name.as_ref(); + if self.uri().starts_with("db://") { + return self + .database() + .drop_materialized_view_async(name, namespace_path) + .await + .map(|_| ()); } - Ok(views) + let table = self + .open_table(name) + .namespace(namespace_path.to_vec()) + .execute() + .await?; + MaterializedView::from_table(table).await?; + self.drop_table(name, namespace_path).await + } + + /// Start dropping a materialized view and return its cleanup job. + /// + /// This validates that the named resource is a materialized view rather + /// than an ordinary table. Call [`Job::wait`] before assuming physical + /// cleanup has finished. + /// + /// ```no_run + /// # use lancedb::Connection; + /// # async fn drop_view(conn: &Connection) -> lancedb::Result<()> { + /// let job = conn.drop_materialized_view_async("daily_sales", &[]).await?; + /// job.wait().await?; + /// # Ok(()) + /// # } + /// ``` + pub async fn drop_materialized_view_async( + &self, + name: impl AsRef, + namespace_path: &[String], + ) -> Result { + let name = name.as_ref(); + if self.uri().starts_with("db://") { + return self + .database() + .drop_materialized_view_async(name, namespace_path) + .await; + } + let table = self + .open_table(name) + .namespace(namespace_path.to_vec()) + .execute() + .await?; + MaterializedView::from_table(table).await?; + self.drop_table_async(name, namespace_path).await } } @@ -1523,9 +1686,86 @@ mod tests { .data_type(), &DataType::UInt64 ); + assert_eq!(view.table().count_rows(None).await.unwrap(), 3); + } + + #[tokio::test] + async fn test_with_no_data_skips_initial_refresh() { + let conn = people_db().await; + let view = conn + .create_materialized_view("empty", "people") + .with_no_data(true) + .execute() + .await + .unwrap(); assert_eq!(view.table().count_rows(None).await.unwrap(), 0); } + #[tokio::test] + async fn test_create_and_refresh_async_jobs() { + let conn = people_db().await; + let create_job = conn + .create_materialized_view("async_view", "people") + .with_no_data(true) + .execute_async() + .await + .unwrap(); + assert!(create_job.id().is_none()); + create_job.wait().await.unwrap(); + + let view = conn.open_materialized_view("async_view").await.unwrap(); + assert_eq!(view.table().count_rows(None).await.unwrap(), 0); + + let refresh_job = view.refresh().execute_async().await.unwrap(); + assert!(refresh_job.id().is_none()); + let result = refresh_job.wait().await.unwrap(); + assert_eq!(result.mode, RefreshMode::Rebuild); + assert_eq!(result.rows_written, 3); + assert_eq!(view.table().count_rows(None).await.unwrap(), 3); + + let drop_job = conn + .drop_materialized_view_async("async_view", &[]) + .await + .unwrap(); + assert!(drop_job.id().is_none()); + drop_job.wait().await.unwrap(); + assert!(conn.open_table("async_view").execute().await.is_err()); + } + + #[tokio::test] + async fn test_drop_materialized_view_rejects_plain_tables() { + let conn = people_db().await; + let error = conn + .drop_materialized_view("people", &[]) + .await + .unwrap_err(); + assert!(matches!(error, Error::NotAMaterializedView { .. })); + + conn.create_materialized_view("drop_me", "people") + .with_no_data(true) + .execute() + .await + .unwrap(); + conn.drop_materialized_view("drop_me", &[]).await.unwrap(); + assert!(conn.open_table("drop_me").execute().await.is_err()); + } + + #[tokio::test] + async fn test_remote_query_quotes_resource_identifiers() { + let conn = people_db().await; + let query = conn + .create_materialized_view("unused", "odd\"source") + .source_namespace(vec!["raw data".into()]) + .select([("double\"age", "age * 2")]) + .only_if("age >= 18") + .limit(10) + .query(); + assert_eq!( + query, + "SELECT age * 2 AS \"double\"\"age\" FROM \"raw data\".\"odd\"\"source\" WHERE age >= 18 LIMIT 10" + ); + } + /// No projection selects every source column, expanded now: the schema /// captured at creation is the definition. #[tokio::test] @@ -1673,14 +1913,9 @@ mod tests { .unwrap(); let views = conn.list_materialized_views().await.unwrap(); - assert_eq!( - views.iter().map(|v| v.name.as_str()).collect::>(), - vec!["adults"] - ); - let MaterializedViewKind::Select(definition) = &views[0].kind else { - panic!("expected a select view"); - }; - assert_eq!(definition.filter.as_deref(), Some("age >= 18")); + assert_eq!(views, vec!["adults"]); + let view = conn.open_materialized_view("adults").await.unwrap(); + assert_eq!(view.definition().filter.as_deref(), Some("age >= 18")); } /// The creation option outranks a connection configured to create @@ -1763,7 +1998,7 @@ mod tests { /// A newer-kind view must not disappear from the listing. #[tokio::test] - async fn test_unrecognized_kind_is_listed_with_its_kind() { + async fn test_unrecognized_kind_is_listed_by_name() { let conn = people_db().await; conn.create_materialized_view("v", "people") .execute() @@ -1781,36 +2016,7 @@ mod tests { .unwrap(); let views = conn.list_materialized_views().await.unwrap(); - assert_eq!(views.len(), 1); - assert_eq!(views[0].name, "v"); - assert_eq!( - views[0].kind, - MaterializedViewKind::Unrecognized { - kind: "join".into() - } - ); - } - - /// Remote connections are refused before any request is made. - #[cfg(feature = "remote")] - #[tokio::test] - async fn test_remote_connection_is_refused_up_front() { - let conn = connect("db://nowhere") - .api_key("sk_test") - .region("us-east-1") - .execute() - .await - .unwrap(); - let err = conn - .create_materialized_view("v", "src") - .execute() - .await - .unwrap_err(); - assert!(matches!(err, Error::NotSupported { .. })); - let err = conn.open_materialized_view("v").await.unwrap_err(); - assert!(matches!(err, Error::NotSupported { .. })); - let err = conn.list_materialized_views().await.unwrap_err(); - assert!(matches!(err, Error::NotSupported { .. })); + assert_eq!(views, vec!["v"]); } /// A definition must evaluate identically across refreshes; anything @@ -2461,6 +2667,7 @@ mod tests { let view = conn .create_materialized_view("adults", "people") + .with_no_data(true) .namespace(vec!["ns".to_string()]) .source_namespace(vec!["ns".to_string()]) .select([("name", "name")]) diff --git a/rust/lancedb/src/materialized_view/refresh.rs b/rust/lancedb/src/materialized_view/refresh.rs index 433b185f7..62b3db22f 100644 --- a/rust/lancedb/src/materialized_view/refresh.rs +++ b/rust/lancedb/src/materialized_view/refresh.rs @@ -1657,6 +1657,7 @@ mod tests { async fn doubled_view(conn: &Connection) -> MaterializedView { conn.create_materialized_view("doubled", "src") + .with_no_data(true) .select([("x", "x"), ("twice", "x * 2")]) .execute() .await @@ -1724,6 +1725,7 @@ mod tests { let (conn, _) = db_with_source(vec![1, 20, 3, 40]).await; let view = conn .create_materialized_view("big", "src") + .with_no_data(true) .select([("x", "x")]) .only_if("x > 10") .execute() @@ -1749,6 +1751,7 @@ mod tests { .await .unwrap(); conn.create_materialized_view("democrats", "src") + .with_no_data(true) .select([("id", "id")]) .only_if(r#""PartyAbbrev" = 'D'"#) .execute() @@ -1784,6 +1787,7 @@ mod tests { .unwrap(); let view = conn .create_materialized_view("legacy_view", "legacy_src") + .with_no_data(true) .select([("id", "id")]) .only_if(r#""PartyAbbrev" = 'X'"#) .execute() @@ -1851,6 +1855,7 @@ mod tests { let (conn, source) = db_with_source(vec![1, 20]).await; let view = conn .create_materialized_view("big", "src") + .with_no_data(true) .select([("x", "x")]) .only_if("x > 10") .execute() @@ -1872,6 +1877,7 @@ mod tests { let (conn, source) = db_with_source(vec![20]).await; let view = conn .create_materialized_view("big", "src") + .with_no_data(true) .select([("x", "x")]) .only_if("x > 10") .execute() @@ -1929,6 +1935,7 @@ mod tests { .unwrap(); let view = conn .create_materialized_view("legacy_doubled", "legacy_src") + .with_no_data(true) .select([("x", "x"), ("twice", "x * 2")]) .execute() .await @@ -2007,6 +2014,7 @@ mod tests { let (conn, source) = db_with_source(vec![1, 2, 3]).await; let view = conn .create_materialized_view("drifting_view", "src") + .with_no_data(true) .select([("x", "x"), ("twice", "x * 2")]) .execute() .await @@ -2093,6 +2101,7 @@ mod tests { let (conn, source) = db_with_source(vec![1, 2, 3]).await; let view = conn .create_materialized_view("atomic_view", "src") + .with_no_data(true) .select([("x", "x"), ("twice", "x * 2")]) .execute() .await @@ -2195,6 +2204,7 @@ mod tests { let (conn, _) = db_with_source(vec![1, 2, 3]).await; let view = conn .create_materialized_view("raced_rebuild", "src") + .with_no_data(true) .select([("x", "x"), ("twice", "x * 2")]) .execute() .await @@ -2291,6 +2301,7 @@ mod tests { let (conn, source) = db_with_source(vec![1, 2, 3]).await; let view = conn .create_materialized_view("raced_incremental", "src") + .with_no_data(true) .select([("x", "x"), ("twice", "x * 2")]) .execute() .await @@ -2409,6 +2420,7 @@ mod tests { let (conn, source) = db_with_source(vec![1, 2, 3]).await; let view = conn .create_materialized_view("empty", "src") + .with_no_data(true) .select([("x", "x")]) .limit(0) .execute() @@ -2576,6 +2588,7 @@ mod tests { let (conn, source) = db_with_source(vec![1, 2]).await; let view = conn .create_materialized_view("capped", "src") + .with_no_data(true) .select([("x", "x")]) .limit(2) .execute() @@ -2608,6 +2621,7 @@ mod tests { let (conn, source) = db_with_source(vec![1, 2, 3]).await; let view = conn .create_materialized_view("capped", "src") + .with_no_data(true) .select([("x", "x")]) .limit(4) .execute() @@ -2677,6 +2691,7 @@ mod tests { let (conn, _) = db_with_source(vec![1, 2]).await; let view = conn .create_materialized_view("none", "src") + .with_no_data(true) .select([("x", "x")]) .only_if("x > 100") .execute() @@ -2720,6 +2735,7 @@ mod tests { let (conn, source) = db_with_source(vec![1]).await; let view = conn .create_materialized_view("v", "src") + .with_no_data(true) .select([("twice", "x * 2")]) .execute() .await @@ -2779,6 +2795,7 @@ mod tests { let second = conn .create_materialized_view("second", "doubled") + .with_no_data(true) .only_if("twice > 10") .execute() .await @@ -3146,6 +3163,7 @@ mod tests { let (conn, _) = db_with_source(vec![1, 2]).await; let view = conn .create_materialized_view("v", "src") + .with_no_data(true) .select([("double value", "x * 2")]) .execute() .await @@ -3199,6 +3217,7 @@ mod tests { // An active-LSM source is refused at create. let err = conn .create_materialized_view("v", "src") + .with_no_data(true) .execute() .await .unwrap_err(); @@ -3209,6 +3228,7 @@ mod tests { table.unset_lsm_write_spec().await.unwrap(); let view = conn .create_materialized_view("v", "src") + .with_no_data(true) .execute() .await .unwrap(); diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 1d6f9abe8..7180b04ce 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -26,6 +26,7 @@ use crate::database::{ use crate::error::Result; use crate::function::{FunctionRegistrationRequest, FunctionVersion}; use crate::job::Job; +use crate::materialized_view::CreateMaterializedViewRequest; use crate::remote::job::{RemoteJob, job_state_to_client}; use crate::remote::util::stream_as_body; use crate::table::BaseTable; @@ -587,6 +588,127 @@ impl Database for RemoteDatabase { }) } + async fn create_materialized_view_async( + &self, + request: CreateMaterializedViewRequest, + ) -> Result { + let identifier = build_table_identifier( + &request.name, + &request.namespace_path, + &self.client.id_delimiter, + ); + let req = self + .client + .post(&format!("/v1/materialized_view/{identifier}/create")) + .json(&serde_json::json!({ + "query": request.query, + "with_no_data": request.with_no_data, + })); + let (request_id, response) = self.client.send(req).await?; + let response = self.client.check_response(&request_id, response).await?; + let status = response.status(); + let body = response.text().await.err_to_http(request_id.clone())?; + let job_id = extract_job_id(&body); + + if request.with_no_data { + return Ok(match job_id { + Some(job_id) => Job::new(Box::new(RemoteJob::new(self.client.clone(), job_id))), + None => Job::new_done(), + }); + } + if status != StatusCode::ACCEPTED { + return Err(Error::Http { + source: "materialized-view creation with data must return 202 Accepted".into(), + request_id, + status_code: Some(status), + }); + } + let job_id = job_id.ok_or_else(|| Error::Http { + source: "materialized-view creation response did not contain a valid job_id".into(), + request_id, + status_code: Some(status), + })?; + Ok(Job::new(Box::new(RemoteJob::new( + self.client.clone(), + job_id, + )))) + } + + async fn drop_materialized_view_async( + &self, + name: &str, + namespace_path: &[String], + ) -> Result { + let identifier = build_table_identifier(name, namespace_path, &self.client.id_delimiter); + let request = self + .client + .post(&format!("/v1/materialized_view/{identifier}/drop")); + let (request_id, response) = self.client.send(request).await?; + let response = self.client.check_response(&request_id, response).await?; + let status = response.status(); + let body = response.text().await.err_to_http(request_id.clone())?; + if status != StatusCode::ACCEPTED { + return Err(Error::Http { + source: "materialized-view drop must return 202 Accepted".into(), + request_id, + status_code: Some(status), + }); + } + let job_id = extract_job_id(&body).ok_or_else(|| Error::Http { + source: "materialized-view drop response did not contain a valid job_id".into(), + request_id, + status_code: Some(status), + })?; + self.table_cache + .remove(&build_cache_key(name, namespace_path)) + .await; + Ok(Job::new(Box::new(RemoteJob::new( + self.client.clone(), + job_id, + )))) + } + + async fn list_materialized_views(&self, namespace_path: &[String]) -> Result> { + #[derive(serde::Deserialize)] + struct ListMaterializedViewsResponse { + #[serde(default)] + views: Vec, + #[serde(default)] + page_token: Option, + } + + let namespace_id = build_namespace_identifier(namespace_path, &self.client.id_delimiter); + let path = format!("/v1/namespace/{namespace_id}/materialized_view/list"); + let mut views = Vec::new(); + let mut page_token: Option = None; + let mut seen_page_tokens = HashSet::new(); + loop { + let mut req = self.client.get(&path); + if let Some(token) = &page_token { + req = req.query(&[("page_token", token)]); + } + let (request_id, response) = self.client.send(req).await?; + let response = self.client.check_response(&request_id, response).await?; + let status = response.status(); + let response: ListMaterializedViewsResponse = + response.json().await.err_to_http(request_id.clone())?; + views.extend(response.views); + let Some(next_page_token) = response.page_token.filter(|token| !token.is_empty()) + else { + break; + }; + if !seen_page_tokens.insert(next_page_token.clone()) { + return Err(Error::Http { + source: "Materialized-view listing response repeated a page_token".into(), + request_id, + status_code: Some(status), + }); + } + page_token = Some(next_page_token); + } + Ok(views) + } + async fn create_function_async( &self, request: FunctionRegistrationRequest, @@ -1292,6 +1414,8 @@ mod tests { use lance_namespace_impls::{DynamicContextProvider, OperationInfo}; use crate::connection::ConnectBuilder; + use crate::database::Database; + use crate::materialized_view::CreateMaterializedViewRequest; use crate::{ Connection, Error, database::CreateTableMode, @@ -1331,6 +1455,103 @@ mod tests { assert_eq!(key1, key6, "Same inputs should produce same cache key"); } + #[tokio::test] + async fn test_create_materialized_view_uses_item_route_and_job() { + let db = super::RemoteDatabase::new_mock(|request| { + assert_eq!(request.method(), "POST"); + assert_eq!( + request.url().path(), + "/v1/materialized_view/analytics$adults/create" + ); + let body = request + .body() + .and_then(reqwest::Body::as_bytes) + .and_then(|bytes| serde_json::from_slice::(bytes).ok()) + .unwrap(); + assert_eq!( + body["query"], + "SELECT age AS \"age\" FROM \"raw\".\"people\" WHERE age >= 18 LIMIT 10" + ); + assert_eq!(body["with_no_data"], false); + http::Response::builder() + .status(202) + .body(serde_json::json!({"job_id": "j1-mv-create"}).to_string()) + .unwrap() + }); + let job = db + .create_materialized_view_async(CreateMaterializedViewRequest { + name: "adults".into(), + namespace_path: vec!["analytics".into()], + query: "SELECT age AS \"age\" FROM \"raw\".\"people\" WHERE age >= 18 LIMIT 10" + .into(), + with_no_data: false, + }) + .await + .unwrap(); + assert_eq!(job.id(), Some("j1-mv-create")); + } + + #[tokio::test] + async fn test_drop_materialized_view_uses_item_route_and_job() { + let db = super::RemoteDatabase::new_mock(|request| { + assert_eq!(request.method(), "POST"); + assert_eq!( + request.url().path(), + "/v1/materialized_view/analytics$adults/drop" + ); + assert!(request.body().is_none()); + http::Response::builder() + .status(202) + .body(serde_json::json!({"job_id": "j1-mv-drop"}).to_string()) + .unwrap() + }); + let job = db + .drop_materialized_view_async("adults", &["analytics".into()]) + .await + .unwrap(); + assert_eq!(job.id(), Some("j1-mv-drop")); + } + + #[tokio::test] + async fn test_list_materialized_views_follows_empty_pages() { + let page = Arc::new(AtomicUsize::new(0)); + let db = super::RemoteDatabase::new_mock({ + let page = page.clone(); + move |request| { + assert_eq!(request.method(), "GET"); + assert_eq!( + request.url().path(), + "/v1/namespace/analytics/materialized_view/list" + ); + match page.fetch_add(1, Ordering::SeqCst) { + 0 => { + assert!(request.url().query().is_none()); + http::Response::builder() + .status(200) + .body( + serde_json::json!({"views": [], "page_token": "next"}).to_string(), + ) + .unwrap() + } + 1 => { + assert_eq!(request.url().query(), Some("page_token=next")); + http::Response::builder() + .status(200) + .body(serde_json::json!({"views": ["adults"]}).to_string()) + .unwrap() + } + _ => panic!("listing requested too many pages"), + } + } + }); + assert_eq!( + db.list_materialized_views(&["analytics".into()]) + .await + .unwrap(), + ["adults"] + ); + } + #[tokio::test] async fn test_retries() { // We'll record the request_id here, to check it matches the one in the error. diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 5c32e3cdd..747e2b73c 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -17,6 +17,9 @@ use crate::index::IndexStatistics; use crate::index::scalar::FtsQuery; use crate::index::waiter::wait_for_index; use crate::job::Job; +use crate::materialized_view::{ + MaterializedViewDefinition, MaterializedViewInfo, RefreshMaterializedViewResult, ViewProjection, +}; use crate::query::{QueryFilter, QueryRequest, Select, VectorQueryRequest}; use crate::remote::job::RemoteJob; use crate::table::AddColumnsResult; @@ -250,10 +253,17 @@ struct FreshnessJob { inner: RemoteJob, freshness: Arc>, version: Arc>>, - track_refresh_result: bool, + tracked_result: TrackedJobResult, freshness_request: FreshnessHeaders, } +#[derive(Clone, Copy)] +enum TrackedJobResult { + None, + RefreshColumn, + MaterializedView, +} + #[async_trait] impl crate::job::JobHandle for FreshnessJob { fn id(&self) -> Option<&str> { @@ -279,22 +289,26 @@ impl crate::job::JobHandle for FreshnessJob { let result = crate::job::JobHandle::wait(&self.inner).await?; let version = self.version.read().await; if version.is_none() { - let result_version = self - .track_refresh_result - .then(|| result.value()) - .flatten() - .and_then(|value| { + let result_version = match self.tracked_result { + TrackedJobResult::None => None, + TrackedJobResult::RefreshColumn => result.value().and_then(|value| { serde_json::from_value::(value.clone()) .ok() - }) - .map(|result| { - result - .published_version - .map_or(result.source_version, |version| { - version.max(result.source_version) + .map(|result| { + result + .published_version + .map_or(result.source_version, |version| { + version.max(result.source_version) + }) }) - }) - .filter(|version| *version != 0); + }), + TrackedJobResult::MaterializedView => result.value().and_then(|value| { + serde_json::from_value::(value.clone()) + .ok() + .map(|result| result.version) + }), + } + .filter(|version| *version != 0); if let Some(version) = result_version { self.freshness_request .observe_version(&self.freshness, version); @@ -2051,6 +2065,106 @@ impl BaseTable for RemoteTable { fn id(&self) -> &str { &self.identifier } + async fn materialized_view_info(&self) -> Result { + #[derive(Deserialize)] + struct Projection { + output_column: String, + expression: String, + } + + #[derive(Deserialize)] + struct DescribeMaterializedViewResponse { + source_table: String, + #[serde(default)] + source_namespace: Vec, + #[serde(default)] + projections: Vec, + #[serde(default)] + filter: Option, + #[serde(default)] + limit: Option, + #[serde(default)] + inputs: Vec, + #[serde(default)] + incarnation: Option, + } + + let request = self.client.post(&format!( + "/v1/materialized_view/{}/describe", + self.identifier + )); + let (request_id, response) = self.send(request, true).await?; + let response = self.check_table_response(&request_id, response).await?; + let response: DescribeMaterializedViewResponse = + response.json().await.err_to_http(request_id)?; + Ok(MaterializedViewInfo { + definition: MaterializedViewDefinition { + source_table: response.source_table, + source_namespace: response.source_namespace, + projections: response + .projections + .into_iter() + .map(|projection| ViewProjection { + output: projection.output_column, + expression: projection.expression, + }) + .collect(), + filter: response.filter, + limit: response.limit, + inputs: response.inputs, + }, + incarnation: response.incarnation, + }) + } + + async fn refresh_materialized_view_async( + &self, + full: bool, + source_version: Option, + expected_incarnation: Option<&str>, + ) -> Result> { + self.check_mutable().await?; + let mut body = serde_json::json!({ "full": full }); + if let Some(source_version) = source_version { + body["source_version"] = source_version.into(); + } + if let Some(expected_incarnation) = expected_incarnation { + body["expected_incarnation"] = expected_incarnation.into(); + } + let request = self + .client + .post(&format!( + "/v1/materialized_view/{}/refresh", + self.identifier + )) + .json(&body); + let freshness_request = self.snapshot_freshness_headers(); + let (request_id, response) = self + .send_with_freshness(request, true, freshness_request) + .await?; + let response = self.check_table_response(&request_id, response).await?; + let status = response.status(); + let body = response.text().await.err_to_http(request_id.clone())?; + if status != StatusCode::ACCEPTED { + return Err(Error::Http { + source: "materialized-view refresh must return 202 Accepted".into(), + request_id, + status_code: Some(status), + }); + } + let job_id = extract_job_id(&body).ok_or_else(|| Error::Http { + source: "materialized-view refresh response did not contain a valid job_id".into(), + request_id, + status_code: Some(status), + })?; + Ok(Job::new_typed(Box::new(FreshnessJob { + inner: RemoteJob::new(self.client.clone(), job_id), + freshness: self.freshness.clone(), + version: self.version.clone(), + tracked_result: TrackedJobResult::MaterializedView, + freshness_request, + }))) + } async fn query_snapshot(&self) -> Result> { let description = self.describe().await?; let TableDescription { @@ -2901,7 +3015,7 @@ impl BaseTable for RemoteTable { inner: RemoteJob::new(self.client.clone(), job_id), freshness: self.freshness.clone(), version: self.version.clone(), - track_refresh_result: false, + tracked_result: TrackedJobResult::None, freshness_request: self.snapshot_freshness_headers(), })), None => Job::new_done(), @@ -3384,7 +3498,7 @@ impl BaseTable for RemoteTable { inner: RemoteJob::new(self.client.clone(), response.job_id), freshness: self.freshness.clone(), version: self.version.clone(), - track_refresh_result: true, + tracked_result: TrackedJobResult::RefreshColumn, freshness_request: self.snapshot_freshness_headers(), }))) } @@ -12017,17 +12131,77 @@ mod tests { } #[tokio::test] - async fn test_materialized_view_refused_without_a_request() { - // Materialized views are local-only. The table-level entry the - // bindings use must refuse a remote table before reading its schema, - // so the panicking handler is the assertion. - let table = Table::new_with_handler("my_table", |request| -> http::Response { - panic!("unexpected request: {}", request.url().path()) + async fn test_materialized_view_describe_and_refresh() { + let table = Table::new_with_handler("my_table", |request| match request.url().path() { + "/v1/materialized_view/my_table/describe" => http::Response::builder() + .status(200) + .body( + json!({ + "name": "my_table", + "source_table": "source", + "source_namespace": ["analytics"], + "projections": [{ + "output_column": "double_x", + "expression": "x * 2" + }], + "filter": "x > 0", + "limit": 10, + "inputs": ["x"], + "incarnation": "inc-1" + }) + .to_string(), + ) + .unwrap(), + "/v1/materialized_view/my_table/refresh" => { + assert_eq!(request.method(), "POST"); + assert_eq!( + request_body_json(&request), + json!({ + "full": true, + "source_version": 7, + "expected_incarnation": "inc-1" + }) + ); + http::Response::builder() + .status(202) + .body(json!({"job_id": "j1-mv-refresh"}).to_string()) + .unwrap() + } + "/v1/jobs/describe" => http::Response::builder() + .status(200) + .body( + json!({ + "job_id": "j1-mv-refresh", + "job_state": "DONE", + "result": { + "mode": "rebuild", + "rows_written": 2, + "source_version": 7, + "version": 9 + } + }) + .to_string(), + ) + .unwrap(), + path => panic!("unexpected request: {path}"), }); - let err = crate::MaterializedView::from_table(table) + let view = crate::MaterializedView::from_table(table).await.unwrap(); + assert_eq!(view.definition().source_table, "source"); + assert_eq!(view.definition().source_namespace, ["analytics"]); + assert_eq!(view.definition().inputs, ["x"]); + assert_eq!(view.incarnation(), Some("inc-1")); + + let result = view + .refresh() + .full(true) + .source_version(7) + .expect_incarnation("inc-1") + .execute() .await - .unwrap_err(); - assert!(matches!(err, Error::NotSupported { .. }), "got {err:?}"); + .unwrap(); + assert_eq!(result.mode, crate::RefreshMode::Rebuild); + assert_eq!(result.rows_written, 2); + assert_eq!(result.version, 9); } #[tokio::test] diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 5732955f5..0cf6a0e4f 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -563,6 +563,29 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { fn id(&self) -> &str; /// Get the arrow [Schema] of the table. async fn schema(&self) -> Result; + /// Read this table's materialized-view definition and incarnation. + #[doc(hidden)] + async fn materialized_view_info( + &self, + ) -> Result { + let schema = self.schema().await?; + crate::materialized_view::materialized_view_info_from_metadata( + self.name(), + schema.metadata(), + ) + } + /// Submit a materialized-view refresh. + #[doc(hidden)] + async fn refresh_materialized_view_async( + &self, + _full: bool, + _source_version: Option, + _expected_incarnation: Option<&str>, + ) -> Result> { + Err(Error::NotSupported { + message: "remote materialized-view refresh is not supported on this table type".into(), + }) + } /// Create a read-only handle pinned to the table's current active revision. /// /// The returned handle is independent from later refreshes or checkouts on From 3fca33fcb5555175df53c7e4243be3ebab3caf62 Mon Sep 17 00:00:00 2001 From: Bruno Ramirez Date: Tue, 15 Sep 2026 13:48:31 -0600 Subject: [PATCH 15/43] fix: shut down the shared Tokio runtime on interpreter exit (#4175) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Short-lived Python processes using this client can occasionally crash with SIGABRT during interpreter shutdown, even after every operation they ran completed successfully. The cause is the shared Tokio runtime backing every async call: it's never told to shut down at normal process exit, only reset (and deliberately leaked) on `fork()`. Its worker threads keep running, uncoordinated with the interpreter, until the process actually ends, and if one is mid-task exactly as `Py_Finalize` starts tearing down interpreter state, it can panic on state that's already gone. That panic happens on a background thread with no PyO3-wrapped call frame to catch it, so Rust aborts the whole process instead of just failing that one call. This PR gives the runtime a coordinated, bounded shutdown by registering a Python `atexit` callback that runs while the interpreter is still fully valid. Getting the exit lifecycle right took a few rounds of review. Earlier versions freed the runtime as soon as `Arc::strong_count` looked low, but that's the wrong signal — it reflects who currently holds a reference, not who's logically still in flight. That mistake showed up three ways: a caller could dereference memory already freed out from under it; an install already in progress could finish invisibly after `shutdown()` had already decided there was nothing to do; and a spawned task could end up as the final owner of the `Runtime`, so completing it dropped the runtime from inside one of its own worker threads, which Tokio itself forbids and panics on (this reproduced unprompted in this branch's own test suite). Fixing all three meant replacing reference-count-based tracking with an explicit counter of in-flight top-level calls that `shutdown()` waits on directly. This was accomplished with the following changes: - The runtime lives in an `ArcSwapOption`, where `Tagged` pairs the `Runtime` with the fork generation it was built in. - An `OUTSTANDING` counter, incremented before a top-level `spawn`/`spawn_blocking`/`block_on` call does anything else and decremented only once it has truly finished (via an `OutstandingGuard` token that carries no reference to the runtime), is what `shutdown()` waits on — not `Arc::strong_count` or whether the slot looks empty. This closes the install-race and makes it impossible for a task's own completion to be the runtime's final drop. - Once `shutdown()`'s bound elapses, it stops waiting and attempts retirement anyway, rather than returning with the runtime and its workers left fully alive. - `spawn`/`spawn_blocking` use `Handle::try_current()` to pin any nested spawn (`future_into_py` spawns a task that itself spawns a second one for the real work) to whichever runtime is already executing it, so a reclaim landing between the two calls can't split one logical operation across two different runtime instances. - The fork-child handler now only bumps a bare `GENERATION` counter — no `ArcSwapOption` call of any kind from that context, since `swap`/`compare_and_swap` do real reader-reconciliation work (thread-local state, potentially an allocation) that isn't safe in a forked child. `get_runtime()` compares its installed runtime's generation against the live counter from ordinary context and rebuilds on a mismatch. - Registered `shutdown_runtime` as a Python `atexit` callback in the `_lancedb` module init, running with the GIL released (`Python::detach`) since the bounded wait could otherwise deadlock against any in-flight task that itself needs the GIL. ### Testing - Unit tests in `runtime.rs` cover: shutdown with no runtime created, shutdown after use and lazy rebuild afterward, calling shutdown twice in a row, a concurrent stress test racing many threads against shutdown, a nested-spawn test reproducing `future_into_py`'s own spawn-within-a-spawn shape under concurrent shutdown, a test confirming a top-level task in flight survives a concurrent shutdown reclaim, and a test forcing the install-vs-shutdown race directly. - Built the wheel and ran a concurrent reproducer (many threads hammering the client while `atexit` fires) over 100 times with no hangs or crashes, plus a 30-second-join variant and repeated runs of a short-lived process confirming clean exits with no added latency. --------- Co-authored-by: Claude Sonnet 5 --- Cargo.lock | 1 + python/Cargo.toml | 1 + python/src/lib.rs | 26 ++- python/src/runtime.rs | 490 +++++++++++++++++++++++++++++++++++++++--- 4 files changed, 485 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 090e2c654..0f30926f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5597,6 +5597,7 @@ dependencies = [ name = "lancedb-python" version = "0.39.0-beta.8" dependencies = [ + "arc-swap", "arrow", "async-trait", "bytes", diff --git a/python/Cargo.toml b/python/Cargo.toml index d7ec8cd61..314664e14 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -15,6 +15,7 @@ name = "_lancedb" crate-type = ["cdylib"] [dependencies] +arc-swap = "1.9" arrow = { workspace = true, features = ["pyarrow"] } async-trait.workspace = true bytes.workspace = true diff --git a/python/src/lib.rs b/python/src/lib.rs index f83776e7b..0bd7d5d6d 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -8,8 +8,8 @@ use expr::{PyExpr, expr_col, expr_func, expr_lit}; use index::IndexConfig; use permutation::{PyAsyncPermutationBuilder, PyPermutationReader}; use pyo3::{ - Bound, PyResult, Python, pymodule, - types::{PyModule, PyModuleMethods}, + Bound, PyResult, Python, pyfunction, pymodule, + types::{PyAnyMethods, PyModule, PyModuleMethods}, wrap_pyfunction, }; use query::{FTSQuery, HybridQuery, Query, VectorQuery}; @@ -38,8 +38,24 @@ pub mod sql; pub mod table; pub mod util; +/// Shut down the shared Tokio runtime (see `runtime::shutdown`). +/// +/// Registered below as a Python `atexit` callback rather than called +/// directly: `atexit` runs while the interpreter is still fully valid, +/// which is the coordinated, bounded exit the runtime otherwise never gets. +/// +/// Runs the actual wait with the GIL released (`Python::detach`): shutdown +/// blocks the calling thread waiting on the runtime's own worker threads, +/// and if any in-flight task needs the GIL to finish (e.g. one that calls +/// back into Python), holding it here while waiting on that same task would +/// deadlock rather than time out. +#[pyfunction] +fn shutdown_runtime(py: Python<'_>) { + py.detach(|| runtime::shutdown(std::time::Duration::from_secs(5))); +} + #[pymodule] -pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { +pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { let env = Env::new() .filter_or("LANCEDB_LOG", "warn") .write_style("LANCEDB_LOG_STYLE"); @@ -98,5 +114,9 @@ pub fn _lancedb(_py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(expr_lit, m)?)?; m.add_function(wrap_pyfunction!(expr_func, m)?)?; m.add("__version__", env!("CARGO_PKG_VERSION"))?; + // Give the shared runtime a coordinated, bounded shutdown at normal + // process exit -- see `shutdown_runtime` and `runtime::shutdown` for why. + py.import("atexit")? + .call_method1("register", (wrap_pyfunction!(shutdown_runtime, m)?,))?; Ok(()) } diff --git a/python/src/runtime.rs b/python/src/runtime.rs index 13951a7aa..170f7e592 100644 --- a/python/src/runtime.rs +++ b/python/src/runtime.rs @@ -4,20 +4,84 @@ //! Fork-safe wrapper around tokio + pyo3-async-runtimes. //! //! `pyo3_async_runtimes::tokio` keeps its multi-threaded runtime in a -//! `OnceLock` that can never be replaced. Tokio's worker threads do not +//! `OnceLock` that can never be replaced. Tokio's worker threads do not //! survive `fork()`, so once a child inherits a "frozen" runtime, every -//! `future_into_py` call hangs forever. +//! `future_into_py` call hangs forever. Normal (non-fork) process exit has +//! its own gap: nothing tells the runtime to shut down, so its worker +//! threads keep running, uncoordinated with the interpreter, right up until +//! the process ends. If one of them is mid-task exactly as `Py_Finalize` +//! starts tearing down interpreter state, it can panic on state that's +//! already gone -- and since that happens on a background thread with no +//! PyO3-wrapped call frame to catch it, Rust aborts the whole process +//! rather than failing that one call. [`shutdown`], registered as a Python +//! `atexit` callback, closes that gap by giving the runtime a coordinated, +//! bounded exit while the interpreter is still fully valid. //! -//! We sidestep the global by routing every future through our own -//! [`LanceRuntime`] (a [`pyo3_async_runtimes::generic::Runtime`] impl) backed -//! by an [`AtomicPtr`] to a tokio runtime that we own. A `pthread_atfork` -//! child handler nulls the pointer; the next `spawn` rebuilds the runtime in -//! the child. This mirrors the pattern used in the Lance Python bindings. +//! Getting both of these right at once took a few tries; the design here +//! rests on three separate mechanisms, each solving one problem the others +//! cannot: +//! +//! **`OUTSTANDING`, not `Arc::strong_count`, decides when the runtime is +//! idle.** Early versions tried to infer "is anything still using this +//! runtime" from how many `Arc` clones existed. That signal is +//! wrong in both directions: a clone taken only for the instant a task is +//! *submitted* says nothing about whether that task has actually finished +//! running (`Runtime::shutdown_timeout` gives spawned, non-blocking tasks +//! no grace period at all -- a task "keeps running until it yields, then is +//! dropped" -- so a reclaim landing right after submission would silently +//! abandon it before it ever got to run); and a clone held for a task's +//! *whole* lifetime can end up making that task the final owner of the +//! `Runtime`, so completing it drops the `Runtime` from inside one of its +//! own worker threads, which tokio itself forbids ("cannot drop a runtime +//! in a context where blocking is not allowed") and panics. `OUTSTANDING` +//! is an explicit courtesy counter instead: every top-level `spawn`, +//! `spawn_blocking`, or `block_on` call increments it before it starts and +//! decrements it (via [`OutstandingGuard`]) when it is truly done, entirely +//! decoupled from how many `Arc` clones exist at any instant. `shutdown` +//! waits for it to reach zero before ever touching the runtime, which also +//! closes a narrower race: because the counter is incremented *before* +//! `get_runtime()` is even called, an install already in progress when +//! `shutdown` runs is never invisible to it the way an empty slot would be. +//! If the counter never reaches zero within the bound, `shutdown` stops +//! waiting and forces the retirement attempt anyway -- silently returning +//! with the runtime and its workers still fully alive would just recreate +//! the exact race this function exists to close, for any call slower than +//! the grace period. +//! +//! **Tasks never hold an `Arc`.** Because `OUTSTANDING` (not +//! reference counting) is what `shutdown` waits on, a top-level task only +//! needs to carry an `OutstandingGuard` -- a token whose `Drop` is a plain +//! atomic decrement -- not a clone of the runtime itself. That is what +//! makes it impossible for a task's completion to become the final, +//! worker-thread-side drop of the `Runtime`: nothing a task holds ever +//! *is* the `Runtime`. +//! +//! **`atfork_child` touches nothing but a plain counter.** `future_into_py` +//! spawns a task that, once running, spawns a second one to do the real +//! work and awaits its `JoinHandle`; if a nested call re-resolved "the +//! current runtime" independently, a reclaim landing between the two calls +//! could bind them to different instances. `spawn`/`spawn_blocking` close +//! that with `Handle::try_current`: a call already running on one of our +//! worker threads stays pinned to that instance, so only the first, +//! outermost call of a chain ever consults [`get_runtime`]. That leaves +//! fork as the other place identity can change, and it has to be handled +//! without ever calling into `ArcSwapOption` from the child handler itself +//! -- `swap`/`compare_and_swap` reconcile reader "debts" internally (via +//! thread-local state, and potentially an allocation), none of which is +//! safe to run in a forked child that may have inherited another thread's +//! lock mid-acquisition. `atfork_child` therefore does nothing but bump a +//! bare `GENERATION` counter; [`get_runtime`] compares the generation its +//! installed runtime was built in against the live counter on every call, +//! from ordinary (non-signal) context, and treats a mismatch as "stale, +//! rebuild" -- exactly the check `atfork_child` used to perform directly. use std::future::Future; use std::pin::Pin; -use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::time::{Duration, Instant}; +use arc_swap::ArcSwapOption; use pyo3::{Bound, PyAny, PyResult, Python, conversion::IntoPyObject}; use pyo3_async_runtimes::{ TaskLocals, @@ -25,8 +89,31 @@ use pyo3_async_runtimes::{ }; use tokio::{runtime, task}; -static RUNTIME: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); -static RUNTIME_INSTALLING: AtomicBool = AtomicBool::new(false); +/// A runtime tagged with the fork generation it was built in, so a stale +/// (post-fork, dead-worker-threads) instance can be told apart from a live +/// one without `atfork_child` ever having to touch it directly. +struct Tagged { + runtime: runtime::Runtime, + generation: u64, +} + +impl std::ops::Deref for Tagged { + type Target = runtime::Runtime; + fn deref(&self) -> &runtime::Runtime { + &self.runtime + } +} + +static RUNTIME: ArcSwapOption = ArcSwapOption::const_empty(); +/// Bumped only by `atfork_child`, and only ever read elsewhere. This is the +/// entire fork-safety mechanism: no lock, no allocation, no thread-local +/// access -- just one atomic add, which is all a `pthread_atfork` child +/// handler is ever safe to do. +static GENERATION: AtomicU64 = AtomicU64::new(0); +/// Count of top-level `spawn`/`spawn_blocking`/`block_on` calls that have +/// started but not yet finished. See the module docs for why this, and not +/// `Arc::strong_count`, is what `shutdown` waits on. +static OUTSTANDING: AtomicU64 = AtomicU64::new(0); static ATFORK_INSTALLED: AtomicBool = AtomicBool::new(false); fn create_runtime() -> runtime::Runtime { @@ -37,23 +124,67 @@ fn create_runtime() -> runtime::Runtime { .expect("Failed to build tokio runtime") } -fn get_runtime() -> &'static runtime::Runtime { +/// Get a live, owned handle to the shared runtime, rebuilding it if the +/// installed one predates the most recent `fork()`. +fn get_runtime() -> Arc { + let current_gen = GENERATION.load(Ordering::SeqCst); loop { - let ptr = RUNTIME.load(Ordering::SeqCst); - if !ptr.is_null() { - return unsafe { &*ptr }; + let existing = RUNTIME.load_full(); + if let Some(existing) = &existing + && existing.generation == current_gen + { + return Arc::clone(existing); } - if !RUNTIME_INSTALLING.fetch_or(true, Ordering::SeqCst) { - break; + if !ATFORK_INSTALLED.fetch_or(true, Ordering::SeqCst) { + install_atfork(); } - std::thread::yield_now(); + // Built optimistically, outside any lock: on the rare race where two + // threads both find the slot empty (or stale), one candidate wins + // the compare-and-swap below and the other is simply dropped here, + // tearing down its own (never shared, never used) worker pool the + // ordinary way. + let candidate = Arc::new(Tagged { + runtime: create_runtime(), + generation: current_gen, + }); + let previous = RUNTIME.compare_and_swap(&existing, Some(Arc::clone(&candidate))); + let won = match (&*previous, &existing) { + (None, None) => true, + (Some(prev), Some(exist)) => Arc::ptr_eq(prev, exist), + _ => false, + }; + if won { + if let Some(stale) = existing { + // A prior generation's runtime: its worker threads are dead + // in this process (they do not survive fork), so dropping it + // normally would try to join them and hang. Leak it instead. + std::mem::forget(stale); + } + return candidate; + } + // Someone else's candidate (or a concurrent shutdown) won; go around + // and reload. } - if !ATFORK_INSTALLED.fetch_or(true, Ordering::SeqCst) { - install_atfork(); +} + +/// RAII token tracked by [`OUTSTANDING`]. Held for the duration of a +/// top-level `block_on` call, or moved into a top-level `spawn`/ +/// `spawn_blocking` task so it decrements only once that task's *entire* +/// body -- including anything it nested-spawns and awaits -- has run to +/// completion or been dropped without completing. +struct OutstandingGuard; + +impl OutstandingGuard { + fn new() -> Self { + OUTSTANDING.fetch_add(1, Ordering::SeqCst); + Self + } +} + +impl Drop for OutstandingGuard { + fn drop(&mut self) { + OUTSTANDING.fetch_sub(1, Ordering::SeqCst); } - let new_ptr = Box::into_raw(Box::new(create_runtime())); - RUNTIME.store(new_ptr, Ordering::SeqCst); - unsafe { &*new_ptr } } /// Block the current thread on a future using the shared runtime. @@ -62,16 +193,78 @@ fn get_runtime() -> &'static runtime::Runtime { /// building a namespace client). Must not be called from within the runtime's /// own worker threads. pub fn block_on(fut: F) -> F::Output { + let _guard = OutstandingGuard::new(); get_runtime().block_on(fut) } -/// Runs in async-signal context after `fork()` in the child. We can only -/// touch atomics here; we deliberately leak the previous runtime because -/// dropping a tokio `Runtime` would try to join its (now-dead) worker -/// threads and hang. +/// Gracefully quiesce the shared runtime, meant to run at normal process exit. +/// +/// Waits (bounded by `timeout`) for [`OUTSTANDING`] to reach zero -- i.e. +/// for every top-level call already under way to actually finish, not just +/// for `Arc::strong_count` to look low -- before ever touching the runtime. +/// If the bound elapses first, it stops waiting and attempts retirement +/// anyway: leaving the runtime and its worker threads untouched would just +/// recreate the exact race this function exists to close, for any call +/// slower than the grace period. +/// +/// Retirement itself removes the runtime from the slot and calls +/// `shutdown_timeout` rather than a bare `drop`: dropping a tokio `Runtime` +/// waits (in the worst case indefinitely) for its worker threads to join, +/// whereas `shutdown_timeout` gives real in-flight work -- a connection +/// pool's keep-alive, a graceful close -- a bounded chance to finish first, +/// then forcibly ends whatever has not. Reclaiming can only proceed once +/// `Arc::try_unwrap` proves no other reference remains; if some transient +/// `get_runtime()` caller is, at that exact instant, still between loading +/// the slot and finishing its own call, this abandons the runtime instead +/// of forcing the issue -- the same trade `atfork_child` already makes. +/// +/// Neither of the two ways this can fail to cleanly retire the runtime -- +/// the wait timing out, or `try_unwrap` losing that race -- has any other +/// signal to report through (`shutdown_timeout` itself returns nothing), +/// so both log a warning instead of failing silently. +pub fn shutdown(timeout: Duration) { + let deadline = Instant::now() + timeout; + loop { + let outstanding = OUTSTANDING.load(Ordering::SeqCst); + if outstanding == 0 { + break; + } + if Instant::now() >= deadline { + log::warn!( + "lancedb: runtime shutdown timed out with {outstanding} call(s) still in flight; forcing shutdown anyway, some in-flight work may be abandoned" + ); + break; + } + std::thread::sleep(Duration::from_millis(1)); + } + let Some(current) = RUNTIME.load_full() else { + return; + }; + RUNTIME.compare_and_swap(&Some(Arc::clone(¤t)), None); + match Arc::try_unwrap(current) { + Ok(tagged) => { + tagged + .runtime + .shutdown_timeout(deadline.saturating_duration_since(Instant::now())); + } + Err(_) => { + // Some transient `get_runtime()` caller is, at this exact + // instant, still between loading the slot and finishing its own + // call: we have no owned handle to call `shutdown_timeout` on, + // and no way to force one (tokio has no shutdown API over a + // shared reference). Nothing more to do but say so. + log::warn!( + "lancedb: runtime shutdown could not obtain exclusive ownership; the shared runtime was left running" + ); + } + } +} + +/// Runs in async-signal context after `fork()` in the child. Touches +/// nothing but a plain atomic add -- see the module docs for why even +/// `ArcSwapOption::swap` is not safe to call here. extern "C" fn atfork_child() { - RUNTIME.store(std::ptr::null_mut(), Ordering::SeqCst); - RUNTIME_INSTALLING.store(false, Ordering::SeqCst); + GENERATION.fetch_add(1, Ordering::SeqCst); } #[cfg(not(windows))] @@ -102,11 +295,26 @@ impl Runtime for LanceRuntime { type JoinError = LanceJoinError; type JoinHandle = Pin> + Send>>; + /// `pyo3_async_runtimes::generic::future_into_py` spawns a task that, + /// once it starts running, spawns a second one to do the real work and + /// awaits its `JoinHandle`. `Handle::try_current` pins that nested call + /// to whatever runtime is already executing it, so only the first, + /// outermost call of a chain -- one running on a thread outside any + /// runtime -- ever consults [`get_runtime`] or [`OUTSTANDING`]. fn spawn(fut: F) -> Self::JoinHandle where F: Future + Send + 'static, { - let handle = get_runtime().spawn(fut); + let handle = match tokio::runtime::Handle::try_current() { + Ok(handle) => handle.spawn(fut), + Err(_) => { + let guard = OutstandingGuard::new(); + get_runtime().spawn(async move { + let _guard = guard; + fut.await; + }) + } + }; Box::pin(async move { handle.await.map_err(LanceJoinError) }) } @@ -114,7 +322,16 @@ impl Runtime for LanceRuntime { where F: FnOnce() + Send + 'static, { - let handle = get_runtime().spawn_blocking(f); + let handle = match tokio::runtime::Handle::try_current() { + Ok(handle) => handle.spawn_blocking(f), + Err(_) => { + let guard = OutstandingGuard::new(); + get_runtime().spawn_blocking(move || { + let _guard = guard; + f(); + }) + } + }; Box::pin(async move { handle.await.map_err(LanceJoinError) }) } } @@ -149,3 +366,216 @@ where { pyo3_async_runtimes::generic::future_into_py::(py, fut) } + +#[cfg(test)] +mod tests { + use super::*; + + // RUNTIME, GENERATION, OUTSTANDING, and ATFORK_INSTALLED are process-wide + // statics, and Rust's test harness runs tests in parallel by default, + // so separate test functions below would otherwise race each other + // through this shared state (this reproduced in CI: one test's + // in-flight task got reclaimed by a *different* test's concurrent + // `shutdown()` call, and another observed `OUTSTANDING` left non-zero + // by a still-running sibling). Every test takes this lock first so + // only one of them touches the shared runtime state at a time; a + // poisoned lock (a previous test's genuine failure) is still honored + // rather than cascading into every later test as an unrelated panic. + static TEST_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); + + fn lock_runtime_state_for_test() -> std::sync::MutexGuard<'static, ()> { + TEST_LOCK + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + // A task's own completion must never be the final drop of the shared + // `Runtime`: tasks carry only an `OutstandingGuard` (a plain counter + // token), never an `Arc`, specifically so this can't happen. + // Getting this wrong panics ("cannot drop a runtime in a context where + // blocking is not allowed") -- this reproduced unprompted, twice, in a + // single run of `test_nested_spawn_survives_concurrent_shutdown` under + // an earlier design that held the `Arc` for a task's whole lifetime. + #[test] + #[allow(unused_must_use)] // fire-and-forget spawn, same as future_into_py itself + fn test_top_level_task_survives_concurrent_shutdown_reclaim() { + use std::sync::mpsc; + let _lock = lock_runtime_state_for_test(); + + for _ in 0..50 { + let (tx, rx) = mpsc::channel::<()>(); + + LanceRuntime::spawn(async move { + for _ in 0..5 { + task::yield_now().await; + } + let _ = tx.send(()); + }); + + shutdown(Duration::from_secs(2)); + + rx.recv_timeout(Duration::from_secs(5)) + .expect("top-level task was abandoned by a concurrent shutdown reclaim"); + } + } + + #[test] + fn test_shutdown_stops_and_the_runtime_rebuilds_lazily_after() { + let _lock = lock_runtime_state_for_test(); + + // No runtime created yet in this process: shutdown must be a no-op, + // not a null-pointer dereference. + shutdown(Duration::from_secs(1)); + + // Force the runtime into existence, then shut it down. Bounded by + // the timeout, so a hang here means shutdown itself is broken, not + // that the test is slow. + assert_eq!(block_on(async { 1 + 1 }), 2); + shutdown(Duration::from_secs(5)); + + // A caller after shutdown -- e.g. a stray call racing with the + // atexit callback -- must get a fresh, working runtime rather than + // a dangling reference into the one just torn down. + assert_eq!(block_on(async { 2 + 2 }), 4); + + // Shutting down twice in a row (e.g. atexit firing more than once) + // must not panic or double-free. + shutdown(Duration::from_secs(5)); + shutdown(Duration::from_secs(5)); + } + + // Adapted from the concurrent reproducer that found the original bug: + // many threads hammering the runtime while shutdown races them, not + // just the sequential rebuild path above. Before every reader held its + // own `Arc`, this could dereference a runtime `shutdown` had already + // freed, or hang forever. Repeated, since a race is not guaranteed to + // show up on any single attempt. + #[test] + fn test_shutdown_is_safe_concurrently_with_live_callers() { + use std::sync::Barrier; + use std::sync::atomic::AtomicBool as StopFlag; + let _lock = lock_runtime_state_for_test(); + + for _ in 0..50 { + let barrier = Arc::new(Barrier::new(9)); + let stop = Arc::new(StopFlag::new(false)); + let workers: Vec<_> = (0..8) + .map(|_| { + let barrier = Arc::clone(&barrier); + let stop = Arc::clone(&stop); + std::thread::spawn(move || { + barrier.wait(); + while !stop.load(Ordering::Relaxed) { + assert_eq!(block_on(async { 1 + 1 }), 2); + } + }) + }) + .collect(); + + barrier.wait(); + shutdown(Duration::from_millis(50)); + stop.store(true, Ordering::Relaxed); + for worker in workers { + worker.join().unwrap(); + } + } + } + + // Reproduces the actual bug mechanism, not just concurrent `block_on` + // traffic: `future_into_py` spawns an outer task that, once running, + // spawns a second (inner) one for the real work and awaits its + // `JoinHandle` (see `pyo3_async_runtimes::generic::future_into_py_with_locals`). + // Before `spawn`/`spawn_blocking` pinned nested calls to whatever + // runtime is already executing them, a reclaim landing between the + // outer and inner spawn could bind them to two different runtime + // instances -- and if the outer task's own runtime was the one torn + // down while it awaited the inner task, it never resumed. Every wait + // here is bounded so a reintroduced bug fails this test instead of + // hanging the suite. + #[test] + #[allow(unused_must_use)] // fire-and-forget outer spawn, same as future_into_py itself + fn test_nested_spawn_survives_concurrent_shutdown() { + use std::sync::mpsc; + let _lock = lock_runtime_state_for_test(); + + for _ in 0..50 { + let stop = Arc::new(AtomicBool::new(false)); + let (done_tx, done_rx) = mpsc::channel::<()>(); + + let workers: Vec<_> = (0..8) + .map(|_| { + let stop = Arc::clone(&stop); + let done_tx = done_tx.clone(); + std::thread::spawn(move || { + while !stop.load(Ordering::Relaxed) { + let (tx, rx) = mpsc::sync_channel::<()>(1); + LanceRuntime::spawn(async move { + let inner = LanceRuntime::spawn(async { + let _ = 1 + 1; + }); + let _ = inner.await; + let _ = tx.send(()); + }); + // A hang here across a concurrent shutdown is + // exactly the bug this test exists to catch. + let _ = rx.recv_timeout(Duration::from_secs(2)); + } + let _ = done_tx.send(()); + }) + }) + .collect(); + + std::thread::sleep(Duration::from_millis(10)); + shutdown(Duration::from_millis(50)); + stop.store(true, Ordering::Relaxed); + + for _ in 0..8 { + done_rx + .recv_timeout(Duration::from_secs(10)) + .expect("worker hung after shutdown raced a nested spawn"); + } + for worker in workers { + worker.join().unwrap(); + } + } + } + + // Forces the exact interleaving the install-race finding described: an + // installer registers as outstanding (as `spawn`'s outermost branch + // does, before it ever calls `get_runtime()`) and then pauses, while a + // concurrent `shutdown()` must not decide "nothing here" and return + // before that install actually completes and is retired in turn. + #[test] + fn test_shutdown_waits_for_a_racing_install() { + use std::sync::mpsc; + let _lock = lock_runtime_state_for_test(); + + // Clean slate: no runtime installed, OUTSTANDING at zero. + shutdown(Duration::from_secs(5)); + + let (installer_ready_tx, installer_ready_rx) = mpsc::channel::<()>(); + let (proceed_tx, proceed_rx) = mpsc::channel::<()>(); + + let installer = std::thread::spawn(move || { + let _guard = OutstandingGuard::new(); + installer_ready_tx.send(()).unwrap(); + proceed_rx.recv().unwrap(); + assert_eq!(get_runtime().block_on(async { 1 + 1 }), 2); + }); + + installer_ready_rx.recv().unwrap(); + let shutdown_thread = std::thread::spawn(|| shutdown(Duration::from_secs(5))); + // Give shutdown's polling loop several chances to (wrongly) observe + // an idle runtime before the installer is allowed to proceed. + std::thread::sleep(Duration::from_millis(50)); + proceed_tx.send(()).unwrap(); + + installer.join().unwrap(); + shutdown_thread.join().unwrap(); + + // shutdown must have waited for the install to finish and then + // retired it, not returned early and left it stranded. + assert!(RUNTIME.load_full().is_none()); + assert_eq!(OUTSTANDING.load(Ordering::SeqCst), 0); + } +} From 95055c4c545b39126ce7f5bce4cb00be8055fb84 Mon Sep 17 00:00:00 2001 From: Lance Release Date: Tue, 15 Sep 2026 21:05:23 +0000 Subject: [PATCH 16/43] =?UTF-8?q?Bump=20version:=200.39.0-beta.8=20?= =?UTF-8?q?=E2=86=92=200.39.0-beta.9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 23f097af5..aae26f441 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.39.0-beta.8" +current_version = "0.39.0-beta.9" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 0f30926f5..85c55b7fa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5474,7 +5474,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.39.0-beta.8" +version = "0.39.0-beta.9" dependencies = [ "ahash", "anyhow", @@ -5570,7 +5570,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.39.0-beta.8" +version = "0.39.0-beta.9" dependencies = [ "arrow-array", "arrow-buffer", @@ -5595,7 +5595,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.39.0-beta.8" +version = "0.39.0-beta.9" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 1b5f14153..1272fd71a 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.39.0-beta.8 + 0.39.0-beta.9 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 5f9702aaa..d85915b67 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.39.0-beta.8 + 0.39.0-beta.9 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 9b1c19f96..16e39b909 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.39.0-beta.8 + 0.39.0-beta.9 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 636289907..51243382f 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.39.0-beta.8" +version = "0.39.0-beta.9" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 7e8684be8..e7ada7229 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.39.0-beta.8", + "version": "0.39.0-beta.9", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 274875fec..4333b10c7 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.39.0-beta.8", + "version": "0.39.0-beta.9", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index c01cac42f..d0b0f18fa 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.39.0-beta.8", + "version": "0.39.0-beta.9", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 1398953d4..e7ac1eb12 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.39.0-beta.8", + "version": "0.39.0-beta.9", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 9823eda62..5eadc836f 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.39.0-beta.8", + "version": "0.39.0-beta.9", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index ce6d2922f..880637a0f 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.39.0-beta.8", + "version": "0.39.0-beta.9", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 8f6a919b2..de9b54f07 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.39.0-beta.8", + "version": "0.39.0-beta.9", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index db979c1e7..b23100926 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.39.0-beta.8", + "version": "0.39.0-beta.9", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 314664e14..4cf914f6a 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.39.0-beta.8" +version = "0.39.0-beta.9" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 1934ff8ce..2c3d6dd84 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.39.0-beta.8" +version = "0.39.0-beta.9" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 1d2a5d084b48ef89bcf9fdcd4e938dbdd6db3a1e Mon Sep 17 00:00:00 2001 From: Will Jones Date: Tue, 15 Sep 2026 15:46:32 -0700 Subject: [PATCH 17/43] fix: accept all-null batches and plain JSON strings for json columns (#4067) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two ways of writing to a `json` column failed or silently corrupted data. **All-null batches were rejected.** `add()` refused a batch whose values for a `json` column were all null, while every plain Arrow type accepted the same batch. This bites row-at-a-time inserts hardest: a one-row batch with no value for an optional column is trivially all-null, so most such writes failed. pyarrow infers `null` as the column's type, and the write path had no handling for it — casting to the table's type dropped the field metadata that identifies the column as `lance.json`, so lance rejected the batch (`` `val` should have type json but type was large_binary ``). A null-typed input column now becomes typed nulls matching the table's field exactly, metadata included. **Unlabelled JSON text was stored raw.** JSON supplied as plain strings (what pyarrow infers for a column of `str`) was cast to the column's `LargeBinary` storage type and relabelled `lance.json`, putting unparsed text where JSONB was expected. Reads returned the text unnormalized and `json_extract` failed with `InvalidJsonb`. Lance-core does the JSONB encoding, but only for input labelled `arrow.json`, so string input is now labelled rather than cast — at the top level and inside structs. Both fixes are in the shared Rust write path, so they apply to any binding, including hand-built Arrow tables that never pass through Python's list-of-dicts type inference. `_align_field` gets the same JSON-string fix for the legacy Python `_sanitize_data` path, which `on_bad_vectors` and embedding functions still route through. The blob v2 half of the issue landed separately in #4065, which added a `DataType::Null` arm to blob coercion. This PR keeps that implementation and adds end-to-end add-path coverage for it. The tests from #4066 are included here and pass, so that PR's Python-layer inference changes are no longer needed to close the issue. Fixes #3759 --------- Co-authored-by: Claude Opus 5 (1M context) --- python/python/lancedb/table.py | 70 ++- python/python/tests/test_blob.py | 74 +++ python/python/tests/test_table.py | 197 ++++++- rust/lancedb/src/table/add_data.rs | 216 ++++++++ rust/lancedb/src/table/datafusion/cast.rs | 629 +++++++++++++++++++++- rust/lancedb/tests/blob_integration.rs | 47 ++ 6 files changed, 1181 insertions(+), 52 deletions(-) diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 84ae4e836..95b10422a 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -634,26 +634,43 @@ def _align_field_types( return new_fields -def _align_list_value_field( - value_field: pa.Field, target_value_field: pa.Field -) -> pa.Field: - # A list has exactly one child, so the inferred child name ("item") aligns - # positionally and adopts the table's child name; pa.Table.cast renames it. - return _align_field(value_field, target_value_field).with_name( - target_value_field.name - ) +def _align_container_child(child: pa.Field, target_child: pa.Field) -> pa.Field: + # A list has one child, a map one key and one item, so an inferred child name + # ("item") aligns positionally and adopts the table's; pa.Table.cast renames it. + return _align_field(child, target_child).with_name(target_child.name) + + +def _arrow_json_storage_type(input_type: pa.DataType) -> Optional[pa.DataType]: + """The storage type arrow.json would use for ``input_type``. + + Returns None if the type cannot hold JSON text. + """ + if pa.types.is_string(input_type) or pa.types.is_string_view(input_type): + return pa.string() + if pa.types.is_large_string(input_type): + return pa.large_string() + return None def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field: - # Preserve arrow.json input until it reaches Lance. LanceDB exposes stored - # JSON columns as lance.json (JSONB-backed LargeBinary), but casting the - # input to that storage type here merely relabels the raw JSON bytes as + # LanceDB exposes stored JSON columns as lance.json (JSONB-backed LargeBinary), but + # casting the input to that storage type here merely relabels the raw JSON bytes as # JSONB. Lance must see arrow.json so it can perform the JSONB encoding. - if ( - _field_extension_name(field) == "arrow.json" - and _field_extension_name(target_field) == "lance.json" - ): - return field + if _field_extension_name(target_field) == "lance.json": + if _field_extension_name(field) == "arrow.json": + return field + # Plain JSON text, which is what pyarrow infers for a column of `str`, only + # needs the arrow.json label. + json_storage = _arrow_json_storage_type(field.type) + if json_storage is not None: + # Labelled through metadata rather than pa.json_(), which only exists on + # newer PyArrow; Lance reads the extension name off the field either way. + return pa.field( + field.name, + json_storage, + field.nullable, + {"ARROW:extension:name": "arrow.json"}, + ) if pa.types.is_struct(target_field.type): if pa.types.is_struct(field.type): new_type = pa.struct( @@ -667,7 +684,7 @@ def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field: elif pa.types.is_list(target_field.type): if _is_list_like(field.type): new_type = pa.list_( - _align_list_value_field( + _align_container_child( field.type.value_field, target_field.type.value_field ) ) @@ -676,7 +693,7 @@ def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field: elif pa.types.is_large_list(target_field.type): if _is_list_like(field.type): new_type = pa.large_list( - _align_list_value_field( + _align_container_child( field.type.value_field, target_field.type.value_field ) ) @@ -685,13 +702,28 @@ def _align_field(field: pa.Field, target_field: pa.Field) -> pa.Field: elif pa.types.is_fixed_size_list(target_field.type): if _is_list_like(field.type): new_type = pa.list_( - _align_list_value_field( + _align_container_child( field.type.value_field, target_field.type.value_field ), target_field.type.list_size, ) else: new_type = target_field.type + elif pa.types.is_map(target_field.type): + if pa.types.is_map(field.type): + # A map has exactly one key and one item field, so like a list's child they + # align positionally and adopt the table's names. + new_type = pa.map_( + _align_container_child( + field.type.key_field, target_field.type.key_field + ), + _align_container_child( + field.type.item_field, target_field.type.item_field + ), + keys_sorted=target_field.type.keys_sorted, + ) + else: + new_type = target_field.type else: new_type = target_field.type return pa.field(field.name, new_type, field.nullable, target_field.metadata) diff --git a/python/python/tests/test_blob.py b/python/python/tests/test_blob.py index 298b021df..0d3a89c21 100644 --- a/python/python/tests/test_blob.py +++ b/python/python/tests/test_blob.py @@ -751,6 +751,80 @@ def test_fetch_blobs_preserves_null_and_empty_values(): assert blobs[3].as_py() == b"present" +def test_add_all_null_list_to_blob_column(): + table = _blob_table("all_null_add", [{"id": 1, "image": None}]) + + hits = table.search().to_arrow() + blobs = table.fetch_blobs("image", hits) + assert len(blobs) == 1 + assert blobs[0].as_py() is None + + +def test_add_all_null_list_to_blob_column_with_sanitizer(): + db = lancedb.connect("memory:///") + schema = pa.schema([pa.field("id", pa.int64()), lancedb.blob("image")]) + table = db.create_table("all_null_sanitized_add", schema=schema) + + table.add([{"id": 1, "image": None}], on_bad_vectors="fill") + + hits = table.search().to_arrow() + blobs = table.fetch_blobs("image", hits) + assert len(blobs) == 1 + assert blobs[0].as_py() is None + + +def test_add_all_null_list_to_nested_blob_column(): + db = lancedb.connect("memory:///") + blob_field = lancedb.blob("image") + info_field = pa.field("info", pa.struct([blob_field])) + info = pa.StructArray.from_arrays( + [_blob_array("image", [b"seed"])], fields=[blob_field] + ) + seed = pa.Table.from_arrays( + [pa.array([0], type=pa.int64()), info], + schema=pa.schema([pa.field("id", pa.int64()), info_field]), + ) + table = db.create_table("nested_null_add", data=seed) + + table.add([{"id": 1, "info": {"image": None}}]) + table.add([{"id": 2, "info": {"image": None}}], on_bad_vectors="fill") + + hits = table.search().where("id > 0").to_arrow() + blobs = table.fetch_blobs("info.image", hits) + assert len(blobs) == 2 + assert all(blob.as_py() is None for blob in blobs) + + +@pytest.mark.parametrize("large_list", [False, True], ids=["list", "large_list"]) +def test_add_list_of_dicts_to_blob_list_column(large_list): + db = lancedb.connect("memory:///") + blob_field = lancedb.blob("image") + blob_values = _blob_array("image", [b"seed"]) + if large_list: + items_field = pa.field("items", pa.large_list(blob_field)) + items = pa.LargeListArray.from_arrays( + pa.array([0, 1], type=pa.int64()), blob_values + ) + else: + items_field = pa.field("items", pa.list_(blob_field)) + items = pa.ListArray.from_arrays(pa.array([0, 1], type=pa.int32()), blob_values) + seed = pa.Table.from_arrays( + [pa.array([0], type=pa.int64()), items], + schema=pa.schema([pa.field("id", pa.int64()), items_field]), + ) + table = db.create_table(f"blob_{large_list}_list_add", data=seed) + + table.add([{"id": 1, "items": [None]}]) + table.add( + [{"id": 2, "items": [b"a", None]}], + on_bad_vectors="fill", + ) + + ids = table.search().select(["id"]).to_arrow()["id"].to_pylist() + assert sorted(ids) == [0, 1, 2] + assert pa.types.is_large_list(table.schema.field("items").type) is large_list + + def test_fetch_blob_ranges_aligns_repeated_ranges_and_nulls(): table = _blob_table( "range_alignment", diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index ac7dc660d..a3cbf68f1 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -2979,28 +2979,29 @@ async def test_merge_insert_async(mem_db_async: AsyncConnection): assert (await table.to_arrow()).sort_by("a") == expected +def _json_arrow_table(schema, rows): + json_type = schema.field("j").type + json_values = pa.ExtensionArray.from_storage( + json_type, + pa.array([value for _, value in rows], type=json_type.storage_type), + ) + return pa.Table.from_arrays( + [pa.array([row_id for row_id, _ in rows]), json_values], schema=schema + ) + + @pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") @pytest.mark.asyncio async def test_merge_insert_encodes_json(mem_db_async: AsyncConnection): - json_type = pa.json_() - schema = pa.schema([pa.field("id", pa.string()), pa.field("j", json_type)]) - - def json_table(rows): - json_values = pa.ExtensionArray.from_storage( - json_type, - pa.array([value for _, value in rows], type=json_type.storage_type), - ) - return pa.Table.from_arrays( - [pa.array([row_id for row_id, _ in rows]), json_values], schema=schema - ) + schema = pa.schema([pa.field("id", pa.string()), pa.field("j", pa.json_())]) table = await mem_db_async.create_table("json_merge", schema=schema) - await table.add(json_table([("a", '{"k": 1}'), ("b", '{"k": 9}')])) + await table.add(_json_arrow_table(schema, [("a", '{"k": 1}'), ("b", '{"k": 9}')])) await ( table.merge_insert("id") .when_matched_update_all() - .execute(json_table([("a", '{"k": 2}')])) + .execute(_json_arrow_table(schema, [("a", '{"k": 2}')])) ) rows = sorted(await table.query().to_list(), key=lambda row: row["id"]) @@ -3015,20 +3016,176 @@ async def test_merge_insert_encodes_json(mem_db_async: AsyncConnection): @pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") @pytest.mark.asyncio async def test_add_sanitization_encodes_json(mem_db_async: AsyncConnection): - json_type = pa.json_() - schema = pa.schema([pa.field("id", pa.string()), pa.field("j", json_type)]) - json_values = pa.ExtensionArray.from_storage( - json_type, pa.array(['{"k": 3}'], type=json_type.storage_type) - ) - data = pa.Table.from_arrays([pa.array(["c"]), json_values], schema=schema) + schema = pa.schema([pa.field("id", pa.string()), pa.field("j", pa.json_())]) table = await mem_db_async.create_table("json_add", schema=schema) - await table.add(data, on_bad_vectors="fill") + await table.add( + _json_arrow_table(schema, [("c", '{"k": 3}')]), on_bad_vectors="fill" + ) rows = await table.query().where("json_extract(j, '$.k') = '3'").to_list() assert rows == [{"id": "c", "j": '{"k":3}'}] +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +@pytest.mark.asyncio +async def test_add_all_null_json_batch(mem_db_async: AsyncConnection): + """A batch of dicts whose json values are all None infers as pa.null(), which used + to fail with a `json` vs `large_binary` schema mismatch. A row-at-a-time insert of + an optional json column always looks like this.""" + schema = pa.schema([pa.field("id", pa.string()), pa.field("j", pa.json_())]) + table = await mem_db_async.create_table("json_nulls", schema=schema) + + await table.add([{"id": "a", "j": None}]) + assert await table.count_rows() == 1 + + # ... and again once real JSON has been written. + await table.add(_json_arrow_table(schema, [("b", '{"k": 9}')])) + await table.add([{"id": "c", "j": None}]) + + rows = sorted(await table.query().to_list(), key=lambda row: row["id"]) + assert rows == [ + {"id": "a", "j": None}, + {"id": "b", "j": '{"k":9}'}, + {"id": "c", "j": None}, + ] + + # The nulls must not disturb reads of the column. + filtered = await table.query().where("json_extract(j, '$.k') = '9'").to_list() + assert filtered == [{"id": "b", "j": '{"k":9}'}] + assert len(await table.query().where("j IS NULL").to_list()) == 2 + + +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +def test_add_all_null_json_batch_sync(mem_db: DBConnection): + schema = pa.schema([pa.field("id", pa.string()), pa.field("j", pa.json_())]) + table = mem_db.create_table("json_nulls_sync", schema=schema) + + table.add([{"id": "a", "j": None}]) + + assert table.count_rows() == 1 + assert table.to_arrow()["j"].to_pylist() == [None] + + +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("values", "expected"), + [ + ([None], [None]), + ([None, '{"k": 1}'], [None, '{"k":1}']), + (['{"k": 2}'], ['{"k":2}']), + ], +) +async def test_add_list_of_dicts_to_json_column( + mem_db_async: AsyncConnection, values, expected +): + schema = pa.schema([pa.field("id", pa.int64()), pa.field("value", pa.json_())]) + table = await mem_db_async.create_table("json_list_add", schema=schema) + + await table.add([{"id": idx, "value": value} for idx, value in enumerate(values)]) + + rows = (await table.to_arrow()).sort_by("id").to_pylist() + assert [row["value"] for row in rows] == expected + + +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +@pytest.mark.asyncio +async def test_add_list_of_dicts_to_nested_json_column( + mem_db_async: AsyncConnection, +): + json_field = pa.field("value", pa.json_()) + info_field = pa.field("info", pa.struct([json_field])) + info = pa.StructArray.from_arrays( + [pa.array(['{"seed": 0}'], type=pa.json_())], fields=[json_field] + ) + seed = pa.Table.from_arrays( + [pa.array([0], type=pa.int64()), info], + schema=pa.schema([pa.field("id", pa.int64()), info_field]), + ) + table = await mem_db_async.create_table("nested_json_list_add", data=seed) + + await table.add([{"id": 1, "info": {"value": '{"k": 1}'}}]) + await table.add([{"id": 2, "info": {"value": '{"k": 2}'}}], on_bad_vectors="fill") + + rows = (await table.to_arrow()).sort_by("id").to_pylist() + assert rows == [ + {"id": 0, "info": {"value": '{"seed":0}'}}, + {"id": 1, "info": {"value": '{"k":1}'}}, + {"id": 2, "info": {"value": '{"k":2}'}}, + ] + + +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +@pytest.mark.asyncio +async def test_add_list_of_dicts_to_json_list_column(mem_db_async: AsyncConnection): + """JSON inside a list must be JSONB-encoded, not stored as the raw text. + + Storing raw text appends without error but leaves the column unreadable, so the + round trip is checked with a filter as well as by value. + """ + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("docs", pa.list_(pa.field("item", pa.json_()))), + ] + ) + table = await mem_db_async.create_table("json_list_add", schema=schema) + + await table.add([{"id": 1, "docs": ['{"k": 1}', '{"k": 2}']}]) + await table.add([{"id": 2, "docs": ['{"k": 3}']}], on_bad_vectors="fill") + + rows = (await table.to_arrow()).sort_by("id").to_pylist() + assert rows == [ + {"id": 1, "docs": ['{"k":1}', '{"k":2}']}, + {"id": 2, "docs": ['{"k":3}']}, + ] + + matched = await table.query().where("json_extract(docs[1], '$.k') = 3").to_arrow() + assert matched.column("id").to_pylist() == [2] + + +@pytest.mark.skipif(not hasattr(pa, "json_"), reason="requires PyArrow JSON type") +@pytest.mark.asyncio +async def test_add_map_of_json_values(mem_db_async: AsyncConnection): + """JSON in a map's values needs the same encoding a list's items do.""" + schema = pa.schema( + [ + pa.field("id", pa.int64()), + pa.field("m", pa.map_(pa.string(), pa.json_())), + ] + ) + table = await mem_db_async.create_table( + "json_map_add", + schema=schema, + storage_options={"new_table_data_storage_version": "2.2"}, + ) + + def batch(row_id: int, text: str) -> pa.Table: + return pa.table( + { + "id": pa.array([row_id], type=pa.int64()), + "m": pa.array([[("k", text)]], type=pa.map_(pa.string(), pa.string())), + } + ) + + await table.add(batch(1, '{"x": 1}')) + await table.add(batch(2, '{"x": 2}'), on_bad_vectors="fill") + + rows = (await table.to_arrow()).sort_by("id").to_pylist() + assert rows == [ + {"id": 1, "m": [("k", '{"x":1}')]}, + {"id": 2, "m": [("k", '{"x":2}')]}, + ] + + matched = ( + await table.query() + .where("json_extract(element_at(m, 'k')[1], '$.x') = '2'") + .to_arrow() + ) + assert matched.column("id").to_pylist() == [2] + + def test_create_with_embedding_function(mem_db: DBConnection): class MyTable(LanceModel): text: str diff --git a/rust/lancedb/src/table/add_data.rs b/rust/lancedb/src/table/add_data.rs index 15ccf9f66..0bfec9d68 100644 --- a/rust/lancedb/src/table/add_data.rs +++ b/rust/lancedb/src/table/add_data.rs @@ -1104,4 +1104,220 @@ mod tests { } } } + + /// A batch whose json values are all null infers as `DataType::Null` (this is what + /// pyarrow produces for a one-row insert with no value). The column's lance.json + /// identity lives in the field metadata, so dropping it while casting used to make + /// lance-core reject the batch as a schema mismatch. + #[tokio::test] + async fn test_add_all_null_json_column() { + use arrow_array::{Array, cast::AsArray, new_null_array}; + + let table_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + lance_arrow::json::json_field("data", true), + ])); + + let db = connect("memory://").execute().await.unwrap(); + let table = db + .create_empty_table("json_nulls", table_schema) + .execute() + .await + .unwrap(); + + let null_batch = |ids: Vec| { + let len = ids.len(); + RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("data", DataType::Null, true), + ])), + vec![ + Arc::new(arrow_array::Int64Array::from(ids)), + new_null_array(&DataType::Null, len), + ], + ) + .unwrap() + }; + + // A single all-null row as the very first write, then again after real JSON has + // been written - both scenarios from the bug report. + table.add(null_batch(vec![1])).execute().await.unwrap(); + + let arrow_json_field = Field::new("data", DataType::Utf8, true).with_metadata( + std::collections::HashMap::from([( + lance_arrow::ARROW_EXT_NAME_KEY.to_string(), + lance_arrow::json::ARROW_JSON_EXT_NAME.to_string(), + )]), + ); + let populated = RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + arrow_json_field, + ])), + vec![ + Arc::new(arrow_array::Int64Array::from(vec![2])), + Arc::new(arrow_array::StringArray::from(vec![Some(r#"{"a": 1}"#)])), + ], + ) + .unwrap(); + table.add(populated).execute().await.unwrap(); + table.add(null_batch(vec![3])).execute().await.unwrap(); + + assert_eq!(table.count_rows(None).await.unwrap(), 3); + + let results: Vec = table + .query() + .execute() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let batch = arrow_select::concat::concat_batches(&results[0].schema(), &results).unwrap(); + let ids = batch + .column_by_name("id") + .unwrap() + .as_primitive::(); + let json_strs = batch.column_by_name("data").unwrap().as_string::(); + for row in 0..batch.num_rows() { + match ids.value(row) { + 2 => assert_eq!(json_strs.value(row), r#"{"a":1}"#), + _ => assert!(json_strs.is_null(row), "row {row} expected null"), + } + } + } + + /// JSON text with no arrow.json label - what pyarrow infers for a column of `str` - is + /// encoded as JSONB rather than stored verbatim, at the top level and inside a struct. + #[tokio::test] + async fn test_add_unlabelled_json_strings() { + use arrow_array::{Array, cast::AsArray}; + use arrow_schema::Fields; + + let table_schema = Arc::new(Schema::new(vec![ + lance_arrow::json::json_field("data", true), + Field::new( + "info", + DataType::Struct(vec![lance_arrow::json::json_field("value", true)].into()), + true, + ), + ])); + + let db = connect("memory://").execute().await.unwrap(); + let table = db + .create_empty_table("json_strings", table_schema) + .execute() + .await + .unwrap(); + + let nested_children: Fields = vec![Field::new("value", DataType::Utf8, true)].into(); + let input_schema = Arc::new(Schema::new(vec![ + Field::new("data", DataType::Utf8, true), + Field::new("info", DataType::Struct(nested_children.clone()), true), + ])); + let batch = RecordBatch::try_new( + input_schema, + vec![ + Arc::new(arrow_array::StringArray::from(vec![ + Some(r#"{"a": 1}"#), + None, + ])), + Arc::new(arrow_array::StructArray::new( + nested_children, + vec![Arc::new(arrow_array::StringArray::from(vec![ + Some(r#"{"b": 2}"#), + None, + ]))], + None, + )), + ], + ) + .unwrap(); + table.add(batch).execute().await.unwrap(); + + let results: Vec = table + .query() + .execute() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let batch = arrow_select::concat::concat_batches(&results[0].schema(), &results).unwrap(); + assert_eq!(batch.num_rows(), 2); + + let data = batch.column_by_name("data").unwrap().as_string::(); + assert_eq!(data.value(0), r#"{"a":1}"#); + assert!(data.is_null(1)); + + let nested = batch + .column_by_name("info") + .unwrap() + .as_struct() + .column_by_name("value") + .unwrap() + .as_string::(); + assert_eq!(nested.value(0), r#"{"b":2}"#); + assert!(nested.is_null(1)); + } + + /// A null struct row survives a cast of one of its children, even when that child is + /// non-nullable. Lance checks a non-nullable child for nulls without applying the + /// parent's validity, so rebuilding the struct must leave the children untouched. + #[tokio::test] + async fn test_add_null_struct_with_non_nullable_child() { + use arrow_array::{Array, cast::AsArray}; + use arrow_schema::Fields; + + let table_schema = Arc::new(Schema::new(vec![Field::new( + "s", + DataType::Struct(vec![Field::new("x", DataType::Int64, false)].into()), + true, + )])); + + let db = connect("memory://").execute().await.unwrap(); + let table = db + .create_empty_table("null_struct", table_schema) + .execute() + .await + .unwrap(); + + // Int32 rather than the table's Int64, so the struct goes through reconstruction. + let input_children: Fields = vec![Field::new("x", DataType::Int32, false)].into(); + let input_schema = Arc::new(Schema::new(vec![Field::new( + "s", + DataType::Struct(input_children.clone()), + true, + )])); + let batch = RecordBatch::try_new( + input_schema, + vec![Arc::new(arrow_array::StructArray::new( + input_children, + vec![Arc::new(arrow_array::Int32Array::from(vec![0, 6]))], + Some(arrow::buffer::NullBuffer::from(vec![false, true])), + ))], + ) + .unwrap(); + table.add(batch).execute().await.unwrap(); + + let results: Vec = table + .query() + .execute() + .await + .unwrap() + .try_collect() + .await + .unwrap(); + let batch = arrow_select::concat::concat_batches(&results[0].schema(), &results).unwrap(); + let s = batch.column_by_name("s").unwrap().as_struct(); + assert!(s.is_null(0)); + assert_eq!( + s.column_by_name("x") + .unwrap() + .as_primitive::() + .value(1), + 6 + ); + } } diff --git a/rust/lancedb/src/table/datafusion/cast.rs b/rust/lancedb/src/table/datafusion/cast.rs index bc47caeb2..0df1cb4ba 100644 --- a/rust/lancedb/src/table/datafusion/cast.rs +++ b/rust/lancedb/src/table/datafusion/cast.rs @@ -1,20 +1,25 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors -use std::sync::Arc; +use std::collections::HashMap; +use std::sync::{Arc, LazyLock}; +use arrow_array::StructArray; +use arrow_array::cast::AsArray; use arrow_cast::can_cast_types; use arrow_schema::{DataType, Field, FieldRef, Fields, Schema}; use datafusion::functions::core::{get_field, named_struct}; -use datafusion_common::ScalarValue; use datafusion_common::config::ConfigOptions; +use datafusion_common::metadata::FieldMetadata; +use datafusion_common::{DataFusionError, Result as DFResult, ScalarValue}; +use datafusion_expr::{ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; use datafusion_physical_expr::ScalarFunctionExpr; use datafusion_physical_expr::expressions::{CastExpr, Literal}; use datafusion_physical_plan::expressions::Column; use datafusion_physical_plan::projection::ProjectionExec; use datafusion_physical_plan::{ExecutionPlan, PhysicalExpr}; -use lance_arrow::FieldExt; -use lance_arrow::json::{is_arrow_json_field, is_json_field}; +use lance_arrow::json::{ARROW_JSON_EXT_NAME, has_json_fields, is_arrow_json_field, is_json_field}; +use lance_arrow::{ARROW_EXT_NAME_KEY, FieldExt}; use super::blob_coerce::coerce_blob_expr; use crate::{Error, Result}; @@ -67,18 +72,33 @@ fn build_field_exprs( let input_field = &input_fields[input_idx]; let input_expr = get_input_expr(input_idx); - // Special case: input is arrow.json (PyArrow pa.json_() extension type backed by - // Utf8/LargeUtf8) and the table field is lance.json (backed by LargeBinary). - // Lance-core's write path already handles the arrow.json → lance.json conversion - // (including JSONB encoding), so we pass the expression through unchanged and let - // lance-core deal with it. Attempting to cast Utf8 → LargeBinary here would - // produce a field whose metadata still identifies it as arrow.json, which then - // causes a schema-mismatch error inside lance-core. - if is_arrow_json_field(input_field) && is_json_field(table_field) { + // PyArrow's pa.json_() is already labelled arrow.json, which is what lance-core wants + // to see, so pass it straight through. + if is_json_field(table_field) && is_arrow_json_field(input_field) { result.push((input_expr, Arc::clone(input_field) as FieldRef)); continue; } + // Anything else destined for a json column needs its JSON leaves labelled; see + // `json_write_target`. Structs are excluded because the recursion below rebuilds them + // field by field, which also handles reordered and partial input. + if !matches!(table_field.data_type(), DataType::Struct(_)) + && let Some(target) = json_write_target(input_field, table_field) + && can_cast_types(input_field.data_type(), target.data_type()) + { + // The label goes on the cast's target field rather than the field returned + // alongside it, because DataFusion derives the projection's output schema from + // `PhysicalExpr::return_field`. + let target: FieldRef = Arc::new(target); + let expr = Arc::new(CastExpr::new_with_target_field( + input_expr, + target.clone(), + None, + )); + result.push((expr, target)); + continue; + } + // Blob columns accept raw binary on write; exact matches pass through below. if table_field.is_blob_v2() && input_field.as_ref() != table_field.as_ref() { result.push(coerce_blob_expr( @@ -90,6 +110,18 @@ fn build_field_exprs( continue; } + // A column whose values are all null infers as `Null` (pyarrow does this for a list of + // dicts), so there is no input type to cast from. Emit typed nulls carrying the table + // field verbatim: a plain cast would drop the field metadata, and extension columns + // such as lance.json are identified by that metadata alone, so lance-core would then + // reject the batch as a schema mismatch. + if matches!(input_field.data_type(), DataType::Null) + && !matches!(table_field.data_type(), DataType::Null) + { + result.push((null_literal(table_field)?, table_field.clone())); + continue; + } + let expr = match (input_field.data_type(), table_field.data_type()) { // Both are structs: recurse into sub-fields to handle subschemas and casts. (DataType::Struct(in_children), DataType::Struct(tbl_children)) @@ -137,7 +169,16 @@ fn build_field_exprs( config.clone(), )); - result.push((ns_expr, output_field)); + result.push(( + restore_struct_validity( + ns_expr, + input_expr, + input_field, + &output_field, + config.clone(), + ), + output_field, + )); continue; } // Types match: pass through. @@ -171,6 +212,193 @@ fn build_field_exprs( Ok(result) } +// `named_struct` returns a struct with no null bitmap, and the `get_field` calls feeding it +// read each child without applying the parent's validity, so a null input struct would come +// back non-null with its masked children exposed. Move the input's null buffer onto the +// rebuilt struct. +fn restore_struct_validity( + rebuilt: Arc, + input_expr: Arc, + input_field: &FieldRef, + output_field: &FieldRef, + config: Arc, +) -> Arc { + if !input_field.is_nullable() { + return rebuilt; + } + + Arc::new(ScalarFunctionExpr::new( + &format!("restore_validity({})", output_field.name()), + RESTORE_VALIDITY_UDF.clone(), + vec![rebuilt, input_expr], + output_field.clone(), + config, + )) +} + +static RESTORE_VALIDITY_UDF: LazyLock> = + LazyLock::new(|| Arc::new(datafusion_expr::ScalarUDF::from(RestoreValidityUdf::new()))); + +/// Returns its first argument, a struct, carrying the null buffer of its second. +/// +/// Selecting a typed null for the null rows instead would nullify their children too, which +/// Lance rejects outright for a non-nullable child, even where the parent masks it. Children +/// therefore have to survive the round trip byte for byte. +#[derive(Debug, Hash, PartialEq, Eq)] +struct RestoreValidityUdf { + signature: Signature, +} + +impl RestoreValidityUdf { + fn new() -> Self { + Self { + signature: Signature::any(2, Volatility::Immutable), + } + } +} + +impl ScalarUDFImpl for RestoreValidityUdf { + fn name(&self) -> &str { + "restore_validity" + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, arg_types: &[DataType]) -> DFResult { + Ok(arg_types[0].clone()) + } + + fn invoke_with_args(&self, args: ScalarFunctionArgs) -> DFResult { + let rows = args.number_rows; + let rebuilt = args.args[0].to_array(rows)?; + let nulls_from = args.args[1].to_array(rows)?; + + let rebuilt = rebuilt.as_struct_opt().ok_or_else(|| { + DataFusionError::Internal(format!( + "restore_validity expects a struct, got {}", + rebuilt.data_type() + )) + })?; + + let nulls = nulls_from.logical_nulls(); + let (fields, columns, _) = rebuilt.clone().into_parts(); + let restored = StructArray::try_new_with_length(fields, columns, nulls, rows)?; + Ok(ColumnarValue::Array(Arc::new(restored))) + } +} + +// The storage type arrow.json would use for `input`, or None if it cannot hold JSON text. +fn arrow_json_storage_type(input: &DataType) -> Option { + match input { + // arrow.json only recognises Utf8 and LargeUtf8 storage, so a view has to be cast. + DataType::Utf8 | DataType::Utf8View => Some(DataType::Utf8), + DataType::LargeUtf8 => Some(DataType::LargeUtf8), + _ => None, + } +} + +fn arrow_json_field(name: &str, storage: DataType, nullable: bool) -> Field { + Field::new(name, storage, nullable).with_metadata(HashMap::from([( + ARROW_EXT_NAME_KEY.to_string(), + ARROW_JSON_EXT_NAME.to_string(), + )])) +} + +/// Rewrite `table_field` so that every lance.json leaf the input supplies as text becomes an +/// arrow.json leaf, leaving the rest of the shape untouched. +/// +/// Lance-core encodes JSON text into JSONB on write, but only for leaves labelled arrow.json. +/// Casting to the lance.json storage type instead relabels raw text as JSONB, which appends +/// successfully but leaves the column unreadable, so the label has to reach every leaf however +/// deeply it is nested. Returns `None` when there is nothing to relabel. +fn json_write_target(input_field: &Field, table_field: &Field) -> Option { + if is_json_field(table_field) { + let storage = if is_arrow_json_field(input_field) { + input_field.data_type().clone() + } else { + arrow_json_storage_type(input_field.data_type())? + }; + return Some(arrow_json_field( + table_field.name(), + storage, + table_field.is_nullable(), + )); + } + + if !has_json_fields(table_field) { + return None; + } + + let relabelled = match (input_field.data_type(), table_field.data_type()) { + ( + DataType::List(input_item) + | DataType::LargeList(input_item) + | DataType::FixedSizeList(input_item, _), + DataType::List(table_item) + | DataType::LargeList(table_item) + | DataType::FixedSizeList(table_item, _), + ) => { + let item: FieldRef = Arc::new(json_write_target(input_item, table_item)?); + match table_field.data_type() { + DataType::List(_) => DataType::List(item), + DataType::LargeList(_) => DataType::LargeList(item), + DataType::FixedSizeList(_, len) => DataType::FixedSizeList(item, *len), + _ => unreachable!("matched a list type above"), + } + } + (DataType::Map(input_entries, _), DataType::Map(table_entries, sorted)) => { + let entries = json_write_target(input_entries, table_entries)?; + DataType::Map(Arc::new(entries), *sorted) + } + (DataType::Struct(input_children), DataType::Struct(table_children)) => { + let mut children = Vec::with_capacity(table_children.len()); + let mut relabelled_any = false; + for table_child in table_children { + let relabelled_child = input_children + .iter() + .find(|f| f.name() == table_child.name()) + .and_then(|input_child| json_write_target(input_child, table_child)); + match relabelled_child { + Some(child) => { + relabelled_any = true; + children.push(Arc::new(child)); + } + None => children.push(table_child.clone()), + } + } + if !relabelled_any { + return None; + } + DataType::Struct(children.into()) + } + _ => return None, + }; + + Some( + Field::new(table_field.name(), relabelled, table_field.is_nullable()) + .with_metadata(table_field.metadata().clone()), + ) +} + +// The field's metadata is attached to the literal itself, because DataFusion derives the +// projection's output schema from `PhysicalExpr::return_field` rather than from the field we +// return alongside the expression. +fn null_literal(field: &FieldRef) -> Result> { + let scalar = ScalarValue::try_new_null(field.data_type()).map_err(|e| Error::InvalidInput { + message: format!( + "cannot build null literal for column '{}' of type {}: {e}", + field.name(), + field.data_type() + ), + })?; + Ok(Arc::new(Literal::new_with_metadata( + scalar, + Some(FieldMetadata::from(field.as_ref())), + ))) +} + #[cfg(test)] mod tests { use std::sync::Arc; @@ -715,4 +943,379 @@ mod tests { assert!(result.column(0).is_null(1)); assert_eq!(v2, r#"{"y": 2}"#); } + + /// Plain JSON text (what pyarrow infers for a column of `str`, and what a caller writing + /// JSON by hand supplies) has to be labelled arrow.json so lance-core encodes it as JSONB. + /// Casting it to the table field's LargeBinary storage type would store the raw text. + #[rstest::rstest] + #[case::utf8(DataType::Utf8, DataType::Utf8)] + #[case::large_utf8(DataType::LargeUtf8, DataType::LargeUtf8)] + #[case::utf8_view(DataType::Utf8View, DataType::Utf8)] + #[tokio::test] + async fn test_unlabelled_string_into_lance_json_gets_arrow_json_label( + #[case] input_type: DataType, + #[case] expected_type: DataType, + ) { + use lance_arrow::json::{is_arrow_json_field, json_field}; + + let table_schema = Schema::new(vec![json_field("data", true)]); + + let input_schema = Arc::new(Schema::new(vec![Field::new("data", input_type, true)])); + let values = vec![Some(r#"{"x": 1}"#), None]; + let input_array = arrow_cast::cast( + &StringArray::from(values) as &dyn arrow_array::Array, + input_schema.field(0).data_type(), + ) + .unwrap(); + let input_batch = RecordBatch::try_new(input_schema, vec![input_array]).unwrap(); + + let plan = plan_from_batch(input_batch).await; + let projected = cast_to_table_schema(plan, &table_schema).unwrap(); + + let out_field = projected.schema().field_with_name("data").unwrap().clone(); + assert_eq!(out_field.data_type(), &expected_type); + assert!( + is_arrow_json_field(&out_field), + "output field must be labelled arrow.json, got {:?}", + out_field.metadata() + ); + + let result = collect(projected).await; + assert_eq!(result.num_rows(), 2); + assert_eq!(result.column(0).null_count(), 1); + } + + /// A json leaf inside a list is relabelled too. The outer field is a list, so without + /// recursing into it the generic container cast would turn the text into LargeBinary + /// labelled lance.json - an append that succeeds but stores unreadable JSON. + #[rstest::rstest] + #[case::unlabelled(DataType::Utf8, false)] + #[case::already_labelled(DataType::Utf8, true)] + #[tokio::test] + async fn test_unlabelled_list_item_into_lance_json_gets_arrow_json_label( + #[case] item_type: DataType, + #[case] input_labelled: bool, + ) { + use lance_arrow::json::{is_arrow_json_field, json_field}; + + use super::arrow_json_field; + + let table_schema = Schema::new(vec![Field::new( + "docs", + DataType::List(Arc::new(json_field("item", true))), + true, + )]); + + let input_item = if input_labelled { + Arc::new(arrow_json_field("item", item_type, true)) + } else { + Arc::new(Field::new("item", item_type, true)) + }; + let input_schema = Arc::new(Schema::new(vec![Field::new( + "docs", + DataType::List(input_item.clone()), + true, + )])); + let values = StringArray::from(vec![Some(r#"{"k": 1}"#), Some(r#"{"k": 2}"#)]); + let input_batch = RecordBatch::try_new( + input_schema, + vec![Arc::new(ListArray::new( + input_item, + OffsetBuffer::new(vec![0, 1, 2].into()), + Arc::new(values), + None, + ))], + ) + .unwrap(); + + let plan = plan_from_batch(input_batch).await; + let projected = cast_to_table_schema(plan, &table_schema).unwrap(); + + let out_field = projected.schema().field_with_name("docs").unwrap().clone(); + let DataType::List(out_item) = out_field.data_type() else { + panic!("expected a list, got {}", out_field.data_type()); + }; + assert!( + is_arrow_json_field(out_item), + "the list item must be labelled arrow.json, got {out_item:?}" + ); + + let result = collect(projected).await; + assert_eq!(result.num_rows(), 2); + } + + /// The same, for a json column nested inside a struct: the struct is rebuilt from its + /// children, so the label has to travel on the child field. + #[tokio::test] + async fn test_unlabelled_struct_child_into_lance_json_gets_arrow_json_label() { + use lance_arrow::json::{is_arrow_json_field, json_field}; + + let table_schema = Schema::new(vec![Field::new( + "info", + DataType::Struct( + vec![ + Field::new("id", DataType::Int64, true), + json_field("value", true), + ] + .into(), + ), + true, + )]); + + let input_children: Fields = vec![ + Field::new("id", DataType::Int64, true), + Field::new("value", DataType::Utf8, true), + ] + .into(); + let input_schema = Arc::new(Schema::new(vec![Field::new( + "info", + DataType::Struct(input_children.clone()), + true, + )])); + let input_batch = RecordBatch::try_new( + input_schema, + vec![Arc::new(StructArray::new( + input_children, + vec![ + Arc::new(Int64Array::from(vec![1, 2])), + Arc::new(StringArray::from(vec![Some(r#"{"a": 1}"#), None])), + ], + None, + ))], + ) + .unwrap(); + + let plan = plan_from_batch(input_batch).await; + let projected = cast_to_table_schema(plan, &table_schema).unwrap(); + + let out_field = projected.schema().field_with_name("info").unwrap().clone(); + let DataType::Struct(out_children) = out_field.data_type() else { + panic!("expected a struct, got {}", out_field.data_type()); + }; + let value = out_children.iter().find(|f| f.name() == "value").unwrap(); + assert!( + is_arrow_json_field(value), + "nested field must be labelled arrow.json, got {:?}", + value.metadata() + ); + + let result = collect(projected).await; + let info: &StructArray = result.column(0).as_any().downcast_ref().unwrap(); + assert_eq!(info.column_by_name("value").unwrap().null_count(), 1); + } + + /// An all-null column comes through as `DataType::Null` (pyarrow infers that for a batch + /// of dicts whose values are all `None`). The lance.json extension metadata lives on the + /// field alone, so it has to be carried into the output schema or lance-core rejects the + /// batch with a "json vs large_binary" schema mismatch. + #[tokio::test] + async fn test_null_column_into_lance_json_keeps_extension_metadata() { + use lance_arrow::ARROW_EXT_NAME_KEY; + use lance_arrow::json::{JSON_EXT_NAME, json_field}; + + let table_schema = Schema::new(vec![ + Field::new("id", DataType::Int64, false), + json_field("data", true), + ]); + + let input_schema = Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("data", DataType::Null, true), + ])); + let input_batch = RecordBatch::try_new( + input_schema, + vec![ + Arc::new(Int64Array::from(vec![0, 1, 2])), + arrow_array::new_null_array(&DataType::Null, 3), + ], + ) + .unwrap(); + + let plan = plan_from_batch(input_batch).await; + let projected = cast_to_table_schema(plan, &table_schema).unwrap(); + + let out_field = projected.schema().field_with_name("data").unwrap().clone(); + assert_eq!(out_field.data_type(), &DataType::LargeBinary); + assert_eq!( + out_field + .metadata() + .get(ARROW_EXT_NAME_KEY) + .map(|s| s.as_str()), + Some(JSON_EXT_NAME), + "output field must still identify itself as lance.json" + ); + + let result = collect(projected).await; + assert_eq!(result.num_rows(), 3); + assert_eq!(result.column_by_name("data").unwrap().null_count(), 3); + } + + /// The same, for a lance.json column nested inside a struct: the struct is rebuilt from + /// its children, so each child field must keep its own metadata, and a null struct must + /// stay null even though it is rebuilt child by child. + #[tokio::test] + async fn test_null_struct_child_into_lance_json_keeps_extension_metadata() { + use lance_arrow::ARROW_EXT_NAME_KEY; + use lance_arrow::json::{JSON_EXT_NAME, json_field}; + + let table_schema = Schema::new(vec![Field::new( + "meta", + DataType::Struct( + vec![ + Field::new("id", DataType::Int64, true), + json_field("doc", true), + ] + .into(), + ), + true, + )]); + + let input_children: Fields = vec![ + Field::new("id", DataType::Int64, true), + Field::new("doc", DataType::Null, true), + ] + .into(); + let input_schema = Arc::new(Schema::new(vec![Field::new( + "meta", + DataType::Struct(input_children.clone()), + true, + )])); + let input_batch = RecordBatch::try_new( + input_schema, + vec![Arc::new(StructArray::new( + input_children, + vec![ + Arc::new(Int64Array::from(vec![7, 8])), + arrow_array::new_null_array(&DataType::Null, 2), + ], + Some(arrow::buffer::NullBuffer::from(vec![false, true])), + ))], + ) + .unwrap(); + + let plan = plan_from_batch(input_batch).await; + let projected = cast_to_table_schema(plan, &table_schema).unwrap(); + + let out_field = projected.schema().field_with_name("meta").unwrap().clone(); + let DataType::Struct(out_children) = out_field.data_type() else { + panic!("expected a struct, got {}", out_field.data_type()); + }; + let doc = out_children.iter().find(|f| f.name() == "doc").unwrap(); + assert_eq!(doc.data_type(), &DataType::LargeBinary); + assert_eq!( + doc.metadata().get(ARROW_EXT_NAME_KEY).map(|s| s.as_str()), + Some(JSON_EXT_NAME) + ); + + let result = collect(projected).await; + let meta: &StructArray = result.column(0).as_any().downcast_ref().unwrap(); + assert!(meta.is_null(0), "a null struct must stay null once rebuilt"); + assert!(meta.is_valid(1)); + assert_eq!(meta.column_by_name("doc").unwrap().null_count(), 2); + } + + /// Any struct whose children need adjusting is rebuilt child by child, so the parent's + /// nulls have to be restored afterwards - not only for the extension-column cases. + #[tokio::test] + async fn test_null_struct_stays_null_when_child_is_cast() { + let input_children: Fields = vec![Field::new("x", DataType::Int32, true)].into(); + let table_schema = Schema::new(vec![Field::new( + "s", + DataType::Struct(vec![Field::new("x", DataType::Int64, true)].into()), + true, + )]); + + let input_schema = Arc::new(Schema::new(vec![Field::new( + "s", + DataType::Struct(input_children.clone()), + true, + )])); + let input_batch = RecordBatch::try_new( + input_schema, + vec![Arc::new(StructArray::new( + input_children, + vec![Arc::new(Int32Array::from(vec![5, 6]))], + Some(arrow::buffer::NullBuffer::from(vec![false, true])), + ))], + ) + .unwrap(); + + let plan = plan_from_batch(input_batch).await; + let projected = cast_to_table_schema(plan, &table_schema).unwrap(); + + let result = collect(projected).await; + let s: &StructArray = result.column(0).as_any().downcast_ref().unwrap(); + assert!(s.is_null(0)); + assert!(s.is_valid(1)); + let x: &Int64Array = s.column(0).as_any().downcast_ref().unwrap(); + assert_eq!(x.value(1), 6); + } + + /// Lance rejects a non-nullable child that carries nulls even where the parent masks + /// them, so the null rows have to keep the placeholder children the input gave them. + #[tokio::test] + async fn test_null_struct_keeps_children_of_a_non_nullable_child() { + let input_children: Fields = vec![Field::new("x", DataType::Int32, false)].into(); + let table_schema = Schema::new(vec![Field::new( + "s", + DataType::Struct(vec![Field::new("x", DataType::Int64, false)].into()), + true, + )]); + + let input_schema = Arc::new(Schema::new(vec![Field::new( + "s", + DataType::Struct(input_children.clone()), + true, + )])); + let input_batch = RecordBatch::try_new( + input_schema, + vec![Arc::new(StructArray::new( + input_children, + vec![Arc::new(Int32Array::from(vec![0, 6]))], + Some(arrow::buffer::NullBuffer::from(vec![false, true])), + ))], + ) + .unwrap(); + + let plan = plan_from_batch(input_batch).await; + let projected = cast_to_table_schema(plan, &table_schema).unwrap(); + + let result = collect(projected).await; + let s: &StructArray = result.column(0).as_any().downcast_ref().unwrap(); + assert!(s.is_null(0)); + assert!(s.is_valid(1)); + let x: &Int64Array = s.column(0).as_any().downcast_ref().unwrap(); + assert_eq!(x.null_count(), 0); + assert_eq!(x.value(1), 6); + } + + /// A `Null` input column against a plain table column writes nulls too, including for + /// target types that a DataFusion cast would not reach. + #[tokio::test] + async fn test_null_column_into_struct_column() { + let children: Fields = vec![Field::new("x", DataType::Int32, true)].into(); + let table_schema = Schema::new(vec![Field::new( + "s", + DataType::Struct(children.clone()), + true, + )]); + + let input_schema = Arc::new(Schema::new(vec![Field::new("s", DataType::Null, true)])); + let input_batch = RecordBatch::try_new( + input_schema, + vec![arrow_array::new_null_array(&DataType::Null, 2)], + ) + .unwrap(); + + let plan = plan_from_batch(input_batch).await; + let projected = cast_to_table_schema(plan, &table_schema).unwrap(); + assert_eq!( + projected.schema().field_with_name("s").unwrap().data_type(), + &DataType::Struct(children) + ); + + let result = collect(projected).await; + assert_eq!(result.num_rows(), 2); + assert_eq!(result.column(0).null_count(), 2); + } } diff --git a/rust/lancedb/tests/blob_integration.rs b/rust/lancedb/tests/blob_integration.rs index 118ecf5c9..dd4051657 100644 --- a/rust/lancedb/tests/blob_integration.rs +++ b/rust/lancedb/tests/blob_integration.rs @@ -47,6 +47,21 @@ fn binary_input_batch(ids: &[i64], payloads: &[Option<&[u8]>]) -> RecordBatch { .unwrap() } +/// What pyarrow infers for a batch of dicts whose blob values are all `None`. +fn null_typed_input_batch(ids: &[i64]) -> RecordBatch { + RecordBatch::try_new( + Arc::new(Schema::new(vec![ + Field::new("id", DataType::Int64, false), + Field::new("image", DataType::Null, true), + ])), + vec![ + Arc::new(Int64Array::from(ids.to_vec())), + new_null_array(&DataType::Null, ids.len()), + ], + ) + .unwrap() +} + async fn create_inline_blob_table( db: &Connection, name: &str, @@ -255,6 +270,38 @@ async fn add_accepts_null_blob_rows() -> Result<()> { Ok(()) } +/// A batch whose blob values are all null carries no type information — pyarrow infers +/// `DataType::Null` for it, which is what a row-at-a-time insert of an optional blob column +/// looks like. Such a batch must be accepted, both as the first write and after bytes have +/// already been written. +#[tokio::test] +async fn add_accepts_all_null_typed_blob_column() -> Result<()> { + let tmp = tempdir().unwrap(); + let db = connect(tmp.path().to_str().unwrap()).execute().await?; + let table = db + .create_empty_table("t", blob_table_schema()) + .execute() + .await?; + + table.add(null_typed_input_batch(&[1])).execute().await?; + assert_eq!(table.count_rows(None).await?, 1); + assert!(query_image_struct(&table).await.is_null(0)); + + table + .add(binary_input_batch(&[2], &[Some(b"bytes".as_slice())])) + .execute() + .await?; + table.add(null_typed_input_batch(&[3])).execute().await?; + assert_eq!(table.count_rows(None).await?, 3); + + let row_ids = collect_row_ids(&table).await?; + let bytes = table.fetch_blobs("image", &row_ids).await?; + assert!(bytes.is_null(0)); + assert_eq!(bytes.value(1), b"bytes"); + assert!(bytes.is_null(2)); + Ok(()) +} + #[tokio::test] async fn add_rejects_uncoercible_blob_input() -> Result<()> { let tmp = tempdir().unwrap(); From ba693ae43d2e6afc8e6c325849105b7ba3bd6eac Mon Sep 17 00:00:00 2001 From: Colin Patrick McCabe Date: Tue, 15 Sep 2026 15:49:57 -0700 Subject: [PATCH 18/43] fix: do not allow . and .. as table names (#4191) Do not allow . and .. as table names. They are incompatible with the local filesystem, and confusing in cases where they are supported. --- rust/lancedb/src/utils/mod.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/rust/lancedb/src/utils/mod.rs b/rust/lancedb/src/utils/mod.rs index 352e55f2f..1bfd2d042 100644 --- a/rust/lancedb/src/utils/mod.rs +++ b/rust/lancedb/src/utils/mod.rs @@ -89,6 +89,18 @@ pub fn validate_table_name(name: &str) -> Result<()> { reason: "Table names cannot be empty strings".to_string(), }); } + if name == "." { + return Err(Error::InvalidTableName { + name: name.to_string(), + reason: "Table name cannot be a single dot.".to_string(), + }); + } + if name == ".." { + return Err(Error::InvalidTableName { + name: name.to_string(), + reason: "Table name cannot be two dots.".to_string(), + }); + } if !TABLE_NAME_REGEX.is_match(name) { return Err(Error::InvalidTableName { name: name.to_string(), @@ -804,8 +816,11 @@ mod tests { assert!(validate_table_name("_12345table").is_ok()); assert!(validate_table_name("table.12345").is_ok()); assert!(validate_table_name("table.._dot_..12345").is_ok()); + assert!(validate_table_name("...").is_ok()); assert!(validate_table_name("").is_err()); + assert!(validate_table_name(".").is_err()); + assert!(validate_table_name("..").is_err()); assert!(validate_table_name("my_table!").is_err()); assert!(validate_table_name("my/table").is_err()); assert!(validate_table_name("my@table").is_err()); From 3a1d3be256803642f66fad084078704d3faca38d Mon Sep 17 00:00:00 2001 From: Colin Patrick McCabe Date: Tue, 15 Sep 2026 16:35:37 -0700 Subject: [PATCH 19/43] feat(oidc): support resource and audience (#4193) Support configuring resource and audience for OAuth authorization, token exchange, and refresh requests. --- docs/src/js/interfaces/NativeOAuthConfig.md | 20 ++ docs/src/js/interfaces/OAuthConfig.md | 35 +++ docs/src/js/interfaces/SessionStatus.md | 20 ++ docs/src/js/interfaces/TokenCacheOptions.md | 2 +- nodejs/__test__/oauth.test.ts | 81 ++++-- nodejs/lancedb/oauth.ts | 33 ++- nodejs/src/remote.rs | 24 ++ python/python/lancedb/_lancedb.pyi | 4 + python/python/lancedb/remote/oauth.py | 15 ++ python/src/oauth.rs | 26 ++ python/tests/test_oauth.py | 44 ++- rust/lancedb/src/connection.rs | 4 + rust/lancedb/src/remote/oauth.rs | 280 +++++++++++++++++++- rust/lancedb/src/remote/token_cache.rs | 141 +++++++++- 14 files changed, 683 insertions(+), 46 deletions(-) diff --git a/docs/src/js/interfaces/NativeOAuthConfig.md b/docs/src/js/interfaces/NativeOAuthConfig.md index afe05de9f..074131808 100644 --- a/docs/src/js/interfaces/NativeOAuthConfig.md +++ b/docs/src/js/interfaces/NativeOAuthConfig.md @@ -15,6 +15,16 @@ All token acquisition and refresh is handled in the Rust layer. ## Properties +### audience? + +```ts +optional audience: string; +``` + +Optional provider-specific audience for authorization and token requests. + +*** + ### callbackPort? ```ts @@ -99,6 +109,16 @@ the TTL, each request refreshes the token. *** +### resource? + +```ts +optional resource: string; +``` + +Optional resource indicator for authorization and token requests. + +*** + ### scopes ```ts diff --git a/docs/src/js/interfaces/OAuthConfig.md b/docs/src/js/interfaces/OAuthConfig.md index f7d61c663..0615fb8c5 100644 --- a/docs/src/js/interfaces/OAuthConfig.md +++ b/docs/src/js/interfaces/OAuthConfig.md @@ -26,6 +26,18 @@ const config: OAuthConfig = { }; ``` +Providers requiring an explicit target can set `resource` and/or `audience`: +```typescript +const targeted: OAuthConfig = { + issuerUrl: "https://issuer.example.com", + clientId: "app-id", + clientSecret: "secret", + scopes: ["read"], + resource: "https://api.example.com", + audience: "lancedb-api", +}; +``` + ```typescript const config: OAuthConfig = { issuerUrl: "https://login.microsoftonline.com/{tenant}/v2.0", @@ -51,6 +63,17 @@ before polling begins. ## Properties +### audience? + +```ts +optional audience: string; +``` + +Provider-specific audience, forwarded to authorization and token endpoints, +including refresh requests. Not supported for Azure managed identity. + +*** + ### callbackPort? ```ts @@ -134,6 +157,18 @@ the TTL, each request refreshes the token. *** +### resource? + +```ts +optional resource: string; +``` + +Resource indicator (RFC 8707), forwarded verbatim to authorization and token +endpoints, including refresh requests. Must be an absolute URI without a +fragment. Not supported for Azure managed identity. + +*** + ### scopes ```ts diff --git a/docs/src/js/interfaces/SessionStatus.md b/docs/src/js/interfaces/SessionStatus.md index 3844bfae6..260233d6a 100644 --- a/docs/src/js/interfaces/SessionStatus.md +++ b/docs/src/js/interfaces/SessionStatus.md @@ -11,6 +11,16 @@ Safe, non-secret view of a cached OAuth session, returned by ## Properties +### audience? + +```ts +optional audience: string; +``` + +Provider-specific audience used to obtain the cached session, if configured. + +*** + ### clientId ```ts @@ -65,6 +75,16 @@ prompt. *** +### resource? + +```ts +optional resource: string; +``` + +Resource indicator used to obtain the cached session, if configured. + +*** + ### scopes ```ts diff --git a/docs/src/js/interfaces/TokenCacheOptions.md b/docs/src/js/interfaces/TokenCacheOptions.md index 95b96ef2f..d96337468 100644 --- a/docs/src/js/interfaces/TokenCacheOptions.md +++ b/docs/src/js/interfaces/TokenCacheOptions.md @@ -13,7 +13,7 @@ The cache is opt-in: it is only used when set as `tokenCache` on directory with owner-only permissions, so short-lived processes can reuse an authenticated session instead of re-prompting on every start. -Multiple identities (issuer, client, scopes, flow, client authentication) +Multiple identities (issuer, client, scopes, resource, audience, flow, client authentication) get separate cache entries. Within one identity the most recent login wins. ## Properties diff --git a/nodejs/__test__/oauth.test.ts b/nodejs/__test__/oauth.test.ts index 83ea2791b..0a04bb2aa 100644 --- a/nodejs/__test__/oauth.test.ts +++ b/nodejs/__test__/oauth.test.ts @@ -67,41 +67,64 @@ describe("OAuthSession", () => { expect(() => new OAuthSession(config)).toThrow(/AzureManagedIdentity/); }); - it("logs in via device flow, caches, and logs out", async () => { - const server = new MockIdp(); - await server.start(); - try { - const cacheDir = tempCacheDir(); - const issuerUrl = server.issuerUrl(); + it.each([ + {}, + { + resource: "https://api.example.com/a?x=1&y=two", + audience: "audience + & / ü", + }, + ])( + "logs in via device flow with target %j, caches, and logs out", + async (target) => { + const server = new MockIdp(); + await server.start(); + try { + const cacheDir = tempCacheDir(); + const issuerUrl = server.issuerUrl(); - const session = new OAuthSession(deviceConfig(issuerUrl, cacheDir)); - const status = await session.login(); - expect(status.refreshable).toBe(true); - expect(status.obtainedAt).toBeGreaterThan(0); - expect(server.state.deviceAuthorizations).toBe(1); + const config = { ...deviceConfig(issuerUrl, cacheDir), ...target }; + const session = new OAuthSession(config); + const status = await session.login(); + expect(status.refreshable).toBe(true); + expect(status.resource).toBe(config.resource); + expect(status.audience).toBe(config.audience); + expect(status.obtainedAt).toBeGreaterThan(0); + expect(server.state.deviceAuthorizations).toBe(1); - // An independent session (a fresh "process") sees the cached login. - const other = new OAuthSession(deviceConfig(issuerUrl, cacheDir)); - const cached = await other.status(); - expect(cached.refreshable).toBe(true); + // An independent session (a fresh "process") sees the cached login. + const other = new OAuthSession(config); + const cached = await other.status(); + expect(cached.refreshable).toBe(true); - const logout = await other.logout(); - expect(logout.removed).toBe(true); - const again = await session.logout(); - expect(again.removed).toBe(false); - expect((await session.status()).refreshable).toBe(false); + const logout = await other.logout(); + expect(logout.removed).toBe(true); + const again = await session.logout(); + expect(again.removed).toBe(false); + expect((await session.status()).refreshable).toBe(false); - // Only the initial login used the interactive device flow. - expect(server.state.deviceAuthorizations).toBe(1); - expect(server.state.refreshGrants).toBe(0); - } finally { - server.close(); - } - }, 15000); + // Only the initial login used the interactive device flow. + expect(server.state.deviceAuthorizations).toBe(1); + expect(server.state.refreshGrants).toBe(0); + expect(server.requests).toHaveLength(2); + for (const params of server.requests) { + expect(params.getAll("resource")).toEqual( + config.resource === undefined ? [] : [config.resource], + ); + expect(params.getAll("audience")).toEqual( + config.audience === undefined ? [] : [config.audience], + ); + } + } finally { + server.close(); + } + }, + 15000, + ); }); /** Mock IdP with discovery, device authorization, and rotating refresh. */ class MockIdp { + readonly requests: URLSearchParams[] = []; readonly state = { deviceAuthorizations: 0, refreshGrants: 0, @@ -159,6 +182,10 @@ class MockIdp { return; } + if (url === "/device" || url === "/token") { + this.requests.push(params); + } + if (url === "/device") { this.state.deviceAuthorizations += 1; respond(200, { diff --git a/nodejs/lancedb/oauth.ts b/nodejs/lancedb/oauth.ts index c9c78afa6..162f38997 100644 --- a/nodejs/lancedb/oauth.ts +++ b/nodejs/lancedb/oauth.ts @@ -28,7 +28,7 @@ export enum OAuthFlowType { * directory with owner-only permissions, so short-lived processes can reuse * an authenticated session instead of re-prompting on every start. * - * Multiple identities (issuer, client, scopes, flow, client authentication) + * Multiple identities (issuer, client, scopes, resource, audience, flow, client authentication) * get separate cache entries. Within one identity the most recent login wins. */ export interface TokenCacheOptions { @@ -67,6 +67,18 @@ export interface TokenCacheOptions { * }; * ``` * + * Providers requiring an explicit target can set `resource` and/or `audience`: + * ```typescript + * const targeted: OAuthConfig = { + * issuerUrl: "https://issuer.example.com", + * clientId: "app-id", + * clientSecret: "secret", + * scopes: ["read"], + * resource: "https://api.example.com", + * audience: "lancedb-api", + * }; + * ``` + * * @example Azure Managed Identity: * ```typescript * const config: OAuthConfig = { @@ -109,6 +121,19 @@ export interface OAuthConfig { */ scopes: string[]; + /** + * Resource indicator (RFC 8707), forwarded verbatim to authorization and token + * endpoints, including refresh requests. Must be an absolute URI without a + * fragment. Not supported for Azure managed identity. + */ + resource?: string; + + /** + * Provider-specific audience, forwarded to authorization and token endpoints, + * including refresh requests. Not supported for Azure managed identity. + */ + audience?: string; + /** Authentication flow (default: ClientCredentials). */ flow?: OAuthFlowType; @@ -166,6 +191,12 @@ export interface SessionStatus { /** Canonical (sorted, de-duplicated) scope set of the cached session. */ scopes: string[]; + /** Resource indicator used to obtain the cached session, if configured. */ + resource?: string; + + /** Provider-specific audience used to obtain the cached session, if configured. */ + audience?: string; + /** Flow that produced the cached session. */ flow: string; diff --git a/nodejs/src/remote.rs b/nodejs/src/remote.rs index c619aca35..1db00e832 100644 --- a/nodejs/src/remote.rs +++ b/nodejs/src/remote.rs @@ -188,6 +188,10 @@ pub struct OAuthConfig { /// OAuth scopes to request. For Azure managed identity, exactly one scope /// or resource is required. For example: `["api://{app_id}/.default"]` pub scopes: Vec, + /// Optional resource indicator for authorization and token requests. + pub resource: Option, + /// Optional provider-specific audience for authorization and token requests. + pub audience: Option, /// Authentication flow: "client_credentials", "authorization_code", /// "device_code", or "azure_managed_identity" pub flow: Option, @@ -216,6 +220,8 @@ impl std::fmt::Debug for OAuthConfig { .field("issuer_url", &self.issuer_url) .field("client_id", &self.client_id) .field("scopes", &self.scopes) + .field("resource", &self.resource) + .field("audience", &self.audience) .field("flow", &self.flow) .field( "client_secret", @@ -269,6 +275,8 @@ impl TryFrom for lancedb::remote::oauth::OAuthConfig { client_id: config.client_id, client_secret: config.client_secret, scopes: config.scopes, + resource: config.resource, + audience: config.audience, flow, refresh_buffer_secs: config.refresh_buffer_secs.map(|v| v as u64), token_cache: config.token_cache.map(Into::into), @@ -290,6 +298,10 @@ pub struct SessionStatus { pub client_id: String, /// Canonical (sorted, de-duplicated) scopes of the cached session. pub scopes: Vec, + /// Optional resource indicator for authorization and token requests. + pub resource: Option, + /// Optional provider-specific audience for authorization and token requests. + pub audience: Option, /// Flow that produced the cached session. pub flow: String, /// When the cached session was obtained, as Unix seconds. @@ -374,6 +386,8 @@ impl From for SessionStatus { issuer_url: status.issuer_url, client_id: status.client_id, scopes: status.scopes, + resource: status.resource, + audience: status.audience, flow: status.flow, obtained_at: status.obtained_at.map(|secs| secs as f64), } @@ -418,6 +432,8 @@ mod tests { use_pkce: None, managed_identity_client_id: None, refresh_buffer_secs: None, + resource: None, + audience: None, token_cache: None, }; @@ -442,6 +458,8 @@ mod tests { use_pkce: None, managed_identity_client_id: None, refresh_buffer_secs: None, + resource: None, + audience: None, token_cache: None, }; @@ -463,6 +481,8 @@ mod tests { use_pkce: Some(false), managed_identity_client_id: None, refresh_buffer_secs: None, + resource: Some("urn:resource".into()), + audience: Some("audience".into()), token_cache: None, }; @@ -476,6 +496,8 @@ mod tests { ); assert_eq!(options.callback_port, Some(9000)); assert!(!options.use_pkce); + assert_eq!(converted.resource.as_deref(), Some("urn:resource")); + assert_eq!(converted.audience.as_deref(), Some("audience")); } #[test] @@ -491,6 +513,8 @@ mod tests { use_pkce: None, managed_identity_client_id: None, refresh_buffer_secs: None, + resource: None, + audience: None, token_cache: None, }; diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index 32159dd5a..e18657eb7 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -284,6 +284,10 @@ class JobInfo: def created_at_millis(self) -> int: ... class SessionStatus: + @property + def resource(self) -> Optional[str]: ... + @property + def audience(self) -> Optional[str]: ... @property def refreshable(self) -> bool: ... @property diff --git a/python/python/lancedb/remote/oauth.py b/python/python/lancedb/remote/oauth.py index ad9fd2b0e..e8531db1c 100644 --- a/python/python/lancedb/remote/oauth.py +++ b/python/python/lancedb/remote/oauth.py @@ -86,6 +86,13 @@ class OAuthConfig: Protect AUTHORIZATION_CODE with S256 PKCE (default: True). managed_identity_client_id : Optional[str] Client ID for user-assigned managed identity (AZURE_MANAGED_IDENTITY). + resource : Optional[str] + Resource indicator (RFC 8707), forwarded verbatim to authorization and + token endpoints, including refresh requests. Must be an absolute URI + without a fragment. Not supported for Azure managed identity. + audience : Optional[str] + Provider-specific audience, forwarded to authorization and token + endpoints, including refresh requests. Not supported for Azure managed identity. refresh_buffer_secs : Optional[int] Seconds before expiry to trigger proactive refresh (default: 300). Keep this well below the token TTL; if it is greater than or equal to @@ -106,6 +113,12 @@ class OAuthConfig: ... scopes=["api://lancedb-api/.default"], ... ) + Providers that require an explicit target can use ``resource`` and/or + ``audience`` (these are forwarded unchanged): + + >>> config.resource = "https://api.example.com" + >>> config.audience = "lancedb-api" + Azure Managed Identity: >>> config = OAuthConfig( @@ -150,6 +163,8 @@ class OAuthConfig: managed_identity_client_id: Optional[str] = None refresh_buffer_secs: Optional[int] = None token_cache: Optional[TokenCacheOptions] = None + resource: Optional[str] = None + audience: Optional[str] = None class OAuthSession: diff --git a/python/src/oauth.rs b/python/src/oauth.rs index ff24a9566..080ea4d95 100644 --- a/python/src/oauth.rs +++ b/python/src/oauth.rs @@ -36,6 +36,10 @@ pub struct PyOAuthConfig { pub issuer_url: String, pub client_id: String, pub scopes: Vec, + /// Optional resource indicator for authorization and token requests. + pub resource: Option, + /// Optional provider-specific audience for authorization and token requests. + pub audience: Option, pub flow: String, pub client_secret: Option, pub redirect_uri: Option, @@ -78,6 +82,8 @@ impl TryFrom for OAuthConfig { client_id: py.client_id, client_secret: py.client_secret, scopes: py.scopes, + resource: py.resource, + audience: py.audience, flow, refresh_buffer_secs: py.refresh_buffer_secs, token_cache: py.token_cache.map(TokenCacheOptions::from), @@ -119,6 +125,18 @@ impl PySessionStatus { self.inner.scopes.clone() } + /// Resource indicator used to obtain the cached session. + #[getter] + pub fn resource(&self) -> Option { + self.inner.resource.clone() + } + + /// Provider-specific audience used to obtain the cached session. + #[getter] + pub fn audience(&self) -> Option { + self.inner.audience.clone() + } + /// Flow that produced the cached session. #[getter] pub fn flow(&self) -> String { @@ -242,6 +260,8 @@ mod tests { use_pkce: true, managed_identity_client_id: None, refresh_buffer_secs: None, + resource: None, + audience: None, token_cache: None, } } @@ -269,6 +289,8 @@ mod tests { redirect_uri: Some("http://127.0.0.1:9000/callback".to_string()), callback_port: Some(9000), use_pkce: false, + resource: Some("urn:resource".into()), + audience: Some("audience".into()), ..base_config() }; @@ -282,6 +304,8 @@ mod tests { ); assert_eq!(options.callback_port, Some(9000)); assert!(!options.use_pkce); + assert_eq!(converted.resource.as_deref(), Some("urn:resource")); + assert_eq!(converted.audience.as_deref(), Some("audience")); } #[test] @@ -294,6 +318,8 @@ mod tests { #[test] fn test_token_cache_conversion() { let config = PyOAuthConfig { + resource: None, + audience: None, token_cache: Some(PyTokenCacheOptions { cache_dir: Some("/tmp/oauth-cache".to_string()), lock_timeout_secs: Some(5), diff --git a/python/tests/test_oauth.py b/python/tests/test_oauth.py index 36f687042..583f5a35f 100644 --- a/python/tests/test_oauth.py +++ b/python/tests/test_oauth.py @@ -72,6 +72,8 @@ def test_token_cache_options_default_to_memory_only(): scopes=["openid"], ) assert config.token_cache is None + assert config.resource is None + assert config.audience is None options = oauth.TokenCacheOptions() assert options.cache_dir is None @@ -121,6 +123,7 @@ class _MockIdpState: self.invalid_grant_rejections = 0 self.access_tokens_issued = 0 self.current_refresh = None + self.requests = [] class _MockIdpHandler(BaseHTTPRequestHandler): @@ -156,6 +159,8 @@ class _MockIdpHandler(BaseHTTPRequestHandler): length = int(self.headers.get("Content-Length", 0)) body = self.rfile.read(length).decode() params = urllib.parse.parse_qs(body) + with self.state.lock: + self.state.requests.append(params) if self.path == "/device": with self.state.lock: @@ -212,11 +217,11 @@ def _start_mock_idp() -> tuple[_MockIdpState, HTTPServer]: return state, server -def _run_subprocess(script: Path, issuer_url: str, cache_dir: Path): +def _run_subprocess(script: Path, issuer_url: str, cache_dir: Path, target: dict): env = dict(os.environ) env["LANCEDB_OAUTH_BROWSER"] = "/usr/bin/true" result = subprocess.run( - [sys.executable, str(script), issuer_url, str(cache_dir)], + [sys.executable, str(script), issuer_url, str(cache_dir), json.dumps(target)], capture_output=True, text=True, timeout=120, @@ -230,6 +235,7 @@ def _run_subprocess(script: Path, issuer_url: str, cache_dir: Path): LOGIN_SCRIPT = """ import asyncio +import json import sys from lancedb.remote import OAuthConfig, OAuthFlowType, OAuthSession, TokenCacheOptions @@ -241,15 +247,19 @@ config = OAuthConfig( scopes=["openid"], flow=OAuthFlowType.DEVICE_CODE, token_cache=TokenCacheOptions(cache_dir=cache_dir), + **json.loads(sys.argv[3]), ) session = OAuthSession(config) status = asyncio.run(session.login()) assert status.refreshable, "login must cache a refresh token" +assert status.resource == config.resource +assert status.audience == config.audience print("LOGIN-OK") """ REUSE_SCRIPT = """ import asyncio +import json import sys import lancedb @@ -262,6 +272,7 @@ config = OAuthConfig( scopes=["openid"], flow=OAuthFlowType.DEVICE_CODE, token_cache=TokenCacheOptions(cache_dir=cache_dir), + **json.loads(sys.argv[3]), ) session = OAuthSession(config) @@ -292,7 +303,17 @@ print("REUSE-OK") """ -def test_cross_process_session_reuse_without_new_prompt(tmp_path): +@pytest.mark.parametrize( + "target", + [ + {}, + { + "resource": "https://api.example.com/a?x=1&y=two", + "audience": "audience + & / ü", + }, + ], +) +def test_cross_process_session_reuse_without_new_prompt(tmp_path, target): pytest.importorskip("lancedb") state, server = _start_mock_idp() try: @@ -303,11 +324,11 @@ def test_cross_process_session_reuse_without_new_prompt(tmp_path): reuse_script.write_text(REUSE_SCRIPT) cache_dir = tmp_path / "oauth-cache" - result = _run_subprocess(login_script, issuer_url, cache_dir) + result = _run_subprocess(login_script, issuer_url, cache_dir, target) assert "LOGIN-OK" in result.stdout assert state.device_authorizations == 1 - result = _run_subprocess(reuse_script, issuer_url, cache_dir) + result = _run_subprocess(reuse_script, issuer_url, cache_dir, target) assert "REUSE-OK" in result.stdout assert "DATABASE-UNREACHABLE-AS-EXPECTED" in result.stdout @@ -317,11 +338,14 @@ def test_cross_process_session_reuse_without_new_prompt(tmp_path): assert state.device_authorizations == 1 assert state.invalid_grant_rejections == 0 - logout = asyncio.run( - _remote_oauth() - .OAuthSession(_device_config(_remote_oauth(), issuer_url, cache_dir)) - .logout() - ) + assert len(state.requests) == 3 + for params in state.requests: + for key in ("resource", "audience"): + assert params.get(key) == ([target[key]] if key in target else None) + config = _device_config(_remote_oauth(), issuer_url, cache_dir) + config.resource = target.get("resource") + config.audience = target.get("audience") + logout = asyncio.run(_remote_oauth().OAuthSession(config).logout()) assert logout.removed is True finally: server.shutdown() diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index b0446919b..e136a0923 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -1546,6 +1546,8 @@ mod tests { scopes: vec!["scope".to_string()], flow: crate::remote::OAuthFlow::ClientCredentials, refresh_buffer_secs: None, + resource: None, + audience: None, token_cache: None, }; @@ -1589,6 +1591,8 @@ mod tests { scopes: vec!["scope".to_string()], flow: crate::remote::OAuthFlow::ClientCredentials, refresh_buffer_secs: None, + resource: None, + audience: None, token_cache: None, }; let client_config = crate::remote::ClientConfig { diff --git a/rust/lancedb/src/remote/oauth.rs b/rust/lancedb/src/remote/oauth.rs index 75a8878a8..516219a14 100644 --- a/rust/lancedb/src/remote/oauth.rs +++ b/rust/lancedb/src/remote/oauth.rs @@ -187,6 +187,17 @@ pub struct OAuthConfig { /// For example: `["api://{app_id}/.default"]` pub scopes: Vec, + /// Resource indicator sent to the authorization and token endpoints (RFC 8707). + /// The value is forwarded verbatim, including on refresh requests, and must + /// be an absolute URI without a fragment. + /// Not supported for Azure managed identity. + pub resource: Option, + + /// Provider-specific audience sent to the authorization and token endpoints, + /// including refresh requests. + /// Not supported for Azure managed identity. + pub audience: Option, + /// Authentication flow to use. pub flow: OAuthFlow, @@ -214,6 +225,8 @@ impl std::fmt::Debug for OAuthConfig { &self.client_secret.as_deref().map(|_| ""), ) .field("scopes", &self.scopes) + .field("resource", &self.resource) + .field("audience", &self.audience) .field("flow", &self.flow) .field("refresh_buffer_secs", &self.refresh_buffer_secs) .field("token_cache", &self.token_cache) @@ -366,6 +379,8 @@ struct OidcClient { client_id: String, client_secret: Option, scopes: Vec, + resource: Option, + audience: Option, http_client: Client, discovery: RwLock>, } @@ -380,6 +395,8 @@ impl std::fmt::Debug for OidcClient { &self.client_secret.as_ref().map(|_| ""), ) .field("scopes", &self.scopes) + .field("resource", &self.resource) + .field("audience", &self.audience) .finish() } } @@ -390,6 +407,8 @@ impl OidcClient { client_id: String, client_secret: Option, scopes: Vec, + resource: Option, + audience: Option, ) -> Result { Self::validate_issuer_transport(&issuer_url)?; @@ -413,6 +432,8 @@ impl OidcClient { client_id, client_secret, scopes, + resource, + audience, http_client, discovery: RwLock::new(None), }) @@ -483,6 +504,14 @@ impl OidcClient { self.get_discovery().await.map(|disc| disc.token_endpoint) } + fn target_params(&self) -> impl Iterator { + self.resource + .as_deref() + .map(|value| ("resource", value)) + .into_iter() + .chain(self.audience.as_deref().map(|value| ("audience", value))) + } + fn scopes_string(&self) -> String { self.scopes.join(" ") } @@ -492,10 +521,15 @@ impl OidcClient { endpoint: &str, params: &[(String, String)], ) -> Result { + let mut params = params.to_vec(); + params.extend( + self.target_params() + .map(|(key, value)| (key.to_owned(), value.to_owned())), + ); let resp = self .http_client .post(endpoint) - .form(params) + .form(¶ms) .send() .await .map_err(|e| Error::Runtime { @@ -527,6 +561,10 @@ impl OidcClient { if let Some(secret) = self.client_secret.as_ref() { params.push(("client_secret".to_string(), secret.clone())); } + params.extend( + self.target_params() + .map(|(key, value)| (key.to_owned(), value.to_owned())), + ); let response = self .http_client .post(&endpoint) @@ -581,6 +619,8 @@ impl ClientCredentialsSource { client_id: String, client_secret: Option, scopes: Vec, + resource: Option, + audience: Option, ) -> Result { if client_secret.is_none() { return Err(Error::InvalidInput { @@ -588,7 +628,14 @@ impl ClientCredentialsSource { }); } Ok(Self { - oidc: OidcClient::new(issuer_url, client_id, client_secret, scopes)?, + oidc: OidcClient::new( + issuer_url, + client_id, + client_secret, + scopes, + resource, + audience, + )?, }) } } @@ -719,11 +766,20 @@ impl AuthorizationCodeSource { client_id: String, client_secret: Option, scopes: Vec, + resource: Option, + audience: Option, options: AuthorizationCodeOptions, ) -> Result { let redirect = ResolvedRedirect::new(&options)?; Ok(Self { - oidc: OidcClient::new(issuer_url, client_id, client_secret, scopes)?, + oidc: OidcClient::new( + issuer_url, + client_id, + client_secret, + scopes, + resource, + audience, + )?, options, redirect, }) @@ -750,6 +806,7 @@ impl AuthorizationCodeSource { .append_pair("redirect_uri", &self.redirect.uri) .append_pair("scope", &self.oidc.scopes_string()) .append_pair("state", &state); + query.extend_pairs(self.oidc.target_params()); if let Some(verifier) = code_verifier.as_ref() { let challenge = base64::engine::general_purpose::URL_SAFE_NO_PAD .encode(Sha256::digest(verifier.as_bytes())); @@ -895,9 +952,18 @@ impl DeviceCodeSource { client_id: String, client_secret: Option, scopes: Vec, + resource: Option, + audience: Option, ) -> Result { Ok(Self { - oidc: OidcClient::new(issuer_url, client_id, client_secret, scopes)?, + oidc: OidcClient::new( + issuer_url, + client_id, + client_secret, + scopes, + resource, + audience, + )?, }) } @@ -917,6 +983,11 @@ impl DeviceCodeSource { if let Some(secret) = self.oidc.client_secret.as_ref() { params.push(("client_secret".to_string(), secret.clone())); } + params.extend( + self.oidc + .target_params() + .map(|(key, value)| (key.to_owned(), value.to_owned())), + ); let response = self .oidc .http_client @@ -980,6 +1051,11 @@ impl DeviceCodeSource { params.push(("client_secret".to_string(), secret.clone())); } + params.extend( + self.oidc + .target_params() + .map(|(key, value)| (key.to_owned(), value.to_owned())), + ); let response = match self .oidc .http_client @@ -1287,6 +1363,13 @@ impl TokenSource for AzureImdsSource { /// Shared by [`OAuthHeaderProvider`] and /// [`OAuthSession`](crate::remote::OAuthSession). pub(crate) fn build_token_source(config: &OAuthConfig) -> Result> { + if matches!(config.flow, OAuthFlow::AzureManagedIdentity { .. }) + && (config.resource.is_some() || config.audience.is_some()) + { + return Err(Error::InvalidInput { + message: "resource and audience are not supported for AzureManagedIdentity; configure its resource through scopes".to_string(), + }); + } if config.scopes.is_empty() { return Err(Error::InvalidInput { message: "At least one OAuth scope is required".to_string(), @@ -1298,12 +1381,16 @@ pub(crate) fn build_token_source(config: &OAuthConfig) -> Result Box::new(AuthorizationCodeSource::new( config.issuer_url.clone(), config.client_id.clone(), config.client_secret.clone(), config.scopes.clone(), + config.resource.clone(), + config.audience.clone(), options.clone(), )?), OAuthFlow::DeviceCode => Box::new(DeviceCodeSource::new( @@ -1311,6 +1398,8 @@ pub(crate) fn build_token_source(config: &OAuthConfig) -> Result Box::new(AzureImdsSource::new( config.scopes.clone(), @@ -1441,6 +1530,145 @@ mod tests { use tokio::net::{TcpListener, TcpStream}; use tokio::task::JoinHandle; + #[tokio::test] + async fn test_target_parameters_across_oauth_flows() { + for (resource, audience) in [ + (None, None), + (Some("https://api.example.com/a?x=1&y=two"), None), + (None, Some("audience + & / ü")), + (Some("urn:example:resource"), Some("audience + & / ü")), + ] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let issuer = format!("http://{addr}"); + let server = tokio::spawn(async move { + let mut grants = Vec::new(); + // Three discovery requests and six form submissions. + for _ in 0..9 { + let (mut stream, _) = listener.accept().await.unwrap(); + let (line, body) = read_http_request(&mut stream).await; + let response = if line.starts_with("GET ") { + serde_json::json!({ + "token_endpoint": format!("http://{addr}/token"), + "authorization_endpoint": format!("http://{addr}/authorize"), + "device_authorization_endpoint": format!("http://{addr}/device"), + }) + } else { + let params: Vec<_> = url::form_urlencoded::parse(body.as_bytes()).collect(); + for (key, expected) in [("resource", resource), ("audience", audience)] { + let values: Vec<_> = params + .iter() + .filter(|(name, _)| name == key) + .map(|(_, value)| value.as_ref()) + .collect(); + assert_eq!(values, expected.into_iter().collect::>()); + } + if line.starts_with("POST /device ") { + serde_json::json!({ + "device_code": "device-code", "user_code": "ABCD", + "verification_uri": format!("http://{addr}/verify"), + "expires_in": 60, "interval": 1, + }) + } else { + grants.push( + params + .iter() + .find(|(key, _)| key == "grant_type") + .unwrap() + .1 + .to_string(), + ); + serde_json::json!({"access_token": "access", "refresh_token": "refresh", "expires_in": 3600}) + } + }; + write_json_response(&mut stream, "200 OK", &response.to_string()).await; + } + assert_eq!( + grants, + [ + "client_credentials", + "authorization_code", + "refresh_token", + "urn:ietf:params:oauth:grant-type:device_code", + "refresh_token" + ] + ); + }); + let credentials = ClientCredentialsSource::new( + issuer.clone(), + "client".into(), + Some("secret".into()), + vec!["scope".into()], + resource.map(str::to_owned), + audience.map(str::to_owned), + ) + .unwrap(); + credentials.fetch_token().await.unwrap(); + let browser = AuthorizationCodeSource::new( + issuer.clone(), + "client".into(), + None, + vec!["scope".into()], + resource.map(str::to_owned), + audience.map(str::to_owned), + AuthorizationCodeOptions::new(), + ) + .unwrap(); + let request = browser.build_authorization_request().await.unwrap(); + for (key, expected) in [("resource", resource), ("audience", audience)] { + let values: Vec<_> = request + .url + .query_pairs() + .filter(|(name, _)| name == key) + .map(|(_, value)| value.into_owned()) + .collect(); + assert_eq!( + values, + expected.into_iter().map(str::to_owned).collect::>() + ); + } + browser + .exchange_code("code", Some("verifier")) + .await + .unwrap(); + browser.refresh_token("refresh").await.unwrap(); + let device = DeviceCodeSource::new( + issuer, + "client".into(), + None, + vec!["scope".into()], + resource.map(str::to_owned), + audience.map(str::to_owned), + ) + .unwrap(); + let authorization = device.request_device_authorization().await.unwrap(); + device.poll_for_token(&authorization).await.unwrap(); + device.refresh_token("refresh").await.unwrap(); + server.await.unwrap(); + } + } + + #[test] + fn test_managed_identity_rejects_target_parameters() { + for (resource, audience) in [(Some("urn:resource"), None), (None, Some("audience"))] { + let config = OAuthConfig { + issuer_url: "https://issuer.example.com".into(), + client_id: "client".into(), + client_secret: None, + scopes: vec!["api://app/.default".into()], + flow: OAuthFlow::AzureManagedIdentity { client_id: None }, + resource: resource.map(str::to_owned), + audience: audience.map(str::to_owned), + refresh_buffer_secs: None, + token_cache: None, + }; + let error = OAuthHeaderProvider::new(config).unwrap_err().to_string(); + assert!( + error.contains("resource and audience are not supported for AzureManagedIdentity") + ); + } + } + #[test] fn test_token_state_expiry() { let mut state = TokenState::new(); @@ -1529,6 +1757,8 @@ mod tests { "app-id".to_string(), Some("secret".to_string()), vec!["scope1".to_string(), "scope2".to_string()], + None, + None, ) .unwrap(); @@ -1692,6 +1922,8 @@ mod tests { "client-id".to_string(), None, vec!["openid".to_string()], + None, + None, AuthorizationCodeOptions::new() .redirect_uri(format!("http://127.0.0.1:{port}/callback")), ) @@ -1768,6 +2000,8 @@ mod tests { "client-id".to_string(), None, vec!["openid".to_string(), "profile".to_string()], + None, + None, AuthorizationCodeOptions::new(), ) .unwrap(); @@ -1799,6 +2033,8 @@ mod tests { "client-id".to_string(), None, vec!["openid".to_string()], + None, + None, AuthorizationCodeOptions::new(), ) .unwrap(); @@ -1820,6 +2056,8 @@ mod tests { "client-id".to_string(), Some("secret".to_string()), vec!["openid".to_string()], + None, + None, AuthorizationCodeOptions::new().use_pkce(false), ) .unwrap(); @@ -1840,6 +2078,8 @@ mod tests { "client-id".to_string(), Some("secret".to_string()), vec!["openid".to_string()], + None, + None, AuthorizationCodeOptions::new(), ) .unwrap(); @@ -1866,6 +2106,8 @@ mod tests { "client-id".to_string(), None, vec!["openid".to_string()], + None, + None, AuthorizationCodeOptions::new(), ) .unwrap(); @@ -1889,6 +2131,8 @@ mod tests { "client-id".to_string(), None, vec!["openid".to_string()], + None, + None, AuthorizationCodeOptions::new(), ) .unwrap(); @@ -1910,6 +2154,8 @@ mod tests { "client-id".to_string(), Some("secret".to_string()), vec!["openid".to_string()], + None, + None, ) .unwrap(); @@ -1930,6 +2176,8 @@ mod tests { "client-id".to_string(), None, vec!["openid".to_string()], + None, + None, ) .unwrap(); @@ -1952,6 +2200,8 @@ mod tests { "client-id".to_string(), None, vec!["openid".to_string()], + None, + None, ) .unwrap(); let device = test_device_authorization_response(10, 1); @@ -1971,6 +2221,8 @@ mod tests { "client-id".to_string(), None, vec!["openid".to_string()], + None, + None, ) .unwrap(); let device = test_device_authorization_response(60, 1); @@ -1992,6 +2244,8 @@ mod tests { "client-id".to_string(), None, vec!["openid".to_string()], + None, + None, ) .unwrap(); let device = test_device_authorization_response(60, 1); @@ -2013,6 +2267,8 @@ mod tests { "client-id".to_string(), None, vec!["openid".to_string()], + None, + None, ) .unwrap(); let device = test_device_authorization_response(1, 5); @@ -2192,6 +2448,8 @@ mod tests { scopes: vec!["scope".to_string()], flow: OAuthFlow::ClientCredentials, refresh_buffer_secs: None, + resource: None, + audience: None, token_cache: None, }; @@ -2209,6 +2467,8 @@ mod tests { scopes: vec!["scope".to_string()], flow: OAuthFlow::ClientCredentials, refresh_buffer_secs: None, + resource: None, + audience: None, token_cache: None, }; @@ -2246,6 +2506,8 @@ mod tests { ], flow: OAuthFlow::AzureManagedIdentity { client_id: None }, refresh_buffer_secs: None, + resource: None, + audience: None, token_cache: None, }; assert!(OAuthHeaderProvider::new(config).is_err()); @@ -2259,6 +2521,8 @@ mod tests { "client-id".to_string(), Some("secret".to_string()), vec!["scope".to_string()], + None, + None, ) .unwrap(); @@ -2280,6 +2544,8 @@ mod tests { scopes: vec!["scope".to_string()], flow: OAuthFlow::ClientCredentials, refresh_buffer_secs: None, + resource: None, + audience: None, token_cache: None, }; assert!(OAuthHeaderProvider::new(config).is_err()); @@ -2294,6 +2560,8 @@ mod tests { scopes: vec!["scope".to_string()], flow: OAuthFlow::ClientCredentials, refresh_buffer_secs: None, + resource: None, + audience: None, token_cache: None, }; @@ -2315,6 +2583,8 @@ mod tests { scopes: vec![], flow: OAuthFlow::AzureManagedIdentity { client_id: None }, refresh_buffer_secs: None, + resource: None, + audience: None, token_cache: None, }; assert!(OAuthHeaderProvider::new(config).is_err()); @@ -2330,6 +2600,8 @@ mod tests { scopes: vec!["scope".to_string()], flow: OAuthFlow::ClientCredentials, refresh_buffer_secs: Some(0), + resource: None, + audience: None, token_cache: None, }; let provider = OAuthHeaderProvider::new(config).unwrap(); diff --git a/rust/lancedb/src/remote/token_cache.rs b/rust/lancedb/src/remote/token_cache.rs index b67ced651..7d305b3cb 100644 --- a/rust/lancedb/src/remote/token_cache.rs +++ b/rust/lancedb/src/remote/token_cache.rs @@ -22,7 +22,8 @@ //! owner-checked, and symlink-rejected on Unix; records are replaced //! atomically via `rename` so a crash can never leave a torn file. //! - Cache filenames are SHA-256 hashes of the canonical issuer, client, -//! scope, flow, and client-auth identity. No secret appears in a filename. +//! scope, resource, audience, flow, and client-auth identity. No secret appears +//! in a filename. //! - Refresh-token rotation is serialized across processes with a per-key //! advisory file lock (`flock` on Unix, `LockFileEx` on Windows). The //! operating system releases these locks when a process dies, so a crash @@ -44,6 +45,8 @@ //! scopes: vec!["openid".to_string()], //! flow: OAuthFlow::DeviceCode, //! refresh_buffer_secs: None, +//! resource: Some("https://api.example.com".to_string()), +//! audience: None, //! token_cache: Some( //! TokenCacheOptions::new().cache_dir("/tmp/my-app/oauth-cache"), //! ), @@ -184,6 +187,10 @@ struct CachedTokenRecord { issuer_url: String, client_id: String, scopes: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + resource: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + audience: Option, flow: String, client_auth: String, refresh_token: String, @@ -197,6 +204,8 @@ impl std::fmt::Debug for CachedTokenRecord { .field("issuer_url", &self.issuer_url) .field("client_id", &self.client_id) .field("scopes", &self.scopes) + .field("resource", &self.resource) + .field("audience", &self.audience) .field("flow", &self.flow) .field("client_auth", &self.client_auth) .field("refresh_token", &"") @@ -240,13 +249,15 @@ fn client_auth_key(client_secret: Option<&str>) -> &'static str { } } -/// Identity of one cached session: canonical issuer, client, scopes, flow, -/// and client-auth mode, plus the hashed filename derived from it. +/// Identity of one cached session: canonical issuer, client, scopes, resource, +/// audience, flow, and client-auth mode, plus the hashed filename derived from it. #[derive(Clone, Debug)] struct CacheKey { issuer_url: String, client_id: String, scopes: Vec, + resource: Option, + audience: Option, flow: &'static str, client_auth: &'static str, file_stem: String, @@ -272,11 +283,32 @@ impl CacheKey { flow, client_auth ); + // Keep existing sessions reachable when no target was specified. Targeted + // sessions use a structured encoding so parameter contents cannot collide. + let identity = if config.resource.is_none() && config.audience.is_none() { + identity + } else { + serde_json::to_string(&( + "v2", + &issuer_url, + &config.client_id, + &scopes, + flow, + client_auth, + &config.resource, + &config.audience, + )) + .map_err(|error| Error::Runtime { + message: format!("Failed to encode OAuth cache identity: {error}"), + })? + }; let file_stem = hex_sha256(identity.as_bytes()); Ok(Self { issuer_url, client_id: config.client_id.clone(), scopes, + resource: config.resource.clone(), + audience: config.audience.clone(), flow, client_auth, file_stem, @@ -394,6 +426,8 @@ impl TokenCache { issuer_url: self.key.issuer_url.clone(), client_id: self.key.client_id.clone(), scopes: self.key.scopes.clone(), + resource: self.key.resource.clone(), + audience: self.key.audience.clone(), flow: self.key.flow.to_string(), client_auth: self.key.client_auth.to_string(), refresh_token, @@ -761,6 +795,12 @@ pub struct SessionStatus { /// Canonical (sorted, de-duplicated) scope set of the cached session. pub scopes: Vec, + /// Resource indicator used to obtain the cached session, if configured. + pub resource: Option, + + /// Provider-specific audience used to obtain the cached session, if configured. + pub audience: Option, + /// Flow that produced the cached session. pub flow: String, @@ -803,6 +843,8 @@ pub struct SessionLogout { /// scopes: vec!["openid".to_string()], /// flow: OAuthFlow::DeviceCode, /// refresh_buffer_secs: None, +/// resource: None, +/// audience: None, /// token_cache: Some(TokenCacheOptions::new()), /// }; /// let session = OAuthSession::new(config)?; @@ -873,6 +915,8 @@ impl OAuthSession { issuer_url: record.issuer_url, client_id: record.client_id, scopes: record.scopes, + resource: record.resource, + audience: record.audience, flow: record.flow, obtained_at: Some(record.obtained_at), }, @@ -881,6 +925,8 @@ impl OAuthSession { issuer_url: self.cache.key.issuer_url.clone(), client_id: self.cache.key.client_id.clone(), scopes: self.cache.key.scopes.clone(), + resource: self.cache.key.resource.clone(), + audience: self.cache.key.audience.clone(), flow: self.cache.key.flow.to_string(), obtained_at: None, }, @@ -960,6 +1006,8 @@ mod tests { scopes: vec!["openid".to_string()], flow: OAuthFlow::DeviceCode, refresh_buffer_secs: None, + resource: None, + audience: None, token_cache: Some(TokenCacheOptions::new().cache_dir(cache_dir)), } } @@ -970,6 +1018,7 @@ mod tests { /// `invalid_grant`, which is exactly what real providers do on rotation. struct MockIdp { issuer_url: String, + requests: Arc>>, device_authorizations: Arc, refresh_attempts: Arc, invalid_grant_rejections: Arc, @@ -986,6 +1035,7 @@ mod tests { let issuer_url = format!("http://{addr}"); let server = Self { issuer_url: issuer_url.clone(), + requests: Arc::new(std::sync::Mutex::new(Vec::new())), device_authorizations: Arc::new(AtomicUsize::new(0)), refresh_attempts: Arc::new(AtomicUsize::new(0)), invalid_grant_rejections: Arc::new(AtomicUsize::new(0)), @@ -994,6 +1044,7 @@ mod tests { fail_refreshes: Arc::new(AtomicBool::new(false)), issue_refresh_tokens: Arc::new(AtomicBool::new(true)), }; + let requests = Arc::clone(&server.requests); let device_authorizations = Arc::clone(&server.device_authorizations); let refresh_attempts = Arc::clone(&server.refresh_attempts); let invalid_grant_rejections = Arc::clone(&server.invalid_grant_rejections); @@ -1007,6 +1058,7 @@ mod tests { let Ok((mut stream, _)) = listener.accept().await else { return; }; + let requests = Arc::clone(&requests); let device_authorizations = Arc::clone(&device_authorizations); let refresh_attempts = Arc::clone(&refresh_attempts); let invalid_grant_rejections = Arc::clone(&invalid_grant_rejections); @@ -1016,6 +1068,9 @@ mod tests { let issue_refresh_tokens = Arc::clone(&issue_refresh_tokens); tokio::spawn(async move { let (request_line, body) = read_http_request(&mut stream).await; + if request_line.starts_with("POST ") { + requests.lock().unwrap().push(body.clone()); + } if request_line.starts_with("GET /.well-known/openid-configuration ") { let discovery = format!( r#"{{"token_endpoint":"http://{addr}/token","device_authorization_endpoint":"http://{addr}/device"}}"# @@ -1391,6 +1446,84 @@ mod tests { assert!(!format!("{status:?}").contains("refresh-")); } + #[tokio::test] + async fn test_targeted_cache_refresh_and_logout_isolation() { + let dir = cache_tempdir(); + let idp = MockIdp::start().await; + let mut config = idp.config(dir.path()); + let options = config.token_cache.clone().unwrap(); + let untargeted_key = CacheKey::new(&config).unwrap().file_stem; + assert_eq!( + untargeted_key, + hex_sha256( + format!( + "v1\n{}\nclient-id\nopenid\ndevice_code\npublic", + idp.issuer_url + ) + .as_bytes() + ) + ); + let mut sessions = Vec::new(); + let mut keys = std::collections::HashSet::new(); + for (resource, audience) in [ + (None, None), + (Some("urn:one"), None), + (None, Some("audience & ü")), + (Some("urn:one"), Some("audience & ü")), + (Some("urn:two"), Some("audience & ü")), + (Some("urn:one"), Some("other")), + ] { + config.resource = resource.map(str::to_owned); + config.audience = audience.map(str::to_owned); + let cache = TokenCache::new(&config, &options).unwrap(); + assert!(keys.insert(cache.key.file_stem.clone())); + let record = cache + .record_from_response(&TokenResponse { + access_token: "unused".into(), + refresh_token: Some("seed-refresh".into()), + expires_in: Some(3600), + token_type: None, + }) + .unwrap(); + cache.store(&record).await.unwrap(); + *idp.current_refresh.lock().unwrap() = Some("seed-refresh".into()); + let provider = OAuthHeaderProvider::new(config.clone()).unwrap(); + provider.get_headers().await.unwrap(); + let request = idp.requests.lock().unwrap().last().unwrap().clone(); + let params: std::collections::HashMap<_, _> = + url::form_urlencoded::parse(request.as_bytes()).collect(); + assert_eq!(params.get("resource").map(|s| s.as_ref()), resource); + assert_eq!(params.get("audience").map(|s| s.as_ref()), audience); + assert_eq!(params.get("grant_type").unwrap(), "refresh_token"); + let session = OAuthSession::new(config.clone()).unwrap(); + let status = session.status().await.unwrap(); + assert!(status.refreshable); + assert_eq!(status.resource, config.resource); + assert_eq!(status.audience, config.audience); + sessions.push(session); + } + assert!(sessions.pop().unwrap().logout().await.unwrap().removed); + for session in sessions { + assert!(session.status().await.unwrap().refreshable); + } + assert_eq!(idp.device_authorizations.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn test_legacy_cache_record_without_target_fields() { + let dir = cache_tempdir(); + let config = device_config(dir.path()); + let cache = TokenCache::new(&config, config.token_cache.as_ref().unwrap()).unwrap(); + let legacy = br#"{"version":1,"issuer_url":"https://issuer.example.com","client_id":"client-id","scopes":["openid"],"flow":"device_code","client_auth":"public","refresh_token":"legacy","obtained_at":1}"#; + write_record(dir.path(), &cache.record_path(), legacy).unwrap(); + let session = OAuthSession::new(config).unwrap(); + let status = session.status().await.unwrap(); + assert!(status.refreshable); + assert_eq!(status.resource, None); + assert_eq!(status.audience, None); + assert!(session.logout().await.unwrap().removed); + } + #[test] fn test_cache_key_canonicalizes_scopes_and_issuer() { let mut config = device_config(Path::new("/tmp/cache")); @@ -1744,6 +1877,8 @@ mod tests { issuer_url: "https://issuer.example.com".to_string(), client_id: "client-id".to_string(), scopes: vec!["openid".to_string()], + resource: None, + audience: None, flow: "device_code".to_string(), obtained_at: Some(100), }; From be3215a6ae2e24e2d39f0fb6bb8791ef0e1a45d5 Mon Sep 17 00:00:00 2001 From: mikemikimike <13286568797@163.com> Date: Wed, 16 Sep 2026 09:01:42 +0800 Subject: [PATCH 20/43] feat(nodejs): add JSON field helper (#4082) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Issue Fixes #4063 ## Background The Node.js SDK currently requires callers to know the Arrow extension metadata needed to represent JSON fields. This makes a common LanceDB schema type unnecessarily verbose and easy to get wrong. ## Changes - Add `makeJsonField(name, nullable = true)` to create a UTF-8 Arrow field with the `arrow.json` extension metadata. - Re-export the helper from the public Node.js entry point. - Add coverage for the default nullable behavior, explicit non-nullable fields, and the extension metadata. - Add the generated TypeDoc function page and public globals entry, including a usage example. ## Implementation The helper uses the existing Apache Arrow `Field` type and sets `ARROW:extension:name` to `arrow.json`, matching the metadata convention already used by LanceDB. ## Compatibility This is an additive Node.js API. Existing schema construction and Arrow behavior are unchanged. ## Verification - `pnpm test -- arrow.test.ts --runInBand` — 236 tests passed. - `pnpm exec biome ci lancedb/arrow.ts lancedb/index.ts __test__/arrow.test.ts` — passed. - `git diff --check` — passed. ## Not run / known limitations - `pnpm build` and `pnpm run docs` were attempted after expanding the checkout. Both are blocked locally by the native binding build/type declarations: Cargo did not complete, and TypeDoc reported the missing generated `nodejs/lancedb/native` module. The docs files were generated from the updated TypeScript comments; full build and docs validation are left to CI. --- docs/src/js/functions/makeJsonField.md | 36 ++++++++++++++++++++++++++ docs/src/js/globals.md | 1 + nodejs/__test__/arrow.test.ts | 16 ++++++++++++ nodejs/lancedb/arrow.ts | 24 +++++++++++++++++ nodejs/lancedb/index.ts | 1 + 5 files changed, 78 insertions(+) create mode 100644 docs/src/js/functions/makeJsonField.md diff --git a/docs/src/js/functions/makeJsonField.md b/docs/src/js/functions/makeJsonField.md new file mode 100644 index 000000000..bfa6fd6d7 --- /dev/null +++ b/docs/src/js/functions/makeJsonField.md @@ -0,0 +1,36 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / makeJsonField + +# Function: makeJsonField() + +```ts +function makeJsonField(name, nullable): Field +``` + +Create an Arrow field backed by LanceDB's JSON extension type. + +## Parameters + +* **name**: `string` + The field name. + +* **nullable**: `boolean` = `true` + Whether the field accepts null values. + +## Returns + +`Field` + +## Example + +```ts +import { connect, makeJsonField } from "@lancedb/lancedb"; +import { Schema } from "apache-arrow"; + +const schema = new Schema([makeJsonField("metadata")]); +const db = await connect("/path/to/database"); +await db.createTable("items", [{ metadata: '{"source":"api"}' }], { schema }); +``` diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 422c6c428..7e8344054 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -170,6 +170,7 @@ - [instrumentLanceDbMetrics](functions/instrumentLanceDbMetrics.md) - [isBlobField](functions/isBlobField.md) - [makeArrowTable](functions/makeArrowTable.md) +- [makeJsonField](functions/makeJsonField.md) - [packBits](functions/packBits.md) - [permutationBuilder](functions/permutationBuilder.md) - [tokenize](functions/tokenize.md) diff --git a/nodejs/__test__/arrow.test.ts b/nodejs/__test__/arrow.test.ts index 83b4fae46..6c1d5ecba 100644 --- a/nodejs/__test__/arrow.test.ts +++ b/nodejs/__test__/arrow.test.ts @@ -20,6 +20,7 @@ import { fromTableToBuffer, makeArrowTable, makeEmptyTable, + makeJsonField, } from "../lancedb/arrow"; import { EmbeddingFunction, @@ -28,6 +29,21 @@ import { import { EmbeddingFunctionConfig } from "../lancedb/embedding/registry"; import { sanitizeTable } from "../lancedb/sanitize"; +it("creates a nullable JSON field with the Arrow extension metadata", () => { + const field = makeJsonField("metadata"); + + expect(field.name).toBe("metadata"); + expect(field.type).toEqual(new arrow15.Utf8()); + expect(field.nullable).toBe(true); + expect(field.metadata).toEqual( + new Map([["ARROW:extension:name", "arrow.json"]]), + ); +}); + +it("allows JSON fields to be non-nullable", () => { + expect(makeJsonField("metadata", false).nullable).toBe(false); +}); + // biome-ignore lint/suspicious/noExplicitAny: skip function sampleRecords(): Array> { return [ diff --git a/nodejs/lancedb/arrow.ts b/nodejs/lancedb/arrow.ts index 81b140da7..df875a808 100644 --- a/nodejs/lancedb/arrow.ts +++ b/nodejs/lancedb/arrow.ts @@ -72,6 +72,30 @@ export type FieldLike = metadata?: Map; }; +/** + * Create an Arrow field backed by LanceDB's JSON extension type. + * + * @param name - The field name. + * @param nullable - Whether the field accepts null values. + * @example + * ```ts + * import { connect, makeJsonField } from "@lancedb/lancedb"; + * import { Schema } from "apache-arrow"; + * + * const schema = new Schema([makeJsonField("metadata")]); + * const db = await connect("/path/to/database"); + * await db.createTable("items", [{ metadata: '{"source":"api"}' }], { schema }); + * ``` + */ +export function makeJsonField(name: string, nullable = true): Field { + return new Field( + name, + new Utf8(), + nullable, + new Map([["ARROW:extension:name", "arrow.json"]]), + ); +} + export type DataLike = | import("apache-arrow").Data | { diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index a58cb182e..c30c5c5ba 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -72,6 +72,7 @@ export { export { makeArrowTable, + makeJsonField, MakeArrowTableOptions, Data, VectorColumnOptions, From f7579d2aae5a31563b0e6e749769f8d9c5319177 Mon Sep 17 00:00:00 2001 From: Shiduo Li Date: Tue, 15 Sep 2026 19:47:13 -0700 Subject: [PATCH 21/43] fix(python): correct pylance test extra pin (#4051) Closes #4045 ## What changed - replace the unpublished pylance 9.0.0rc1 test-extra pin with the published 9.0.0 release - restore dependency resolution for editable installs using the tests extra ## Validation - downloaded pylance==9.0.0 from PyPI with pip --no-deps - parsed python/pyproject.toml with tomllib and verified the tests extra - git diff --check --------- Co-authored-by: Xuanwo --- python/pyproject.toml | 2 +- python/uv.lock | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index ffdfdd948..dc5ea603d 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -63,7 +63,7 @@ tests = [ "polars>=0.19, <=1.32.3", "pyarrow<25", "pyarrow-stubs>=16.0", - "pylance==9.0.0rc1", + "pylance==9.0.0", "requests>=2.31.0", "datafusion>=54,<55", "opentelemetry-sdk>=1.30.0", diff --git a/python/uv.lock b/python/uv.lock index c957a4c06..8366dd397 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -2005,7 +2005,7 @@ requires-dist = [ { name = "pyarrow-stubs", marker = "extra == 'tests'", specifier = ">=16.0" }, { name = "pydantic", specifier = ">=2.7.4,<3" }, { name = "pylance", marker = "extra == 'pylance'", specifier = ">=5.0.0b5" }, - { name = "pylance", marker = "extra == 'tests'", specifier = "==9.0.0rc1" }, + { name = "pylance", marker = "extra == 'tests'", specifier = "==9.0.0" }, { name = "pyright", marker = "extra == 'dev'", specifier = ">=1.1.350" }, { name = "pytest", marker = "extra == 'tests'", specifier = ">=7.0" }, { name = "pytest-asyncio", marker = "extra == 'tests'", specifier = ">=0.21" }, @@ -3912,7 +3912,7 @@ crypto = [ [[package]] name = "pylance" -version = "7.0.0" +version = "9.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "lance-namespace" }, @@ -3922,12 +3922,12 @@ dependencies = [ { name = "pyarrow" }, ] wheels = [ - { url = "https://files.pythonhosted.org/packages/ac/ad/2f64921bf346e7075aef24a72595db44821724a3d89a9a92dd24e79632aa/pylance-7.0.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:98422021975be76e72b1572f41b8c9abb3bee5bdc9bfa5e9ce731110a65ed4d1", size = 62134146, upload-time = "2026-05-27T21:59:37.459Z" }, - { url = "https://files.pythonhosted.org/packages/73/1c/c5a01bee0160b55d9a98895cbd33091d038f0a0995b121ab72e629008d02/pylance-7.0.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4bec86ee5b6fbd8bfc493e653f0a1fba0303cfe5492b9b46fc25ab908edc7183", size = 65373684, upload-time = "2026-05-27T22:04:01.584Z" }, - { url = "https://files.pythonhosted.org/packages/eb/da/1fe8b8f7dbfe734d76af76acc994fc360a0d0c79a4874ef69f5a72a58fe3/pylance-7.0.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881491432c53184e52f8d1db8d5f872f39a03f36fb104bec77b33d379519d8b5", size = 69458555, upload-time = "2026-05-27T22:16:50.567Z" }, - { url = "https://files.pythonhosted.org/packages/76/f0/dd505cf3fd0226ab9d94759acd713125af1d3bfacfd80bbd52e3b9f89509/pylance-7.0.0-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:18453999e7fff4f76b16d6b7882c9df0628bd142ff95e2461bd7dd5ee3fe0af3", size = 65394430, upload-time = "2026-05-27T22:05:30.923Z" }, - { url = "https://files.pythonhosted.org/packages/17/ba/2357b81034f28eb00790e258ed140289a6a887a7468ca9df6349fd186b27/pylance-7.0.0-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:04a58051d408c60fe76d41a220dcaf8fea8fb6d1aa0ca78a709b60bc3cc8d19a", size = 69473470, upload-time = "2026-05-27T22:17:18.935Z" }, - { url = "https://files.pythonhosted.org/packages/1f/ec/5c00b6303a67d787f9475141832cbdc513d674ac3dcaeef8a7b169905e65/pylance-7.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:467d4864af047eaab4e1370e2f1e88e2c6f507c079874421116cb41d78bc3629", size = 74792863, upload-time = "2026-05-27T22:19:23.875Z" }, + { url = "https://files.pythonhosted.org/packages/e9/be/45733acd64801991852aac8e658601fd8fc12f76ceb81e57fca690896b90/pylance-9.0.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:8257213501d3298c5b6a344d60938e4bbe4de9f00cd3265371a56d1dc3dd15ca", size = 68377982, upload-time = "2026-07-24T16:53:45.247Z" }, + { url = "https://files.pythonhosted.org/packages/d0/3e/1ef707cb215cc7268c63ad84a91344ad6313d3984b343eeb19a9b708698e/pylance-9.0.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:804eedfa1fda2e703cca8580c76f0b44a1b849b725a8908c06f8acfca811732f", size = 71844362, upload-time = "2026-07-24T16:56:07.6Z" }, + { url = "https://files.pythonhosted.org/packages/c8/fb/a499e5c53ddb75c7de44100fd2bb1f7cc735966200d20292eed7af9ef552/pylance-9.0.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0a0b75595e3766c1d5f4c90abdc52337b4de60f1a52d123a4f8c4e5bcbbbfa8f", size = 75663088, upload-time = "2026-07-24T17:10:31.283Z" }, + { url = "https://files.pythonhosted.org/packages/e9/80/0714e09f64a68dbdf62558955e737a7436df763b9834a6aba506861b5352/pylance-9.0.0-cp310-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:d2f69c5c390ae3710c35a429905fe15f769951777fafb1b72a359e035ec121f7", size = 71866858, upload-time = "2026-07-24T16:56:35.937Z" }, + { url = "https://files.pythonhosted.org/packages/4b/3c/78d3a6d6ca0d843b7c3ac0c30d9cd2cf4635b0c2cabe6ec66583d5bfc1a1/pylance-9.0.0-cp310-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:836268a7832d62f3d5ccbe1c3fca239621971d90297bcfff14a70b3cb6842aa8", size = 75642656, upload-time = "2026-07-24T17:13:11.879Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c1/dc9c9a31e171530ec0add024d922488c046437d32a7087c29e254a7eacc7/pylance-9.0.0-cp310-abi3-win_amd64.whl", hash = "sha256:96441d27a5ed3805300388ccf8f31835cd280c98751f80b6a1a11dcd6808fc43", size = 81668288, upload-time = "2026-07-24T17:05:38.707Z" }, ] [[package]] From 3b37ea2c7a877ba76de5f539d676c1e15cac12fb Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 16 Sep 2026 12:38:59 +0800 Subject: [PATCH 22/43] feat: represent registered Functions by OCI image identity (#4176) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Function versions identify independent Function objects and their numeric revisions. Rust and Python expose the object ID, location, canonical decimal version, metadata, and availability separately from the OCI image digest. Computed-column applications carry the complete object reference, so existing bindings retain their identity after a name is removed and reused. Source authoring keeps the existing create_function/create_function_async, job wait, and column-binding APIs. The server coordinates baking followed by registration; users do not have to manage manifest digests to create a Function. The previous stored Function representation is intentionally unsupported. Existing contract tests and shared wire fixtures are migrated to the new model. This SDK change accompanies the final integration layer https://github.com/lancedb/sophon/pull/7887 in the Sophon stack: https://github.com/lancedb/sophon/pull/7885 → https://github.com/lancedb/sophon/pull/7886 → https://github.com/lancedb/sophon/pull/7887. A metadata-only revision can retain the same executable image; Function version numbers must not be used as image cache keys. --------- Co-authored-by: lancedb automation Co-authored-by: Yang Cen Co-authored-by: Claude Fable 5.1 --- Cargo.toml | 2 + docs/src/python/python.md | 2 + python/python/lancedb/db.py | 26 ++--- python/python/lancedb/functions.py | 53 +++++++--- .../tests/test_first_class_function_slice1.py | 39 +++++--- .../tests/test_first_class_function_slice2.py | 47 +++++---- rust/lancedb/Cargo.toml | 9 +- rust/lancedb/src/connection.rs | 9 +- rust/lancedb/src/database.rs | 4 +- rust/lancedb/src/function.rs | 96 +++++++++++++------ rust/lancedb/src/materialized_view.rs | 6 +- rust/lancedb/src/remote/db.rs | 18 ++-- rust/lancedb/src/remote/job.rs | 2 +- rust/lancedb/src/remote/table.rs | 8 +- rust/lancedb/src/table/computed_columns.rs | 28 +++--- .../tests/first_class_function_slice1.rs | 34 ++++--- .../tests/first_class_function_slice2.rs | 4 +- ...remote_fixed_size_declaration_request.json | 2 +- ...remote_function_application.canonical.json | 2 +- .../v1/remote_function_application.json | 2 +- .../v1/remote_function_application_float.json | 2 +- .../v1/remote_function_binding.canonical.json | 2 +- .../v1/remote_function_binding.json | 2 +- .../v1/remote_function_job.json | 74 ++++++++++---- .../v1/remote_function_version.canonical.json | 2 +- ...mote_multi_output_declaration_request.json | 2 +- .../v1/remote_scalar_declaration_request.json | 2 +- 27 files changed, 314 insertions(+), 165 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e6560dbf7..092180596 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,8 @@ arrow-select = "58.0.0" arrow-cast = "58.0.0" arrow-flight = { version = "58.0.0", features = ["flight-sql-experimental"] } async-trait = "0" +# Smithy JSON 0.63 requires the pre-1.7 Document representation; allow the MSRV pin. +aws-smithy-types = ">=1.3.6, <1.7" bytes = "1" datafusion = { version = "54.0.0", default-features = false } datafusion-catalog = "54.0.0" diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 28e774473..d14e5cf94 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -133,6 +133,8 @@ listing a storage directory. ::: lancedb.functions.PythonAdapterSpec +::: lancedb.functions.FunctionImage + ::: lancedb.functions.FunctionVersion ::: lancedb.functions.PythonRuntimeSpec diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index d09db80c3..4a8820cef 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -743,19 +743,20 @@ class DBConnection(EnforceOverrides): raise NotImplementedError("serialize is not supported for this connection type") def create_function(self, definition: UdfDefinition) -> FunctionVersion: - """Register a scalar Python UDF and wait for its immutable version. + """Build and register a scalar Python UDF, then return its version. + The server builds the OCI image and registers the completed artifact. This is the blocking counterpart of :meth:`create_function_async`. Local connections raise ``NotImplementedError``. """ return self.create_function_async(definition).wait() def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]: - """Register a scalar Python UDF through the remote Function catalog. + """Submit a scalar Python UDF for building and registration. - Submission returns a typed job. The immutable Function version becomes - available only when :meth:`Job.wait` succeeds. Local connections raise - ``NotImplementedError``. + The server-side job builds the OCI image, then registers the completed + artifact. Waiting on the job returns the immutable Function version. + Local connections raise ``NotImplementedError``. """ raise NotImplementedError( "Function catalog operations are not supported for this connection type" @@ -786,10 +787,12 @@ class DBConnection(EnforceOverrides): ) def drop_function(self, name: str, *, version: str) -> bool: - """Drop one exact immutable Function version from the remote catalog. + """Remove the current Function name binding from the remote catalog. - Returns True when the version changed to Dropped and False for an - idempotent replay. Local connections raise NotImplementedError. + The requested version must exist in the currently named object. + Object history and existing computed-column references are retained. + Returns True when the name was removed and False when it was absent. + Local connections raise NotImplementedError. """ raise NotImplementedError( "Function catalog operations are not supported for this connection type" @@ -2421,9 +2424,10 @@ class AsyncConnection(object): async def create_function_async( self, definition: UdfDefinition ) -> AsyncJob[FunctionVersion]: - """Register a scalar Python UDF through the remote Function catalog. + """Submit a scalar Python UDF for building and registration. - The returned typed job resolves to the immutable Function version. + The server-side job builds the OCI image, then registers the completed + artifact. Waiting on the job returns the immutable Function version. Local connections raise ``NotImplementedError``. """ if not isinstance(definition, UdfDefinition): @@ -2449,7 +2453,7 @@ class AsyncConnection(object): ] async def drop_function(self, name: str, *, version: str) -> bool: - """Drop one exact immutable Function version from the remote catalog.""" + """Remove the current name binding, retaining the object and its history.""" return await self._inner.drop_function(name, version) async def list_jobs(self) -> List[JobInfo]: diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index d3fa6da81..692bc5756 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -41,6 +41,7 @@ from typing import ( import pyarrow as pa from pydantic import ( + AfterValidator, BaseModel, ConfigDict, Field, @@ -295,21 +296,39 @@ class PythonRuntimeSpec(_RemoteValue): return self -class FunctionVersion(_RemoteValue): - """An exact immutable Function version returned by Enterprise. +class FunctionImage(_RemoteValue): + """A complete OCI Function image identified by its exact manifest digest.""" - The GPU execution requirement is part of this identity. CPU and memory sizing, - priority, concurrency, and retry policy belong to the execution platform. - """ + manifest_digest: str + descriptor: Mapping[str, Any] + source: bool + + +def _validate_object_version(value: str) -> str: + if int(value) > 2**64 - 1: + raise ValueError("Function version exceeds uint64") + return value + + +_ObjectVersion = Annotated[ + str, + Field(strict=True, pattern=r"^[1-9][0-9]*$"), + AfterValidator(_validate_object_version), +] + + +class FunctionVersion(_RemoteValue): + """A pinned object revision, independent of its executable image digest.""" name: str - version: str - artifact: FunctionArtifact + object_id: str + location: str + version: _ObjectVersion + image: FunctionImage signature: FunctionSignature - runtime: PythonRuntimeSpec - runtime_digest: str - environment_digest: str created_at: str + metadata: Mapping[str, str] + disabled: bool def __call__(self, **inputs: Any) -> FunctionApplication: """Bind this exact version to named table columns. @@ -363,7 +382,13 @@ class FunctionVersion(_RemoteValue): ) ) return FunctionApplication( - function=FunctionVersionRef(name=self.name, version=self.version), + function=FunctionVersionRef( + name=self.name, + object_id=self.object_id, + location=self.location, + version=self.version, + manifest_digest=self.image.manifest_digest, + ), inputs=tuple(bindings), output=self.signature.output, ) @@ -380,7 +405,10 @@ class FunctionRegistrationRequest(_RemoteValue): class FunctionVersionRef(_OpenRemoteValue): name: str - version: str + object_id: str + location: str + version: _ObjectVersion + manifest_digest: str class ApplicationInput(_OpenRemoteValue): @@ -1402,6 +1430,7 @@ __all__ = [ "FunctionRegistrationRequest", "FunctionResultField", "FunctionSignature", + "FunctionImage", "FunctionVersion", "FunctionVersionRef", "InputBinding", diff --git a/python/python/tests/test_first_class_function_slice1.py b/python/python/tests/test_first_class_function_slice1.py index 89172ba3f..459212b0f 100644 --- a/python/python/tests/test_first_class_function_slice1.py +++ b/python/python/tests/test_first_class_function_slice1.py @@ -93,16 +93,22 @@ def test_function_version_identity_is_immutable_and_exact(): value = job_result("remote_function_job.json") version = FunctionVersion.from_json(json.dumps(value)) assert version.name == "embed" - assert version.version == "fv_01K3EXACT" + assert version.version == "1" + assert version.image.manifest_digest.startswith("sha256:") + assert version.version != version.image.manifest_digest with pytest.raises((TypeError, ValueError)): - version.version = "fv_changed" + version.version = "1" with pytest.raises(TypeError, match="immutable"): - version.runtime.env["TOKENIZERS_PARALLELISM"] = "true" + version.image.descriptor["format_version"] = "changed" changed = dict(value) - changed["version"] = "fv_changed" + changed["version"] = "2" assert FunctionVersion(**changed) != version + assert FunctionVersion(**changed).image == version.image + for invalid in [version.image.manifest_digest, "0", "01", "-1", str(2**64)]: + with pytest.raises(ValueError): + FunctionVersion(**{**value, "version": invalid}) def test_function_version_binds_named_columns_as_one_immutable_application(): @@ -137,7 +143,7 @@ def test_function_version_binding_validates_names_and_direct_columns(): def test_function_version_keeps_named_struct_outputs_in_one_application(): value = job_result("remote_function_job.json") value["name"] = "text_features" - value["version"] = "fv_multi_output" + value["version"] = "1" value["signature"] = { "inputs": [ {"name": "title", "arrow_type": "utf8", "nullable": True}, @@ -182,14 +188,14 @@ def test_function_version_keeps_named_struct_outputs_in_one_application(): def test_unknown_fields_and_discriminators_are_forward_decodable(): value = job_result("remote_function_job.json") value["future_version_metadata"] = {"retention_class": "catalog"} - value["runtime"] = {"kind": "wasm", "module_digest": "sha256:wasm"} + value["image"]["descriptor"]["future_interface"] = {"kind": "wasm"} value["signature"]["output"]["kind"] = "future_output_shape" version = FunctionVersion.from_json(json.dumps(value)) - assert version.runtime.kind == "wasm" - assert version.runtime.python_version is None - assert version.runtime.environment is None - assert json.loads(version.to_canonical_json())["runtime"] == {"kind": "wasm"} + assert version.image.descriptor["future_interface"] == {"kind": "wasm"} + assert json.loads(version.to_canonical_json())["image"]["descriptor"][ + "future_interface" + ] == {"kind": "wasm"} assert version.signature.output.kind == "future_output_shape" @@ -222,7 +228,7 @@ def test_function_application_uses_rename_columns_only(): def test_binding_and_refresh_result_keep_stable_remote_fields(): binding = FunctionBinding.from_json(fixture("remote_function_binding.json")) - assert binding.function.version == "fv_01K3TEXT" + assert binding.function.version == "1" assert [output.output_ordinal for output in binding.outputs] == [0, 1] assert binding.input_schema is not None assert binding.output_schema is not None @@ -339,7 +345,16 @@ def test_rename_requires_named_struct_and_keeps_partial_mapping_immutable(): scalar = FunctionApplication.from_json( json.dumps( { - "function": {"name": "embed", "version": "fv_exact"}, + "function": { + "name": "embed", + "version": "1", + "object_id": "fixture", + "location": "memory:///fixture", + "manifest_digest": ( + "sha256:" + "7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6" + ), + }, "inputs": [], "output": { "kind": "scalar", diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index 900f69f8d..8d4f75f67 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -45,6 +45,11 @@ FIXTURES = ( ) +FUNCTION_VERSION = json.loads( + (FIXTURES / "remote_function_version.canonical.json").read_text() +)["version"] + + @udf( pip=["numpy>=2"], env={"MODE": "test"}, @@ -1199,11 +1204,11 @@ def test_local_function_catalog_operations_are_not_supported(tmp_path): with pytest.raises(NotImplementedError, match=message): db.create_function_async(normalize_score) with pytest.raises(NotImplementedError, match=message): - db.get_function("normalize_score", version="fv_exact") + db.get_function("normalize_score", version=FUNCTION_VERSION) with pytest.raises(NotImplementedError, match=message): db.list_functions() with pytest.raises(NotImplementedError, match=message): - db.drop_function("normalize_score", version="fv_exact") + db.drop_function("normalize_score", version=FUNCTION_VERSION) @contextlib.contextmanager @@ -1230,15 +1235,17 @@ def _mock_remote_function_catalog(): if self.path == "/v1/function/normalize_score/create": state["version"] = { "name": "normalize_score", - "version": "fv_exact", - "artifact": { - key: body["artifact"][key] - for key in ("kind", "digest", "entrypoint") - }, + "version": FUNCTION_VERSION, + "object_id": "fixture", + "location": "memory:///fixture", + "metadata": {}, + "disabled": False, + "image": json.loads( + ( + FIXTURES / "remote_function_version.canonical.json" + ).read_text() + )["image"], "signature": body["signature"], - "runtime": body["runtime"], - "runtime_digest": "sha256:runtime", - "environment_digest": "sha256:environment", "created_at": "2026-08-21T00:00:00Z", } response = {"job_id": "job-register"} @@ -1252,10 +1259,10 @@ def _mock_remote_function_catalog(): "result": state["version"], } elif self.path == "/v1/function/normalize_score/describe": - assert body == {"version": "fv_exact"} + assert body == {"version": FUNCTION_VERSION} response = state["version"] elif self.path == "/v1/function/normalize_score/drop": - assert body == {"version": "fv_exact"} + assert body == {"version": FUNCTION_VERSION} response = {"dropped": True} else: status = 404 @@ -1278,7 +1285,7 @@ def _mock_remote_function_catalog(): "functions": [ { "name": "normalize_score", - "version": "fv_exact", + "version": FUNCTION_VERSION, "definition": state["version"], } ], @@ -1314,7 +1321,7 @@ def test_remote_registration_job_and_exact_version_reopen_round_trip(): assert created == reopened assert reopened.name == "normalize_score" - assert reopened.version == "fv_exact" + assert reopened.version == FUNCTION_VERSION create_request = state["requests"][0][1] expected_request = json.loads( normalize_score.registration_request.to_canonical_json() @@ -1334,7 +1341,7 @@ def test_blocking_remote_registration_returns_function_version(): created = db.create_function(normalize_score) assert created.name == "normalize_score" - assert created.version == "fv_exact" + assert created.version == FUNCTION_VERSION assert [path for path, _ in state["requests"]] == [ "/v1/function/normalize_score/create", "/v1/jobs/describe", @@ -1392,12 +1399,12 @@ def test_remote_drop_function_sends_exact_version(): host_override=host, client_config={"retry_config": {"retries": 0}}, ) - assert db.drop_function("normalize_score", version="fv_exact") is True + assert db.drop_function("normalize_score", version=FUNCTION_VERSION) is True assert state["requests"] == [ ( "/v1/function/normalize_score/drop", - {"version": "fv_exact"}, + {"version": FUNCTION_VERSION}, ) ] @@ -1411,11 +1418,13 @@ async def test_async_remote_drop_function_sends_exact_version(): host_override=host, client_config={"retry_config": {"retries": 0}}, ) - assert await db.drop_function("normalize_score", version="fv_exact") is True + assert ( + await db.drop_function("normalize_score", version=FUNCTION_VERSION) is True + ) assert state["requests"] == [ ( "/v1/function/normalize_score/drop", - {"version": "fv_exact"}, + {"version": FUNCTION_VERSION}, ) ] diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 2c3d6dd84..d798d8006 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -67,6 +67,7 @@ serde_json = { workspace = true } async-openai = { version = "0.20.0", optional = true } serde_with = { version = "3.8.1" } tempfile = { workspace = true } +aws-smithy-types = { workspace = true, optional = true } aws-sdk-bedrockruntime = { version = "1.27.0", optional = true } # For remote feature reqwest = { version = "0.12.0", default-features = false, features = [ @@ -114,10 +115,7 @@ aws-sdk-s3 = { version = "1.55.0" } aws-sdk-kms = { version = "1.48.0" } aws-config = { version = "1.5.10" } aws-smithy-runtime = { version = "1.9.1" } -# Constraint only: types 1.7 breaks aws-smithy-json 0.63, which aws-config still -# requires. Bounds must stay inside 1.x and allow the MSRV job's 1.3.6 pin. -# Drop once aws-config moves to aws-smithy-json 0.64. -aws-smithy-types = { version = ">=1.0, <1.7" } +aws-smithy-types.workspace = true datafusion.workspace = true http-body = "1" # Matching reqwest rstest = "0.23.0" @@ -130,6 +128,7 @@ pprof = { version = "0.14", features = ["flamegraph"] } [features] default = [] aws = [ + "dep:aws-smithy-types", "lance/aws", "lance-io/aws", "lance-namespace-impls/dir-aws", @@ -179,7 +178,7 @@ metrics = ["dep:metrics", "lance/metrics", "lance-io/metrics"] metrics-otel = ["metrics", "dep:metrics-util"] fp16kernels = ["lance-linalg/fp16kernels"] s3-test = [] -bedrock = ["dep:aws-sdk-bedrockruntime"] +bedrock = ["dep:aws-sdk-bedrockruntime", "dep:aws-smithy-types"] openai = ["dep:async-openai", "dep:reqwest"] polars = ["dep:polars-arrow", "dep:polars"] sentence-transformers = [ diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index e136a0923..a80df819b 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -581,10 +581,11 @@ impl Connection { ) } - /// Register a Python callable as a new immutable Function version. + /// Build and register a Python callable as an immutable Function version. /// - /// Registration is remote-only and always asynchronous. Waiting on the - /// returned typed job yields the durable [`crate::function::FunctionVersion`]. + /// The server-side job builds the OCI image, then registers the completed + /// artifact. Waiting on the returned typed job yields the durable + /// [`crate::function::FunctionVersion`]. Creation is remote-only. /// Local databases return [`Error::NotSupported`]. pub async fn create_function_async( &self, @@ -630,7 +631,7 @@ impl Connection { self.internal.list_functions().await } - /// Drop one exact immutable Function version from the remote catalog. + /// Remove the current Function name binding, retaining the object history. /// /// Returns `true` when the server appended a Dropped transition and /// `false` for an idempotent replay. Local databases return diff --git a/rust/lancedb/src/database.rs b/rust/lancedb/src/database.rs index b8d19443b..d31f40b29 100644 --- a/rust/lancedb/src/database.rs +++ b/rust/lancedb/src/database.rs @@ -296,7 +296,7 @@ pub trait Database: /// /// See [`CloneTableRequest`] for detailed documentation and examples. async fn clone_table(&self, request: CloneTableRequest) -> Result>; - /// Register an immutable Function version through the remote catalog. + /// Submit a Function creation job that builds an image and registers it. async fn create_function_async( &self, _request: crate::function::FunctionRegistrationRequest, @@ -382,7 +382,7 @@ pub trait Database: async fn list_functions(&self) -> Result> { function_catalog_not_supported() } - /// Drop one exact immutable Function version from the remote catalog. + /// Remove the current Function name binding, retaining the object history. async fn drop_function(&self, _name: &str, _version: &str) -> Result { function_catalog_not_supported() } diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index e67d763ac..d501541c0 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -88,10 +88,18 @@ fn application_has_unknown_nested_fields(value: &Value) -> bool { let Some(application) = value.as_object() else { return false; }; - if application - .get("function") - .is_some_and(|value| has_unknown_keys(value, &["name", "version"])) - { + if application.get("function").is_some_and(|value| { + has_unknown_keys( + value, + &[ + "name", + "object_id", + "location", + "version", + "manifest_digest", + ], + ) + }) { return true; } if application @@ -396,51 +404,75 @@ impl Serialize for PythonRuntimeSpec { } } -/// Immutable Function version returned by the Enterprise catalog. -/// -/// The GPU execution requirement is part of this identity. CPU and memory sizing, -/// priority, concurrency, and retry policy belong to the execution platform. +/// A complete OCI Function image. Its digest is independent of catalog names. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FunctionImage { + pub manifest_digest: String, + pub descriptor: Value, + pub source: bool, +} + +fn deserialize_object_version<'de, D: Deserializer<'de>>( + deserializer: D, +) -> std::result::Result { + let value = String::deserialize(deserializer)?; + match value.parse::() { + Ok(number) if number > 0 && number.to_string() == value => Ok(value), + _ => Err(de::Error::custom( + "Function version must be a canonical positive uint64", + )), + } +} + +/// One immutable Function object revision and its executable artifact. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FunctionVersion { name: String, + object_id: String, + location: String, + #[serde(deserialize_with = "deserialize_object_version")] version: String, - artifact: FunctionArtifact, + image: FunctionImage, signature: FunctionSignature, - runtime: PythonRuntimeSpec, - runtime_digest: String, - environment_digest: String, created_at: String, + metadata: BTreeMap, + disabled: bool, } impl FunctionVersion { + pub fn object_id(&self) -> &str { + &self.object_id + } + pub fn location(&self) -> &str { + &self.location + } + pub fn metadata(&self) -> &BTreeMap { + &self.metadata + } + pub fn disabled(&self) -> bool { + self.disabled + } + pub fn reference(&self) -> FunctionVersionRef { + FunctionVersionRef { + name: self.name.clone(), + object_id: self.object_id.clone(), + location: self.location.clone(), + version: self.version.clone(), + manifest_digest: self.image.manifest_digest.clone(), + } + } pub fn name(&self) -> &str { &self.name } - pub fn version(&self) -> &str { &self.version } - - pub fn artifact(&self) -> &FunctionArtifact { - &self.artifact + pub fn image(&self) -> &FunctionImage { + &self.image } - pub fn signature(&self) -> &FunctionSignature { &self.signature } - - pub fn runtime(&self) -> &PythonRuntimeSpec { - &self.runtime - } - - pub fn runtime_digest(&self) -> &str { - &self.runtime_digest - } - - pub fn environment_digest(&self) -> &str { - &self.environment_digest - } - pub fn created_at(&self) -> &str { &self.created_at } @@ -496,7 +528,11 @@ impl_json!(FunctionRegistrationRequest); #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FunctionVersionRef { pub name: String, + pub object_id: String, + pub location: String, + #[serde(deserialize_with = "deserialize_object_version")] pub version: String, + pub manifest_digest: String, } /// Parameter binding in a FunctionApplication. diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index 342e89bfa..94fcb5dda 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -2775,7 +2775,8 @@ mod tests { FunctionBinding::from_json( &serde_json::json!({ "binding_id": binding_id, - "function": {"name": "embed", "version": "fv_test"}, + "function": {"name": "embed", "version": "1", "object_id": "fixture", "location": "memory:///fixture", + "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs": [{ "parameter": "text", "field_id": -1, "field_path": input, "arrow_type": input_type, "nullable": true, @@ -3052,7 +3053,8 @@ mod tests { let binding = FunctionBinding::from_json( &serde_json::json!({ "binding_id": "fb_pair", - "function": {"name": "pair", "version": "fv_test"}, + "function": {"name": "pair", "version": "1", "object_id": "fixture", "location": "memory:///fixture", + "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs": [{"parameter": "value", "field_id": -1, "field_path": "id", "arrow_type": int, "nullable": true}], "outputs": [ diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index 7180b04ce..c24ede4e1 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -3085,7 +3085,7 @@ mod tests { assert_eq!(job.id(), Some("job-function-1")); let version = job.wait().await.unwrap(); assert_eq!(version.name(), "embed"); - assert_eq!(version.version(), "fv_01K3EXACT"); + assert_eq!(version.version(), "1"); } #[tokio::test] @@ -3098,12 +3098,12 @@ mod tests { assert_eq!(request.url().path(), "/v1/function/embed/describe"); let body: serde_json::Value = serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); - assert_eq!(body, serde_json::json!({"version": "fv_01K3EXACT"})); + assert_eq!(body, serde_json::json!({"version": "1"})); http::Response::builder().status(200).body(VERSION).unwrap() }); - let version = conn.get_function("embed", "fv_01K3EXACT").await.unwrap(); + let version = conn.get_function("embed", "1").await.unwrap(); assert_eq!(version.name(), "embed"); - assert_eq!(version.version(), "fv_01K3EXACT"); + assert_eq!(version.version(), "1"); } #[tokio::test] @@ -3134,7 +3134,7 @@ mod tests { serde_json::json!({ "functions": [{ "name": "embed", - "version": "fv_01K3EXACT", + "version": "1", "definition": version.clone(), }], }) @@ -3147,7 +3147,7 @@ mod tests { let functions = conn.list_functions().await.unwrap(); assert_eq!(functions.len(), 1); assert_eq!(functions[0].name(), "embed"); - assert_eq!(functions[0].version(), "fv_01K3EXACT"); + assert_eq!(functions[0].version(), "1"); } #[tokio::test] @@ -3229,13 +3229,13 @@ mod tests { assert_eq!(request.url().path(), "/v1/function/embed/drop"); let body: serde_json::Value = serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); - assert_eq!(body, serde_json::json!({"version": "fv_01K3EXACT"})); + assert_eq!(body, serde_json::json!({"version": "1"})); http::Response::builder() .status(200) .body(r#"{"dropped":false}"#) .unwrap() }); - assert!(!conn.drop_function("embed", "fv_01K3EXACT").await.unwrap()); + assert!(!conn.drop_function("embed", "1").await.unwrap()); } #[tokio::test] @@ -3254,7 +3254,7 @@ mod tests { http::Response::builder() .status(200) .body(format!( - r#"{{"job_id": "job-1", "job_type": "create_function", "job_state": "{}", "creation_ms": 1, "result": {{"name": "embed", "version": "fv_1"}}}}"#, + r#"{{"job_id": "job-1", "job_type": "create_function", "job_state": "{}", "creation_ms": 1, "result": {{"name": "embed", "version": "1"}}}}"#, state )) .unwrap() diff --git a/rust/lancedb/src/remote/job.rs b/rust/lancedb/src/remote/job.rs index c7acc3c05..80256421a 100644 --- a/rust/lancedb/src/remote/job.rs +++ b/rust/lancedb/src/remote/job.rs @@ -285,7 +285,7 @@ mod tests { async fn typed_remote_job_fixtures_decode_terminal_results() { let function = Job::::new_typed(Box::new(FixtureRemoteJob(FUNCTION_JOB))); let result = function.wait().await.expect("typed FunctionVersion result"); - assert_eq!(result.version(), "fv_01K3EXACT"); + assert_eq!(result.version(), "1"); let refresh = Job::::new_typed(Box::new(FixtureRemoteJob(REFRESH_JOB))); diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 747e2b73c..dbabf9845 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -7856,7 +7856,7 @@ mod tests { }); let application = crate::function::FunctionApplication::from_json( r#"{ - "function":{"name":"embed","version":"fv_01K3EXACT"}, + "function":{"name":"embed","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}], "output":{"kind":"scalar","arrow_type":"list","nullable":false} }"#, @@ -7933,7 +7933,7 @@ mod tests { }); let application = crate::function::FunctionApplication::from_json( r#"{ - "function":{"name":"text_features","version":"fv_01K3TEXT"}, + "function":{"name":"text_features","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs":[ {"parameter":"title","kind":"column","value":{"path":"title"}}, {"parameter":"body","kind":"column","value":{"path":"body"}} @@ -7989,7 +7989,7 @@ mod tests { }); let application = crate::function::FunctionApplication::from_json( r#"{ - "function":{"name":"embed","version":"fv_01K3EXACT"}, + "function":{"name":"embed","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs":[{"parameter":"text","kind":"column","value":{"path":"description"}}], "output":{"kind":"scalar","arrow_type":"fixed_size_list","nullable":false} }"#, @@ -8034,7 +8034,7 @@ mod tests { }); let application = crate::function::FunctionApplication::from_json( r#"{ - "function":{"name":"text_features","version":"fv_01K3TEXT"}, + "function":{"name":"text_features","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs":[ {"parameter":"title","kind":"column","value":{"path":"title"}}, {"parameter":"body","kind":"column","value":{"path":"body"}} diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index a70ac31ae..f87d42426 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -539,7 +539,13 @@ fn ensure_known_binding_shape(value: &Value) -> Result<()> { object .get("function") .ok_or_else(|| invalid_function("Function binding is missing its exact version"))?, - &["name", "version"], + &[ + "name", + "object_id", + "location", + "version", + "manifest_digest", + ], "version reference", )?; for input in object @@ -3022,7 +3028,7 @@ mod tests { fn named_struct_application(columns: &str) -> FunctionApplication { FunctionApplication::from_json(&format!( r#"{{ - "function":{{"name":"text_features","version":"fv_exact"}}, + "function":{{"name":"text_features","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}}, "inputs":[ {{"parameter":"title","kind":"column","value":{{"path":"title"}}}}, {{"parameter":"body","kind":"column","value":{{"path":"body"}}}} @@ -3040,7 +3046,7 @@ mod tests { fn blob_application(output: &str) -> FunctionApplication { FunctionApplication::from_json(&format!( r#"{{ - "function":{{"name":"blob_features","version":"fv_blob"}}, + "function":{{"name":"blob_features","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}}, "inputs":[ {{"parameter":"image","kind":"column","value":{{"path":"image"}}}} ], @@ -3059,7 +3065,7 @@ mod tests { fn single_input_application(path: &str) -> FunctionApplication { FunctionApplication::from_json( &serde_json::json!({ - "function": {"name": "inspect", "version": "fv_nested_blob"}, + "function": {"name": "inspect", "version": "1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs": [{ "parameter": "value", "kind": "column", @@ -3401,7 +3407,7 @@ mod tests { )); let dependent_application = FunctionApplication::from_json( r#"{ - "function":{"name":"dependent","version":"fv_dependent"}, + "function":{"name":"dependent","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs":[ {"parameter":"text","kind":"column","value":{"path":"search_text"}} ], @@ -3532,7 +3538,7 @@ mod tests { let input = ArrowField::new("value", DataType::Int64, false); let application = FunctionApplication::from_json( &serde_json::json!({ - "function": {"name": "embed", "version": "fv_embed"}, + "function": {"name": "embed", "version": "1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs": [{ "parameter": "value", "kind": "column", @@ -3737,7 +3743,7 @@ mod tests { )); let application = FunctionApplication::from_json( &serde_json::json!({ - "function": {"name": "inspect", "version": "fv_nested_blob"}, + "function": {"name": "inspect", "version": "1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs": [], "output": { "kind": "named_struct", @@ -3869,7 +3875,7 @@ mod tests { fn test_unknown_and_mixed_version_function_contracts_fail_closed() { let application = FunctionApplication::from_json( r#"{ - "function":{"name":"f","version":"fv"}, + "function":{"name":"f","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs":[{"parameter":"title","kind":"future_source","value":{"path":"title"}}], "output":{"kind":"scalar","arrow_type":"int64","nullable":false} }"#, @@ -3881,7 +3887,7 @@ mod tests { let future_application = FunctionApplication::from_json( r#"{ - "function":{"name":"f","version":"fv"}, + "function":{"name":"f","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs":[], "output":{"kind":"scalar","arrow_type":"int64","nullable":false}, "future_declaration":{"mode":"managed"} @@ -3895,7 +3901,7 @@ mod tests { let nested_future_application = FunctionApplication::from_json( r#"{ - "function":{"name":"f","version":"fv"}, + "function":{"name":"f","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs":[], "output":{"kind":"scalar","arrow_type":"int64","nullable":false,"assignment":"cell_flag"} }"#, @@ -3961,7 +3967,7 @@ mod tests { let nested_schema = ArrowSchema::new(vec![nested_title, schema.field(1).as_ref().clone()]); let nested_application = FunctionApplication::from_json( r#"{ - "function":{"name":"text_features","version":"fv_exact"}, + "function":{"name":"text_features","version":"1","object_id":"fixture","location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs":[ {"parameter":"title","kind":"column","value":{"path":"title.value"}}, {"parameter":"body","kind":"column","value":{"path":"body"}} diff --git a/rust/lancedb/tests/first_class_function_slice1.rs b/rust/lancedb/tests/first_class_function_slice1.rs index ce020bd53..b70ac7f0a 100644 --- a/rust/lancedb/tests/first_class_function_slice1.rs +++ b/rust/lancedb/tests/first_class_function_slice1.rs @@ -26,8 +26,8 @@ fn function_version_job_result_matches_shared_canonical_golden() { let version = FunctionVersion::from_json(&result.to_string()).expect("FunctionVersion result"); assert_eq!(version.name(), "embed"); - assert_eq!(version.version(), "fv_01K3EXACT"); - assert_eq!(version.runtime_digest(), "sha256:runtime"); + assert_eq!(version.version(), "1"); + assert_ne!(version.image().manifest_digest, version.version()); assert_eq!( version.to_canonical_json().expect("canonical JSON"), fixture("remote_function_version.canonical.json").trim() @@ -45,16 +45,28 @@ fn version_identity_is_immutable_and_exact() { assert_eq!(reopened.version(), version.version()); let mut changed = original; - changed["version"] = Value::String("fv_01K3DIFFERENT".to_string()); + changed["version"] = Value::String("2".to_string()); let changed = FunctionVersion::from_json(&changed.to_string()).expect("changed version"); assert_ne!(changed, version); + assert_eq!(changed.image(), version.image()); + for invalid in [ + version.image().manifest_digest.as_str(), + "0", + "01", + "-1", + "18446744073709551616", + ] { + let mut value = serde_json::to_value(&version).unwrap(); + value["version"] = Value::String(invalid.into()); + assert!(FunctionVersion::from_json(&value.to_string()).is_err()); + } } #[test] fn application_and_binding_match_shared_remote_goldens() { let application = FunctionApplication::from_json(&fixture("remote_function_application.json")) .expect("application fixture"); - assert_eq!(application.function().version, "fv_01K3TEXT"); + assert_eq!(application.function().version, "1"); assert_eq!(application.output().kind, "named_struct"); assert_eq!(application.inputs().len(), 2); assert_eq!( @@ -64,7 +76,7 @@ fn application_and_binding_match_shared_remote_goldens() { let binding = FunctionBinding::from_json(&fixture("remote_function_binding.json")) .expect("binding fixture"); - assert_eq!(binding.function().version, "fv_01K3TEXT"); + assert_eq!(binding.function().version, "1"); assert_eq!(binding.outputs()[0].output_ordinal, 0); assert_eq!(binding.outputs()[1].output_ordinal, 1); assert!(binding.input_schema().is_some()); @@ -113,22 +125,18 @@ fn refresh_job_result_matches_shared_canonical_golden() { fn unknown_fields_and_discriminators_are_forward_decodable() { let mut result = job_result("remote_function_job.json"); result["future_version_metadata"] = serde_json::json!({"retention_class": "catalog"}); - result["runtime"] = serde_json::json!({ - "kind": "wasm", - "module_digest": "sha256:wasm" - }); + result["image"]["descriptor"]["future_interface"] = serde_json::json!({"version": 2}); result["signature"]["output"]["kind"] = Value::String("future_output_shape".to_string()); let version = FunctionVersion::from_json(&result.to_string()).expect("future remote value"); - assert_eq!(version.runtime().kind(), "wasm"); - assert_eq!(version.runtime().python_version(), None); + assert_eq!(version.image().descriptor["future_interface"]["version"], 2); assert_eq!(version.signature().output.kind, "future_output_shape"); assert_eq!( serde_json::from_str::( &version.to_canonical_json().expect("canonical future value") ) - .expect("canonical JSON")["runtime"], - serde_json::json!({"kind": "wasm"}) + .expect("canonical JSON")["image"]["descriptor"]["future_interface"], + serde_json::json!({"version": 2}) ); } diff --git a/rust/lancedb/tests/first_class_function_slice2.rs b/rust/lancedb/tests/first_class_function_slice2.rs index 6d046d9d1..65924f394 100644 --- a/rust/lancedb/tests/first_class_function_slice2.rs +++ b/rust/lancedb/tests/first_class_function_slice2.rs @@ -42,11 +42,11 @@ async fn local_function_catalog_operations_return_stable_not_supported() { let create_error = connection.create_function_async(request).await.unwrap_err(); let lookup_error = connection - .get_function("normalize_score", "fv_exact") + .get_function("normalize_score", "1") .await .unwrap_err(); let drop_error = connection - .drop_function("normalize_score", "fv_exact") + .drop_function("normalize_score", "1") .await .unwrap_err(); for error in [create_error, lookup_error, drop_error] { diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json index fd3944412..1c2374fba 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_fixed_size_declaration_request.json @@ -9,7 +9,7 @@ "application": { "function": { "name": "embed", - "version": "fv_01K3EXACT" + "version": "1", "object_id": "fixture", "location": "memory:///fixture", "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6" }, "inputs": [ { diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.canonical.json index b91dd1061..8efa681aa 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.canonical.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.canonical.json @@ -1 +1 @@ -{"columns":{"normalized_text":"search_text","token_count":"search_token_count"},"function":{"name":"text_features","version":"fv_01K3TEXT"},"inputs":[{"kind":"column","parameter":"title","value":{"path":"title"}},{"kind":"column","parameter":"body","value":{"path":"body"}}],"output":{"fields":[{"arrow_type":"utf8","name":"normalized_text","nullable":false},{"arrow_type":"int64","name":"token_count","nullable":false}],"kind":"named_struct"}} +{"columns":{"normalized_text":"search_text","token_count":"search_token_count"},"function":{"location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","name":"text_features","object_id":"fixture","version":"1"},"inputs":[{"kind":"column","parameter":"title","value":{"path":"title"}},{"kind":"column","parameter":"body","value":{"path":"body"}}],"output":{"fields":[{"arrow_type":"utf8","name":"normalized_text","nullable":false},{"arrow_type":"int64","name":"token_count","nullable":false}],"kind":"named_struct"}} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.json index 177821aaa..d5b4fd5d8 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application.json @@ -1,5 +1,5 @@ { - "function": {"name": "text_features", "version": "fv_01K3TEXT"}, + "function": {"name": "text_features", "version": "1", "object_id": "fixture", "location": "memory:///fixture", "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs": [ {"parameter": "title", "kind": "column", "value": {"path": "title"}}, {"parameter": "body", "kind": "column", "value": {"path": "body"}} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application_float.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application_float.json index c23c8f3ca..98dafcfab 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application_float.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_application_float.json @@ -1,5 +1,5 @@ { - "function": {"name": "score", "version": "fv_01K3FLOAT"}, + "function": {"name": "score", "version": "1", "object_id": "fixture", "location": "memory:///fixture", "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs": [ {"parameter": "threshold", "kind": "literal", "value": 1e-7} ], diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.canonical.json index 143190f78..f1355364a 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.canonical.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.canonical.json @@ -1 +1 @@ -{"binding_id":"fb_01K3TEXT","function":{"name":"text_features","version":"fv_01K3TEXT"},"input_schema":{"fields":[{"name":"title","nullable":true,"type":{"type":"utf8"}},{"name":"body","nullable":true,"type":{"type":"utf8"}}]},"inputs":[{"arrow_type":"utf8","field_id":11,"field_path":"title","nullable":true,"parameter":"title"},{"arrow_type":"utf8","field_id":12,"field_path":"body","nullable":true,"parameter":"body"}],"output_schema":{"fields":[{"name":"search_text","nullable":true,"type":{"type":"utf8"}},{"name":"search_token_count","nullable":true,"type":{"type":"int64"}}]},"outputs":[{"arrow_type":"utf8","nullable":false,"output_field_id":21,"output_name":"search_text","output_ordinal":0,"result_field":"normalized_text"},{"arrow_type":"int64","nullable":false,"output_field_id":22,"output_name":"search_token_count","output_ordinal":1,"result_field":"token_count"}]} +{"binding_id":"fb_01K3TEXT","function":{"location":"memory:///fixture","manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","name":"text_features","object_id":"fixture","version":"1"},"input_schema":{"fields":[{"name":"title","nullable":true,"type":{"type":"utf8"}},{"name":"body","nullable":true,"type":{"type":"utf8"}}]},"inputs":[{"arrow_type":"utf8","field_id":11,"field_path":"title","nullable":true,"parameter":"title"},{"arrow_type":"utf8","field_id":12,"field_path":"body","nullable":true,"parameter":"body"}],"output_schema":{"fields":[{"name":"search_text","nullable":true,"type":{"type":"utf8"}},{"name":"search_token_count","nullable":true,"type":{"type":"int64"}}]},"outputs":[{"arrow_type":"utf8","nullable":false,"output_field_id":21,"output_name":"search_text","output_ordinal":0,"result_field":"normalized_text"},{"arrow_type":"int64","nullable":false,"output_field_id":22,"output_name":"search_token_count","output_ordinal":1,"result_field":"token_count"}]} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.json index dec3fa3f8..ae1666544 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_binding.json @@ -1,6 +1,6 @@ { "binding_id": "fb_01K3TEXT", - "function": {"name": "text_features", "version": "fv_01K3TEXT"}, + "function": {"name": "text_features", "version": "1", "object_id": "fixture", "location": "memory:///fixture", "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs": [ {"parameter": "title", "field_id": 11, "field_path": "title", "arrow_type": "utf8", "nullable": true}, {"parameter": "body", "field_id": 12, "field_path": "body", "arrow_type": "utf8", "nullable": true} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json index 39a279692..3381fd266 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json @@ -3,28 +3,64 @@ "job_type": "create_function", "job_state": "DONE", "creation_ms": 1787270400000, - "spec": {"name": "embed"}, + "spec": { + "name": "embed" + }, "result": { "name": "embed", - "version": "fv_01K3EXACT", - "artifact": { - "kind": "python_callable", - "digest": "sha256:code", - "entrypoint": "embed" - }, + "version": "1", "object_id": "fixture", "location": "memory:///fixture", "metadata": {}, "disabled": false, "signature": { - "inputs": [{"name": "text", "arrow_type": "utf8", "nullable": true}], - "output": {"kind": "scalar", "arrow_type": "list", "nullable": false} + "inputs": [ + { + "name": "text", + "arrow_type": "utf8", + "nullable": true + } + ], + "output": { + "kind": "scalar", + "arrow_type": "list", + "nullable": false + } }, - "runtime": { - "kind": "python", - "python_version": "3.12", - "environment": {"kind": "pip", "packages": ["sentence-transformers>=3"]}, - "env": {"TOKENIZERS_PARALLELISM": "false"} - }, - "runtime_digest": "sha256:runtime", - "environment_digest": "sha256:environment", - "created_at": "2026-08-21T00:00:00Z" + "created_at": "2026-08-21T00:00:00Z", + "image": { + "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6", + "descriptor": { + "format_version": 1, + "python": { + "implementation": "cpython", + "version": "3.12.14", + "abi_tag": "cp312", + "executable": "/usr/local/bin/python3", + "import_paths": [ + "/opt/function/code" + ] + }, + "python_api": 1, + "entrypoint": "app.function:create", + "interface": { + "type": "lance.scalar", + "version": 1 + }, + "schemas": { + "input": "/opt/function/schemas/input.arrow", + "output": "/opt/function/schemas/output.arrow", + "initialization": "/opt/function/schemas/initialization.arrow" + }, + "requires": { + "kernel_min": "4.18.0", + "capabilities": [] + }, + "behavior": { + "result_stability": "input_and_context", + "side_effects": "non_idempotent" + } + }, + "source": false + } }, - "future_job": {"trace_id": "trace-1"} + "future_job": { + "trace_id": "trace-1" + } } diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json index 2670ad0b2..3f0c11973 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json @@ -1 +1 @@ -{"artifact":{"digest":"sha256:code","entrypoint":"embed","kind":"python_callable"},"created_at":"2026-08-21T00:00:00Z","environment_digest":"sha256:environment","name":"embed","runtime":{"env":{"TOKENIZERS_PARALLELISM":"false"},"environment":{"kind":"pip","packages":["sentence-transformers>=3"]},"kind":"python","python_version":"3.12"},"runtime_digest":"sha256:runtime","signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list","kind":"scalar","nullable":false}},"version":"fv_01K3EXACT"} +{"created_at":"2026-08-21T00:00:00Z","disabled":false,"image":{"descriptor":{"behavior":{"result_stability":"input_and_context","side_effects":"non_idempotent"},"entrypoint":"app.function:create","format_version":1,"interface":{"type":"lance.scalar","version":1},"python":{"abi_tag":"cp312","executable":"/usr/local/bin/python3","implementation":"cpython","import_paths":["/opt/function/code"],"version":"3.12.14"},"python_api":1,"requires":{"capabilities":[],"kernel_min":"4.18.0"},"schemas":{"initialization":"/opt/function/schemas/initialization.arrow","input":"/opt/function/schemas/input.arrow","output":"/opt/function/schemas/output.arrow"}},"manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","source":false},"location":"memory:///fixture","metadata":{},"name":"embed","object_id":"fixture","signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list","kind":"scalar","nullable":false}},"version":"1"} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_multi_output_declaration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_multi_output_declaration_request.json index c4ef5009f..02b2d5880 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_multi_output_declaration_request.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_multi_output_declaration_request.json @@ -5,7 +5,7 @@ ], "function": { "application": { - "function": {"name": "text_features", "version": "fv_01K3TEXT"}, + "function": {"name": "text_features", "version": "1", "object_id": "fixture", "location": "memory:///fixture", "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs": [ {"parameter": "title", "kind": "column", "value": {"path": "title"}}, {"parameter": "body", "kind": "column", "value": {"path": "body"}} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_scalar_declaration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_scalar_declaration_request.json index 357834de0..30fe06b91 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_scalar_declaration_request.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_scalar_declaration_request.json @@ -4,7 +4,7 @@ ], "function": { "application": { - "function": {"name": "embed", "version": "fv_01K3EXACT"}, + "function": {"name": "embed", "version": "1", "object_id": "fixture", "location": "memory:///fixture", "manifest_digest": "sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6"}, "inputs": [ {"parameter": "text", "kind": "column", "value": {"path": "description"}} ], From 34ab278a5b3b5d408bb7b6fdae2b12b4bfc780c0 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 16 Sep 2026 15:08:56 +0800 Subject: [PATCH 23/43] fix: expose public FTS paths in remote index listings (#4194) Remote `list_indices()` exposes physical FTS paths such as `docs.item.content`, while index creation, queries, and native table listings use `docs.content`. Normalize FTS columns to the public path after parsing the server response, using the same field-ID-based conversion as native tables. Keep the physical paths on the wire: existing clients need them to resolve the Arrow schema. Cover both legacy responses that fetch index statistics and enriched responses that already include the index type. --- rust/lancedb/src/remote/table.rs | 55 ++++++++++++++++++++++++++++++-- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index dbabf9845..89061c393 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -43,8 +43,8 @@ use crate::table::{ use crate::table::{AnyQuery, Filter, Predicate, PreprocessingOutput, TableStatistics}; use crate::utils::background_cache::BackgroundCache; use crate::utils::{ - MaxBatchLengthStream, TimeoutStream, resolve_arrow_field_path, resolve_arrow_fts_field_path, - supported_btree_data_type, supported_vector_data_type, + MaxBatchLengthStream, TimeoutStream, public_fts_field_path_by_id, resolve_arrow_field_path, + resolve_arrow_fts_field_path, supported_btree_data_type, supported_vector_data_type, }; use crate::{DistanceType, Error}; use crate::{ @@ -2042,7 +2042,19 @@ impl RemoteTable { } let results = futures::future::try_join_all(futures).await?; - Ok(results.into_iter().flatten().collect()) + let mut indices: Vec = results.into_iter().flatten().collect(); + let lance_schema = lance_core::datatypes::Schema::try_from(schema.as_ref())?; + for index in &mut indices { + if index.index_type == IndexType::FTS { + // The wire format uses physical paths for schema resolution. Match + // native tables by exposing list-transparent paths to callers. + for column in &mut index.columns { + let field_id = lance_schema.field_id(column)?; + *column = public_fts_field_path_by_id(&lance_schema, field_id)?; + } + } + } + Ok(indices) } } @@ -6984,6 +6996,43 @@ mod tests { assert_eq!(indices, expected); } + #[rstest] + #[case::legacy(false)] + #[case::enriched(true)] + #[tokio::test] + async fn test_list_indices_fts_public_list_path(#[case] enriched: bool) { + let schema = nested_index_schema(); + let table = Table::new_with_handler("my_table", move |request| { + let body = match request.url().path() { + "/v1/table/my_table/describe/" => describe_response(&schema), + "/v1/table/my_table/index/list/" => serde_json::json!({ + "indexes": [{ + "index_name": "docs_idx", + "columns": ["docs.item.content"], + "index_type": enriched.then_some("FTS"), + }], + }) + .to_string(), + "/v1/table/my_table/index/docs_idx/stats/" => { + assert!(!enriched, "enriched responses must not fetch index stats"); + serde_json::json!({ + "num_indexed_rows": 1, + "num_unindexed_rows": 0, + "index_type": "FTS", + }) + .to_string() + } + path => panic!("Unexpected path: {path}"), + }; + http::Response::builder().status(200).body(body).unwrap() + }); + + let indices = table.list_indices().await.unwrap(); + assert_eq!(indices.len(), 1); + assert_eq!(indices[0].index_type, IndexType::FTS); + assert_eq!(indices[0].columns, vec!["docs.content"]); + } + #[tokio::test] async fn test_list_indices_nested_field_paths() { let schema = nested_index_schema(); From 7991e27f35a207b60a267fa4a12992b03d43924b Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 16 Sep 2026 16:32:34 +0800 Subject: [PATCH 24/43] fix(node): prevent OAuth tests from launching browsers (#4196) OAuth tests set `LANCEDB_OAUTH_BROWSER` inside Jest's sandbox, which does not update the process environment read by Rust. As a result, the native login flow launches a real browser; the [failing macOS main job](https://github.com/lancedb/lancedb/actions/runs/35067074667/job/104699667840) ends by terminating an orphaned Safari process. Set the no-op browser helper while loading Jest configuration, before creating sandboxes and workers, so both local and CI tests inherit it. Check the inherited process environment before OAuth login to catch regressions without opening a browser. Windows uses a no-op command fixture. Test deadlines and worker counts are unchanged. A negative control restoring the sandbox-only assignment fails in the new pre-login check. The full macOS test suite passes with the standard test launcher; the Windows helper has not been executed locally. The [first hosted macOS run](https://github.com/lancedb/lancedb/actions/runs/35071525843/job/104713928139) passes all 843 tests (5 skipped), with no Safari process in the job log. Further normal runs are needed to establish sustained stability. --- nodejs/__test__/fixtures/oauth_browser.cmd | 3 +++ nodejs/__test__/oauth.test.ts | 13 ++++++++++--- nodejs/jest.config.js | 9 +++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 nodejs/__test__/fixtures/oauth_browser.cmd diff --git a/nodejs/__test__/fixtures/oauth_browser.cmd b/nodejs/__test__/fixtures/oauth_browser.cmd new file mode 100644 index 000000000..8e97db307 --- /dev/null +++ b/nodejs/__test__/fixtures/oauth_browser.cmd @@ -0,0 +1,3 @@ +@rem SPDX-License-Identifier: Apache-2.0 +@rem SPDX-FileCopyrightText: Copyright The LanceDB Authors +@exit /b 0 diff --git a/nodejs/__test__/oauth.test.ts b/nodejs/__test__/oauth.test.ts index 0a04bb2aa..627da786f 100644 --- a/nodejs/__test__/oauth.test.ts +++ b/nodejs/__test__/oauth.test.ts @@ -3,6 +3,7 @@ import * as fs from "fs"; import * as http from "http"; +import { execFileSync } from "node:child_process"; import * as os from "os"; import * as path from "path"; import { OAuthConfig, OAuthFlowType, OAuthSession } from "../lancedb/oauth"; @@ -23,9 +24,15 @@ function deviceConfig(issuerUrl: string, cacheDir: string): OAuthConfig { describe("OAuthSession", () => { beforeAll(() => { - // Point the Rust browser helper at a no-op so device-flow logins never - // open a real browser window during tests. - process.env.LANCEDB_OAUTH_BROWSER = "/usr/bin/true"; + // Child processes inherit the real environment, just as Rust reads it. + // Fail before login if the browser override only exists in Jest's sandbox. + const browser = execFileSync( + process.execPath, + ["-p", "process.env.LANCEDB_OAUTH_BROWSER ?? ''"], + { encoding: "utf8" }, + ).trim(); + expect(browser).not.toBe(""); + expect(browser).toBe(process.env.LANCEDB_OAUTH_BROWSER); }); it("reports an absent session and logout is idempotent", async () => { diff --git a/nodejs/jest.config.js b/nodejs/jest.config.js index dc99083a5..c137a4b9e 100644 --- a/nodejs/jest.config.js +++ b/nodejs/jest.config.js @@ -1,3 +1,12 @@ +const path = require("node:path"); + +// Set the real process environment before Jest creates its sandbox and workers. +// Assigning process.env inside a test does not reach Rust's std::env. +process.env.LANCEDB_OAUTH_BROWSER = + process.platform === "win32" + ? path.join(__dirname, "__test__/fixtures/oauth_browser.cmd") + : "/usr/bin/true"; + /** @type {import('ts-jest').JestConfigWithTsJest} */ module.exports = { preset: "ts-jest", From 161f81276dabd722733ad6c059d1305f04a9dfac Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 16 Sep 2026 17:25:02 +0800 Subject: [PATCH 25/43] chore: update lance dependency to v13.0.0-beta.3 (#4197) Update the Rust workspace and Java Lance dependencies to [v13.0.0-beta.3](https://github.com/lance-format/lance/releases/tag/v13.0.0-beta.3), which includes the nullable-list encoding fix in lance-format/lance#9268. Also resolve existing strict Clippy diagnostics in crate-private OAuth modules and the Python token-cache conversion, without changing effective visibility. Validated the rebuilt Python SDK with nullable and nested lists: Match/Phrase searches preserve document coordinates before and after appending data and optimizing the table. --- Cargo.lock | 184 +++++++++++-------------- Cargo.toml | 28 ++-- java/pom.xml | 2 +- python/src/oauth.rs | 2 +- rust/lancedb/src/remote/client.rs | 2 +- rust/lancedb/src/remote/token_cache.rs | 4 +- 6 files changed, 101 insertions(+), 121 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 85c55b7fa..ad6b3d1bd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -583,13 +583,9 @@ dependencies = [ [[package]] name = "asyncband" -version = "0.6.7" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94a214ba60d6231afd0e805e3c27c45a1626d9debaa5a5061c45a1ea1b2f1ed0" -dependencies = [ - "hashbrown 0.17.1", - "slab", -] +checksum = "f2d85fd3d291fabcc40c7232c92c280ec1754fd7b5d7ea769f143222143e179a" [[package]] name = "atoi" @@ -814,7 +810,7 @@ dependencies = [ "http 0.2.12", "http 1.5.0", "http-body 1.1.0", - "lru 0.16.4", + "lru", "percent-encoding", "regex-lite", "sha2 0.11.0", @@ -3511,8 +3507,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow-array", "rand 0.9.5", @@ -3869,9 +3865,9 @@ dependencies = [ [[package]] name = "goosefs-sdk" -version = "0.1.9" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1ea4eee6dcbc31b25ab4fd577adc55b677d2bed3aa3016c44c58fbe1b2298a5" +checksum = "6f521e3beacc809ced36a8aff4a468b1bf61302c8f4dca41c3f21ac0557c22ed" dependencies = [ "arc-swap", "async-trait", @@ -3880,16 +3876,9 @@ dependencies = [ "fastrand", "futures", "hostname", - "io-uring", "itoa", - "libc", - "lru 0.18.2", - "memmap2 0.9.10", - "moka", "prost", - "prost-types", "rand 0.9.5", - "reqwest 0.12.28", "serde", "thiserror 2.0.18", "tokio", @@ -4883,8 +4872,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arc-swap", "arrow", @@ -4956,8 +4945,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow-array", "arrow-buffer", @@ -4979,7 +4968,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow-array", "arrow-buffer", @@ -4993,7 +4982,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow-array", "arrow-schema", @@ -5002,8 +4991,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrayref", "crunchy", @@ -5013,8 +5002,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow-array", "arrow-buffer", @@ -5051,8 +5040,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow", "arrow-array", @@ -5082,8 +5071,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow", "arrow-array", @@ -5100,8 +5089,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "proc-macro2", "quote", @@ -5110,8 +5099,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow-arith", "arrow-array", @@ -5144,8 +5133,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow-arith", "arrow-array", @@ -5177,8 +5166,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arc-swap", "arrow", @@ -5242,8 +5231,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow-array", "arrow-schema", @@ -5265,8 +5254,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow", "arrow-array", @@ -5306,8 +5295,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow-array", "arrow-schema", @@ -5321,8 +5310,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow", "async-trait", @@ -5336,8 +5325,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow", "arrow-ipc", @@ -5390,8 +5379,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow-array", "arrow-buffer", @@ -5405,8 +5394,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow", "arrow-array", @@ -5446,8 +5435,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "arrow-array", "arrow-schema", @@ -5460,8 +5449,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "13.0.0-beta.1" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.1#cfbd1b8fd72ac3028de6991b0c57a5af03e7ece3" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" dependencies = [ "frostem", "icu_segmenter", @@ -5856,15 +5845,6 @@ dependencies = [ "hashbrown 0.16.1", ] -[[package]] -name = "lru" -version = "0.18.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d2f2f9b4ba7e6b24d95e7e899329d35be83bcded72c8540cdd5368932d1d90a" -dependencies = [ - "hashbrown 0.17.1", -] - [[package]] name = "lru-slab" version = "0.1.2" @@ -6586,9 +6566,9 @@ dependencies = [ [[package]] name = "object_store_opendal" -version = "0.60.1" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0206382328a82a28b549e5b2d18b6b9384ac1d82fa1f3c63da06f5ee6f7f054" +checksum = "c479bcc317f0ed98972b7184e0f9cc1bea83a07f60cdade96d7207a48ebfb779" dependencies = [ "async-trait", "asyncband", @@ -6649,9 +6629,9 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "opendal" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950151f9587a51a7bed70a15fa0cff464eae96e41ae7499f97067bdafdf43eb" +checksum = "9fe43e16d96bed57937eb7c4a28559bf4c5fc4a3951656505d1fdd4f856349b4" dependencies = [ "ctor", "opendal-core", @@ -6672,9 +6652,9 @@ dependencies = [ [[package]] name = "opendal-core" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a43405d217dfdfb543f58847336d3af672897dd1939bb7dcf314b63cf364f1c9" +checksum = "de4566a412776e3f65d53dd9fceced5b432a0a9c9a1e8d50ccf58823795e5c5b" dependencies = [ "anyhow", "asyncband", @@ -6698,9 +6678,9 @@ dependencies = [ [[package]] name = "opendal-http-transport-reqwest" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "401999057db611e592f883fcf2cbd6754ff37af587deaadd07b8c1398b2b6b06" +checksum = "f772ce6c137ab43647726116d505a649a88d41928fcd813438eb30b02312bcbb" dependencies = [ "bytes", "futures", @@ -6712,9 +6692,9 @@ dependencies = [ [[package]] name = "opendal-layer-concurrent-limit" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fba1dd0742261925fc0eb910773ec39cbc1336c55d41e13b19f3af970ad5a126" +checksum = "3734c648617e5ad724d49175dde8323550624f9614f50e9e81b7aafae30b1734" dependencies = [ "asyncband", "futures", @@ -6724,9 +6704,9 @@ dependencies = [ [[package]] name = "opendal-layer-logging" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0fd963f9d32dd276521479d7f1f3a265d669b03a75f2062a4570a26b1b17421" +checksum = "0a8daab5a91eac76f94516e7c71e677195c9221840ce8f5e71815e3fc8207742" dependencies = [ "log", "opendal-core", @@ -6734,9 +6714,9 @@ dependencies = [ [[package]] name = "opendal-layer-retry" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06306202c97c54fb41bdbbdcb854c8798823f0022ae7aac7a40c8a1023aa83a6" +checksum = "eb8e22cb1f86ef9af771a8947d3ccf380c8f5b76236f628314dd41c9ba73331a" dependencies = [ "backon", "log", @@ -6745,9 +6725,9 @@ dependencies = [ [[package]] name = "opendal-layer-timeout" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd334cbd0a0bc934146733e74a80a5faf8db8e014781a25f6d9d80d7b87c981" +checksum = "08ac6c77fe0d1b18e0064a02c19b98dbebcab78bf5e7753ad1f3abbc18850ff4" dependencies = [ "opendal-core", "tokio", @@ -6755,9 +6735,9 @@ dependencies = [ [[package]] name = "opendal-service-azblob" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba0d2662ddf0de1f838db5dc48fafb8212ed1a5c7980bc164aa67809696c309c" +checksum = "c6cf31b02e7b44191d97c07254eea109b5da9c17210a29e478c8d0dc26dffb1f" dependencies = [ "base64 0.23.1", "bytes", @@ -6776,9 +6756,9 @@ dependencies = [ [[package]] name = "opendal-service-azdls" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9064ed464286bffbea5955d470082a1e72b9b89f1e8f7a61b9e303c55ec08e4f" +checksum = "354f4a8d26f0fb7639f5c4ad3a924d7c75625830f69550b8c807c080ea0df8bc" dependencies = [ "asyncband", "base64 0.23.1", @@ -6797,9 +6777,9 @@ dependencies = [ [[package]] name = "opendal-service-azure-common" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6348e3c0d7ff77a9b05c5b7f3d744ed395c2b2511812d3b37a934218002248f8" +checksum = "8107324c3f3cb9970fe20526c2c341b7c5769acbd4699bab73bd525f5fd63d62" dependencies = [ "http 1.5.0", "opendal-core", @@ -6807,9 +6787,9 @@ dependencies = [ [[package]] name = "opendal-service-cos" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49a652eadc76b94f9cffa497b3de5250dff53cce394d2974c00dde62c4e4cd81" +checksum = "feb737e6609b4fd42e4773b7b2d8d05031b63cd0df46c7ba68ddd64bae1dd3f3" dependencies = [ "bytes", "http 1.5.0", @@ -6824,9 +6804,9 @@ dependencies = [ [[package]] name = "opendal-service-gcs" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ccbf8450652bfe7b3ae69b7decce090c95c120ad565048f9a5a77dac2917a19" +checksum = "3d80eca6f9227bdf65d43e5b8d5fe75dca8cfab1da07571442a0420400847bea" dependencies = [ "async-trait", "bytes", @@ -6846,23 +6826,23 @@ dependencies = [ [[package]] name = "opendal-service-goosefs" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b53d8c3e1db3176add7aff9ad2637c907409bac19e025ee8e9c072c0061d9c5" +checksum = "2134aca0ef64dbff844cb1e608d28833f2a1b34b884e7d6b307416e966694e9f" dependencies = [ + "asyncband", "bytes", "goosefs-sdk", "log", "opendal-core", "serde", - "tokio", ] [[package]] name = "opendal-service-hf" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c17b59cf22bd2da9f751b8e66db595d5fe546b4c6b6ff0fb84c7bfd27455d7a" +checksum = "0a0cba627efaee637746804b74bc2c2b2b207fe214307e1e2ba31b49aad708a3" dependencies = [ "asyncband", "bytes", @@ -6877,9 +6857,9 @@ dependencies = [ [[package]] name = "opendal-service-oss" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "284373c4a1143d8efaa7d010c1db05856d33a7cad475aaee8405dbcb0660cd96" +checksum = "a0ebfa355b1d4edb0e5b2a5b86d8de9f811fae1d7baca6d65246b230b2afc0bd" dependencies = [ "bytes", "http 1.5.0", @@ -6894,9 +6874,9 @@ dependencies = [ [[package]] name = "opendal-service-s3" -version = "0.59.1" +version = "0.59.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388b1d39b62535c62803754ebef89808859558697366dbedd0299345887ba461" +checksum = "2942f2d8d3d4953c0e8d07cd1244879628d084948e48ce6b6d808c5938f7858d" dependencies = [ "base64 0.23.1", "bytes", diff --git a/Cargo.toml b/Cargo.toml index 092180596..5509e9bb1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=13.0.0-beta.1", default-features = false, "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=13.0.0-beta.1", default-features = false, "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=13.0.0-beta.1", default-features = false, "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=13.0.0-beta.1", "tag" = "v13.0.0-beta.1", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=13.0.0-beta.3", default-features = false, "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=13.0.0-beta.3", default-features = false, "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=13.0.0-beta.3", default-features = false, "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index 16e39b909..a286add75 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 13.0.0-beta.1 + 13.0.0-beta.3 false 2.30.0 1.7 diff --git a/python/src/oauth.rs b/python/src/oauth.rs index 080ea4d95..da5b6e3a3 100644 --- a/python/src/oauth.rs +++ b/python/src/oauth.rs @@ -22,7 +22,7 @@ pub struct PyTokenCacheOptions { impl From for TokenCacheOptions { fn from(py: PyTokenCacheOptions) -> Self { - TokenCacheOptions { + Self { cache_dir: py.cache_dir.map(PathBuf::from), lock_timeout_secs: py.lock_timeout_secs, } diff --git a/rust/lancedb/src/remote/client.rs b/rust/lancedb/src/remote/client.rs index 704afe948..6e4764ad6 100644 --- a/rust/lancedb/src/remote/client.rs +++ b/rust/lancedb/src/remote/client.rs @@ -15,7 +15,7 @@ use crate::remote::retry::{ResolvedRetryConfig, RetryCounter}; const REQUEST_ID_HEADER: HeaderName = HeaderName::from_static("x-request-id"); -pub(crate) fn redact_sensitive_headers(headers: &mut HeaderMap) { +pub fn redact_sensitive_headers(headers: &mut HeaderMap) { const SENSITIVE_HEADERS: [&str; 5] = [ "authorization", "proxy-authorization", diff --git a/rust/lancedb/src/remote/token_cache.rs b/rust/lancedb/src/remote/token_cache.rs index 7d305b3cb..edda93bcf 100644 --- a/rust/lancedb/src/remote/token_cache.rs +++ b/rust/lancedb/src/remote/token_cache.rs @@ -337,7 +337,7 @@ struct LockGuard { } /// The persistent token cache engine for one [`OAuthConfig`]. -pub(crate) struct TokenCache { +pub struct TokenCache { dir: PathBuf, key: CacheKey, lock_timeout: Duration, @@ -951,7 +951,7 @@ impl OAuthSession { /// client-credentials flow, which has no refresh token to persist (a debug /// note is logged). The Azure managed-identity flow is rejected because /// machine identity must not enter a user token cache. -pub(crate) fn token_cache_for_config(config: &OAuthConfig) -> Result>> { +pub fn token_cache_for_config(config: &OAuthConfig) -> Result>> { let Some(options) = &config.token_cache else { return Ok(None); }; From ed100ccc31cfe6834566a06826ba7a9cb61fb81a Mon Sep 17 00:00:00 2001 From: Lance Release Date: Wed, 16 Sep 2026 09:25:49 +0000 Subject: [PATCH 26/43] =?UTF-8?q?Bump=20version:=200.39.0-beta.9=20?= =?UTF-8?q?=E2=86=92=200.39.0-beta.10?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index aae26f441..87b0f1f7c 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.39.0-beta.9" +current_version = "0.39.0-beta.10" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index ad6b3d1bd..20479e8af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5463,7 +5463,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.39.0-beta.9" +version = "0.39.0-beta.10" dependencies = [ "ahash", "anyhow", @@ -5559,7 +5559,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.39.0-beta.9" +version = "0.39.0-beta.10" dependencies = [ "arrow-array", "arrow-buffer", @@ -5584,7 +5584,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.39.0-beta.9" +version = "0.39.0-beta.10" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 1272fd71a..f08277fff 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.39.0-beta.9 + 0.39.0-beta.10 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index d85915b67..def010a76 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.39.0-beta.9 + 0.39.0-beta.10 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index a286add75..ccdbf285b 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.39.0-beta.9 + 0.39.0-beta.10 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 51243382f..3a37a9f5d 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.39.0-beta.9" +version = "0.39.0-beta.10" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index e7ada7229..40562d1b1 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.39.0-beta.9", + "version": "0.39.0-beta.10", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 4333b10c7..aec5307b0 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.39.0-beta.9", + "version": "0.39.0-beta.10", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index d0b0f18fa..5fade6079 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.39.0-beta.9", + "version": "0.39.0-beta.10", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index e7ac1eb12..ee3ff460a 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.39.0-beta.9", + "version": "0.39.0-beta.10", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 5eadc836f..64e9471e9 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.39.0-beta.9", + "version": "0.39.0-beta.10", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 880637a0f..3f0123a45 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.39.0-beta.9", + "version": "0.39.0-beta.10", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index de9b54f07..e3d4ab7eb 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.39.0-beta.9", + "version": "0.39.0-beta.10", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index b23100926..0de36ad95 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.39.0-beta.9", + "version": "0.39.0-beta.10", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 4cf914f6a..d9dc7f1f8 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.39.0-beta.9" +version = "0.39.0-beta.10" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index d798d8006..039103c5e 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.39.0-beta.9" +version = "0.39.0-beta.10" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From f3ef21b8ca50bfdf3d9dd5341b61d20d62e872b3 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 16 Sep 2026 05:11:08 -0700 Subject: [PATCH 27/43] feat: use oauth2 crate for OAuth with configurable client auth (#4181) Stacked on #4173 (`jack/restore-oidc-flows`, base branch mirrored to this repo so the diff shows only this change); context from review: https://github.com/lancedb/lancedb/pull/4173#issuecomment-5674048100. Rebase to `main` once #4173 and #4179 merge. ## What moved to the `oauth2` crate (5.0, no default features) - Authorization URL generation and CSRF state (`authorize_url`, `CsrfToken`) - PKCE S256 challenge/verifier generation and code exchange - Client-credentials, authorization-code, refresh-token, and device-code grant request construction - Device authorization request and the device token polling loop (`authorization_pending`, `slow_down` +5s, expiry deadline, denial, network backoff capped at 10s) - Standard success/error response parsing (`RequestTokenError`) - Token-endpoint client authentication and standards-compliant parameter encoding (RFC 6749 2.3.1 Basic encoding) LanceDB keeps ownership of OIDC discovery (compared `openidconnect`: no measurable win for our 3-field metadata + strict validation, at real dependency cost), HTTPS-or-loopback endpoint enforcement, the loopback callback server, browser/stderr prompts, token caching and refresh orchestration, and the dedicated hardened Azure IMDS source, which is unchanged. ## Client authentication methods New `ClientAuthMethod` enum (`none` | `client_secret_basic` | `client_secret_post`), exposed in Rust, Python, and Node. Unset resolves to `client_secret_basic` when a secret is present (RFC 6749 2.3.1 recommendation and the normal Okta confidential-app default, so a default Okta app works without weakening its configuration) and to `none` for public clients (PKCE/device). Explicit `none` with a secret, or basic/post without one, is rejected. The method applies to client credentials, code exchange, refresh, and device requests. Deliberate behavior change: confidential clients previously always sent the secret in the POST body; they now default to Basic (Keycloak accepts both). No `audience`/`resource` parameters were added: the supported target is an Okta custom authorization server with the API audience configured server-side, so client-provided audience parameters are unnecessary; `add_extra_param` support exists if a concrete provider contract ever needs them. ## Device polling behavior changes (deliberate, tested) - The first token poll now happens immediately rather than after one interval (RFC 8628 allows both). - Transient failures (HTTP 429, 5xx, `temporarily_unavailable`, network errors) now retry with exponential backoff capped at 10s instead of retrying at the fixed interval; polling never spins faster than once per second even if a server reports a zero interval. ## Security and compatibility - Issuer and discovered endpoints (and device verification URIs) still require HTTPS except explicit loopback HTTP, enforced before any crate URL type is built - Token HTTP client keeps the hardened redirect policy that refuses insecure redirect targets; regression test added - Errors never embed raw response bodies (avoids leaking tokens through parse failures); all credential types stay redacted in Debug - Transient conditions (429/5xx/`temporarily_unavailable`) remain retryable in device polling and hard errors elsewhere; refresh keeps rotation and reauthentication semantics - Existing public APIs stay source-compatible except the added `OAuthConfig.client_auth_method` field ## Tests Rust: client-auth methods across code exchange/refresh/client-credentials/device (none/basic/post), auth-method resolution and validation, transient device retries, denial/expiry, redirect rejection, malformed-response leak check, PKCE URL assertions, redaction. Python and Node: enum values, conversion, unknown-method errors, config defaults. Manual Okta validation recipe (no automated Okta credentials): create a custom authorization server with an API audience, one confidential web app (Basic) for authorization-code, one native app (PKCE, no secret), one native device app; point `issuer_url` at the custom server, set `client_auth_method` only for the POST-required case; verify token acquisition, refresh after expiry, and `x-lancedb-credential-type: oidc` against a LanceDB deployment. Never commit tenant URLs or secrets. Co-authored-by: Xuanwo --- Cargo.lock | 21 + Cargo.toml | 1 + docs/src/js/enumerations/ClientAuthMethod.md | 48 + docs/src/js/globals.md | 1 + docs/src/js/interfaces/NativeOAuthConfig.md | 13 + docs/src/js/interfaces/OAuthConfig.md | 14 + nodejs/__test__/remote.test.ts | 37 + nodejs/lancedb/index.ts | 1 + nodejs/lancedb/oauth.ts | 36 + nodejs/src/remote.rs | 85 + python/python/lancedb/remote/__init__.py | 9 +- python/python/lancedb/remote/oauth.py | 36 +- python/src/oauth.rs | 49 +- python/tests/test_oauth.py | 28 + rust/lancedb/Cargo.toml | 2 + rust/lancedb/src/connection.rs | 2 + rust/lancedb/src/remote.rs | 4 +- rust/lancedb/src/remote/oauth.rs | 1578 ++++++++++++------ rust/lancedb/src/remote/token_cache.rs | 30 +- 19 files changed, 1485 insertions(+), 510 deletions(-) create mode 100644 docs/src/js/enumerations/ClientAuthMethod.md diff --git a/Cargo.lock b/Cargo.lock index 20479e8af..4142b4cf1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5526,6 +5526,7 @@ dependencies = [ "metrics-util", "moka", "num-traits", + "oauth2", "object_store 0.14.1", "pin-project", "polars", @@ -6419,6 +6420,25 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" +[[package]] +name = "oauth2" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" +dependencies = [ + "base64 0.22.1", + "chrono", + "getrandom 0.2.17", + "http 1.5.0", + "rand 0.8.6", + "serde", + "serde_json", + "serde_path_to_error", + "sha2 0.10.9", + "thiserror 1.0.69", + "url", +] + [[package]] name = "objc2" version = "0.6.4" @@ -10584,6 +10604,7 @@ dependencies = [ "idna", "percent-encoding", "serde", + "serde_derive", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 5509e9bb1..897bdbe40 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,6 +68,7 @@ rand = "0.9" snafu = "0.8" url = "2" num-traits = "0.2" +oauth2 = { version = "5.0", default-features = false } regex = "1.10" semver = "1.0.25" serde = "1" diff --git a/docs/src/js/enumerations/ClientAuthMethod.md b/docs/src/js/enumerations/ClientAuthMethod.md new file mode 100644 index 000000000..47fbe3164 --- /dev/null +++ b/docs/src/js/enumerations/ClientAuthMethod.md @@ -0,0 +1,48 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / ClientAuthMethod + +# Enumeration: ClientAuthMethod + +How the client authenticates to the OAuth token endpoint. + +The method applies to every OAuth request that carries client +authentication: client-credentials, authorization-code exchange, +refresh-token, and device-authorization requests. The Azure managed +identity flow ignores this option. + +## Enumeration Members + +### ClientSecretBasic + +```ts +ClientSecretBasic: "client_secret_basic"; +``` + +HTTP Basic authentication. This is the RFC 6749 recommended method and +the normal default for confidential clients, including default Okta +applications. Requires `clientSecret`. + +*** + +### ClientSecretPost + +```ts +ClientSecretPost: "client_secret_post"; +``` + +Credentials in the request body, for providers configured to require it. +Requires `clientSecret`. + +*** + +### None + +```ts +None: "none"; +``` + +No client authentication, for public clients using PKCE or the device +flow. Cannot be combined with `clientSecret`. diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 7e8344054..4e09853f6 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -11,6 +11,7 @@ ## Enumerations +- [ClientAuthMethod](enumerations/ClientAuthMethod.md) - [FullTextQueryType](enumerations/FullTextQueryType.md) - [OAuthFlowType](enumerations/OAuthFlowType.md) - [Occur](enumerations/Occur.md) diff --git a/docs/src/js/interfaces/NativeOAuthConfig.md b/docs/src/js/interfaces/NativeOAuthConfig.md index 074131808..8d3f1438d 100644 --- a/docs/src/js/interfaces/NativeOAuthConfig.md +++ b/docs/src/js/interfaces/NativeOAuthConfig.md @@ -35,6 +35,19 @@ Port for the authorization_code loopback callback server. *** +### clientAuthMethod? + +```ts +optional clientAuthMethod: string; +``` + +How the client authenticates to the token endpoint: "none", +"client_secret_basic", or "client_secret_post". Defaults to +"client_secret_basic" when a client secret is set, and "none" for +public clients. + +*** + ### clientId ```ts diff --git a/docs/src/js/interfaces/OAuthConfig.md b/docs/src/js/interfaces/OAuthConfig.md index 0615fb8c5..0342d62ea 100644 --- a/docs/src/js/interfaces/OAuthConfig.md +++ b/docs/src/js/interfaces/OAuthConfig.md @@ -84,6 +84,20 @@ Port for the AuthorizationCode loopback callback server (default: 8400). *** +### clientAuthMethod? + +```ts +optional clientAuthMethod: ClientAuthMethod; +``` + +How the client authenticates to the token endpoint (default: auto). +With a `clientSecret` the default is `ClientAuthMethod.ClientSecretBasic`, +which matches the RFC 6749 recommendation and the default configuration +of Okta confidential applications; without a secret the client is public +and no client authentication is sent. + +*** + ### clientId ```ts diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index 5da0bf724..85d725825 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -5,9 +5,12 @@ import * as http from "http"; import { RequestListener } from "http"; import packageJson = require("../package.json"); import { + ClientAuthMethod, ClientConfig, Connection, ConnectionOptions, + OAuthConfig, + OAuthFlowType, TlsConfig, connect, } from "../lancedb"; @@ -438,6 +441,40 @@ describe("remote connection", () => { ]); }); + describe("OAuthConfig", () => { + it("should expose client auth method values", () => { + expect(ClientAuthMethod.None).toBe("none"); + expect(ClientAuthMethod.ClientSecretBasic).toBe("client_secret_basic"); + expect(ClientAuthMethod.ClientSecretPost).toBe("client_secret_post"); + }); + + it("should accept a confidential client with basic auth", () => { + const config: OAuthConfig = { + issuerUrl: "https://issuer.example.com", + clientId: "client-id", + clientSecret: "secret", + scopes: ["openid"], + flow: OAuthFlowType.AuthorizationCode, + clientAuthMethod: ClientAuthMethod.ClientSecretBasic, + }; + + expect(config.clientAuthMethod).toBe(ClientAuthMethod.ClientSecretBasic); + }); + + it("should accept a public PKCE client without auth method or secret", () => { + const config: OAuthConfig = { + issuerUrl: "https://issuer.example.com", + clientId: "client-id", + scopes: ["openid"], + flow: OAuthFlowType.AuthorizationCode, + usePkce: true, + }; + + expect(config.clientSecret).toBeUndefined(); + expect(config.clientAuthMethod).toBeUndefined(); + }); + }); + describe("TlsConfig", () => { it("should create TlsConfig with all fields", () => { const tlsConfig: TlsConfig = { diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index c30c5c5ba..55078bba3 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -172,6 +172,7 @@ export { } from "./header"; export { + ClientAuthMethod, OAuthConfig, OAuthFlowType, OAuthSession, diff --git a/nodejs/lancedb/oauth.ts b/nodejs/lancedb/oauth.ts index 162f38997..1a9cfe351 100644 --- a/nodejs/lancedb/oauth.ts +++ b/nodejs/lancedb/oauth.ts @@ -47,6 +47,33 @@ export interface TokenCacheOptions { lockTimeoutSecs?: number; } +/** + * How the client authenticates to the OAuth token endpoint. + * + * The method applies to every OAuth request that carries client + * authentication: client-credentials, authorization-code exchange, + * refresh-token, and device-authorization requests. The Azure managed + * identity flow ignores this option. + */ +export enum ClientAuthMethod { + /** + * No client authentication, for public clients using PKCE or the device + * flow. Cannot be combined with `clientSecret`. + */ + None = "none", + /** + * HTTP Basic authentication. This is the RFC 6749 recommended method and + * the normal default for confidential clients, including default Okta + * applications. Requires `clientSecret`. + */ + ClientSecretBasic = "client_secret_basic", + /** + * Credentials in the request body, for providers configured to require it. + * Requires `clientSecret`. + */ + ClientSecretPost = "client_secret_post", +} + /** * OAuth configuration for LanceDB authentication. * @@ -140,6 +167,15 @@ export interface OAuthConfig { /** Client secret (required for ClientCredentials). */ clientSecret?: string; + /** + * How the client authenticates to the token endpoint (default: auto). + * With a `clientSecret` the default is `ClientAuthMethod.ClientSecretBasic`, + * which matches the RFC 6749 recommendation and the default configuration + * of Okta confidential applications; without a secret the client is public + * and no client authentication is sent. + */ + clientAuthMethod?: ClientAuthMethod; + /** Loopback redirect URI for AuthorizationCode. */ redirectUri?: string; diff --git a/nodejs/src/remote.rs b/nodejs/src/remote.rs index 1db00e832..784631267 100644 --- a/nodejs/src/remote.rs +++ b/nodejs/src/remote.rs @@ -197,6 +197,11 @@ pub struct OAuthConfig { pub flow: Option, /// Client secret (required for client_credentials). pub client_secret: Option, + /// How the client authenticates to the token endpoint: "none", + /// "client_secret_basic", or "client_secret_post". Defaults to + /// "client_secret_basic" when a client secret is set, and "none" for + /// public clients. + pub client_auth_method: Option, /// Loopback redirect URI for authorization_code. pub redirect_uri: Option, /// Port for the authorization_code loopback callback server. @@ -227,6 +232,7 @@ impl std::fmt::Debug for OAuthConfig { "client_secret", &self.client_secret.as_deref().map(|_| ""), ) + .field("client_auth_method", &self.client_auth_method) .field("redirect_uri", &self.redirect_uri) .field("callback_port", &self.callback_port) .field("use_pkce", &self.use_pkce) @@ -270,10 +276,27 @@ impl TryFrom for lancedb::remote::oauth::OAuthConfig { } }; + let client_auth_method = match config.client_auth_method.as_deref() { + Some("none") => Some(lancedb::remote::oauth::ClientAuthMethod::None), + Some("client_secret_basic") => { + Some(lancedb::remote::oauth::ClientAuthMethod::ClientSecretBasic) + } + Some("client_secret_post") => { + Some(lancedb::remote::oauth::ClientAuthMethod::ClientSecretPost) + } + None => None, + Some(other) => { + return Err(Error::InvalidInput { + message: format!("Unknown OAuth client auth method: {other}"), + }); + } + }; + Ok(Self { issuer_url: config.issuer_url, client_id: config.client_id, client_secret: config.client_secret, + client_auth_method, scopes: config.scopes, resource: config.resource, audience: config.audience, @@ -427,6 +450,7 @@ mod tests { scopes: vec!["scope".to_string()], flow: Some("typo".to_string()), client_secret: None, + client_auth_method: None, redirect_uri: None, callback_port: None, use_pkce: None, @@ -453,6 +477,7 @@ mod tests { scopes: vec!["scope".to_string()], flow: Some("client_credentials".to_string()), client_secret: Some("super-secret".to_string()), + client_auth_method: None, redirect_uri: None, callback_port: None, use_pkce: None, @@ -476,6 +501,7 @@ mod tests { scopes: vec!["openid".to_string()], flow: Some("authorization_code".to_string()), client_secret: Some("secret".to_string()), + client_auth_method: None, redirect_uri: Some("http://127.0.0.1:9000/callback".to_string()), callback_port: Some(9000), use_pkce: Some(false), @@ -508,6 +534,7 @@ mod tests { scopes: vec!["openid".to_string()], flow: Some("device_code".to_string()), client_secret: None, + client_auth_method: None, redirect_uri: None, callback_port: None, use_pkce: None, @@ -524,4 +551,62 @@ mod tests { lancedb::remote::oauth::OAuthFlow::DeviceCode )); } + + #[test] + fn test_client_auth_method_conversion() { + use lancedb::remote::oauth::ClientAuthMethod; + + for (value, expected) in [ + ("none", ClientAuthMethod::None), + ("client_secret_basic", ClientAuthMethod::ClientSecretBasic), + ("client_secret_post", ClientAuthMethod::ClientSecretPost), + ] { + let config = OAuthConfig { + issuer_url: "https://issuer.example.com".to_string(), + client_id: "client-id".to_string(), + scopes: vec!["openid".to_string()], + flow: Some("device_code".to_string()), + client_secret: None, + client_auth_method: Some(value.to_string()), + resource: None, + audience: None, + redirect_uri: None, + callback_port: None, + use_pkce: None, + managed_identity_client_id: None, + refresh_buffer_secs: None, + token_cache: None, + }; + + let converted = lancedb::remote::oauth::OAuthConfig::try_from(config).unwrap(); + assert_eq!(converted.client_auth_method, Some(expected)); + } + } + + #[test] + fn test_unknown_client_auth_method_returns_invalid_input() { + let config = OAuthConfig { + issuer_url: "https://issuer.example.com".to_string(), + client_id: "client-id".to_string(), + scopes: vec!["openid".to_string()], + flow: Some("device_code".to_string()), + client_secret: None, + client_auth_method: Some("typo".to_string()), + resource: None, + audience: None, + redirect_uri: None, + callback_port: None, + use_pkce: None, + managed_identity_client_id: None, + refresh_buffer_secs: None, + token_cache: None, + }; + + let err = lancedb::remote::oauth::OAuthConfig::try_from(config).unwrap_err(); + assert!(matches!( + err, + Error::InvalidInput { message } + if message == "Unknown OAuth client auth method: typo" + )); + } } diff --git a/python/python/lancedb/remote/__init__.py b/python/python/lancedb/remote/__init__.py index 602d8dbab..b70a869a3 100644 --- a/python/python/lancedb/remote/__init__.py +++ b/python/python/lancedb/remote/__init__.py @@ -9,7 +9,13 @@ from typing import List, Optional from lancedb import __version__ from .header import HeaderProvider -from .oauth import OAuthConfig, OAuthFlowType, OAuthSession, TokenCacheOptions +from .oauth import ( + ClientAuthMethod, + OAuthConfig, + OAuthFlowType, + OAuthSession, + TokenCacheOptions, +) # The API reference renders this module with a single mkdocstrings directive, # which only picks up names listed here. New public names must be added to this @@ -22,6 +28,7 @@ __all__ = [ "HeaderProvider", "OAuthConfig", "OAuthFlowType", + "ClientAuthMethod", "OAuthSession", "TokenCacheOptions", ] diff --git a/python/python/lancedb/remote/oauth.py b/python/python/lancedb/remote/oauth.py index e8531db1c..106418e95 100644 --- a/python/python/lancedb/remote/oauth.py +++ b/python/python/lancedb/remote/oauth.py @@ -22,6 +22,29 @@ class OAuthFlowType(str, Enum): """Azure Managed Identity via IMDS.""" +class ClientAuthMethod(str, Enum): + """How the client authenticates to the OAuth token endpoint. + + The method applies to every OAuth request that carries client + authentication: client-credentials, authorization-code exchange, + refresh-token, and device-authorization requests. The Azure managed + identity flow ignores this option. + """ + + NONE = "none" + """No client authentication, for public clients using PKCE or the device + flow. Cannot be combined with ``client_secret``.""" + + CLIENT_SECRET_BASIC = "client_secret_basic" + """HTTP Basic authentication. This is the RFC 6749 recommended method and + the normal default for confidential clients, including default Okta + applications. Requires ``client_secret``.""" + + CLIENT_SECRET_POST = "client_secret_post" + """Credentials in the request body, for providers configured to require + it. Requires ``client_secret``.""" + + @dataclass class TokenCacheOptions: """Options for the persistent OAuth token cache. @@ -77,6 +100,13 @@ class OAuthConfig: Authentication flow to use. Default: CLIENT_CREDENTIALS. client_secret : Optional[str] Client secret (required for CLIENT_CREDENTIALS). + client_auth_method : Optional[ClientAuthMethod] + How the client authenticates to the token endpoint (default: auto). + With a ``client_secret`` the default is + ``ClientAuthMethod.CLIENT_SECRET_BASIC``, which matches the RFC 6749 + recommendation and the default configuration of Okta confidential + applications; without a secret the client is public and no client + authentication is sent. redirect_uri : Optional[str] Loopback redirect URI for AUTHORIZATION_CODE. The default is ``http://127.0.0.1:{callback_port}/callback``. @@ -140,8 +170,9 @@ class OAuthConfig: ... flow=OAuthFlowType.AUTHORIZATION_CODE, ... ) - Device Authorization with a persistent cache, so later processes reuse - the session without a new device prompt: + Device Authorization, with a persistent cache so later processes reuse + the session without a new device prompt. The verification URL and user + code are written to standard error before polling begins: >>> config = OAuthConfig( ... issuer_url="https://login.microsoftonline.com/{tenant}/v2.0", @@ -157,6 +188,7 @@ class OAuthConfig: scopes: List[str] flow: OAuthFlowType = OAuthFlowType.CLIENT_CREDENTIALS client_secret: Optional[str] = field(default=None, repr=False) + client_auth_method: Optional[ClientAuthMethod] = None redirect_uri: Optional[str] = None callback_port: Optional[int] = None use_pkce: bool = True diff --git a/python/src/oauth.rs b/python/src/oauth.rs index da5b6e3a3..1639f239a 100644 --- a/python/src/oauth.rs +++ b/python/src/oauth.rs @@ -9,7 +9,7 @@ use pyo3::{FromPyObject, PyResult, Python, pyclass, pymethods}; use crate::error::PythonErrorExt; use crate::runtime::future_into_py; use lancedb::error::Error; -use lancedb::remote::oauth::{AuthorizationCodeOptions, OAuthConfig, OAuthFlow}; +use lancedb::remote::oauth::{AuthorizationCodeOptions, ClientAuthMethod, OAuthConfig, OAuthFlow}; use lancedb::remote::{OAuthSession, SessionLogout, SessionStatus, TokenCacheOptions}; /// Python-side persistent token cache options, extracted via FromPyObject. @@ -42,6 +42,7 @@ pub struct PyOAuthConfig { pub audience: Option, pub flow: String, pub client_secret: Option, + pub client_auth_method: Option, pub redirect_uri: Option, pub callback_port: Option, pub use_pkce: bool, @@ -77,10 +78,23 @@ impl TryFrom for OAuthConfig { } }; + let client_auth_method = match py.client_auth_method.as_deref() { + Some("none") => Some(ClientAuthMethod::None), + Some("client_secret_basic") => Some(ClientAuthMethod::ClientSecretBasic), + Some("client_secret_post") => Some(ClientAuthMethod::ClientSecretPost), + None => None, + Some(other) => { + return Err(Error::InvalidInput { + message: format!("Unknown OAuth client auth method: {other}"), + }); + } + }; + Ok(Self { issuer_url: py.issuer_url, client_id: py.client_id, client_secret: py.client_secret, + client_auth_method, scopes: py.scopes, resource: py.resource, audience: py.audience, @@ -255,6 +269,7 @@ mod tests { scopes: vec!["scope".to_string()], flow: "device_code".to_string(), client_secret: None, + client_auth_method: None, redirect_uri: None, callback_port: None, use_pkce: true, @@ -315,6 +330,38 @@ mod tests { assert!(matches!(converted.flow, OAuthFlow::DeviceCode)); } + #[test] + fn test_client_auth_method_conversion() { + for (value, expected) in [ + ("none", ClientAuthMethod::None), + ("client_secret_basic", ClientAuthMethod::ClientSecretBasic), + ("client_secret_post", ClientAuthMethod::ClientSecretPost), + ] { + let config = PyOAuthConfig { + client_auth_method: Some(value.to_string()), + ..base_config() + }; + + let converted = OAuthConfig::try_from(config).unwrap(); + assert_eq!(converted.client_auth_method, Some(expected)); + } + } + + #[test] + fn test_unknown_client_auth_method_returns_invalid_input() { + let config = PyOAuthConfig { + client_auth_method: Some("typo".to_string()), + ..base_config() + }; + + let err = OAuthConfig::try_from(config).unwrap_err(); + assert!(matches!( + err, + Error::InvalidInput { message } + if message == "Unknown OAuth client auth method: typo" + )); + } + #[test] fn test_token_cache_conversion() { let config = PyOAuthConfig { diff --git a/python/tests/test_oauth.py b/python/tests/test_oauth.py index 583f5a35f..c2aaa6c9f 100644 --- a/python/tests/test_oauth.py +++ b/python/tests/test_oauth.py @@ -63,6 +63,34 @@ def test_device_code_flow_value(): assert oauth.OAuthFlowType.DEVICE_CODE.value == "device_code" +def test_client_auth_method_values(): + oauth = _load_oauth_module() + + assert oauth.ClientAuthMethod.NONE.value == "none" + assert oauth.ClientAuthMethod.CLIENT_SECRET_BASIC.value == "client_secret_basic" + assert oauth.ClientAuthMethod.CLIENT_SECRET_POST.value == "client_secret_post" + + +def test_client_auth_method_defaults_to_none(): + oauth = _load_oauth_module() + + config = oauth.OAuthConfig( + issuer_url="https://issuer.example.com", + client_id="client-id", + scopes=["openid"], + client_auth_method=oauth.ClientAuthMethod.CLIENT_SECRET_POST, + ) + + assert config.client_auth_method is oauth.ClientAuthMethod.CLIENT_SECRET_POST + + default_config = oauth.OAuthConfig( + issuer_url="https://issuer.example.com", + client_id="client-id", + scopes=["openid"], + ) + assert default_config.client_auth_method is None + + def test_token_cache_options_default_to_memory_only(): oauth = _load_oauth_module() diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 039103c5e..fd7803d4a 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -82,6 +82,7 @@ reqwest = { version = "0.12.0", default-features = false, features = [ ], optional = true } tonic = { workspace = true, optional = true } http = { version = "1", optional = true } # Matching what is in reqwest +oauth2 = { workspace = true, optional = true } urlencoding = { version = "2", optional = true } base64 = { version = "0.22", optional = true } fs4 = { version = "0.13", optional = true } @@ -159,6 +160,7 @@ remote = [ "dep:prost", "dep:reqwest", "dep:http", + "dep:oauth2", "dep:tonic", "dep:urlencoding", "dep:base64", diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index a80df819b..3f2979523 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -1546,6 +1546,7 @@ mod tests { client_secret: Some("secret".to_string()), scopes: vec!["scope".to_string()], flow: crate::remote::OAuthFlow::ClientCredentials, + client_auth_method: None, refresh_buffer_secs: None, resource: None, audience: None, @@ -1591,6 +1592,7 @@ mod tests { client_secret: Some("secret".to_string()), scopes: vec!["scope".to_string()], flow: crate::remote::OAuthFlow::ClientCredentials, + client_auth_method: None, refresh_buffer_secs: None, resource: None, audience: None, diff --git a/rust/lancedb/src/remote.rs b/rust/lancedb/src/remote.rs index db8d98f12..4441e01ee 100644 --- a/rust/lancedb/src/remote.rs +++ b/rust/lancedb/src/remote.rs @@ -32,5 +32,7 @@ fn extract_job_id(body: &str) -> Option { pub use client::{ClientConfig, HeaderProvider, RetryConfig, TimeoutConfig, TlsConfig}; pub use db::{RemoteDatabaseOptions, RemoteDatabaseOptionsBuilder}; -pub use oauth::{AuthorizationCodeOptions, OAuthConfig, OAuthFlow, OAuthHeaderProvider}; +pub use oauth::{ + AuthorizationCodeOptions, ClientAuthMethod, OAuthConfig, OAuthFlow, OAuthHeaderProvider, +}; pub use token_cache::{OAuthSession, SessionLogout, SessionStatus, TokenCacheOptions}; diff --git a/rust/lancedb/src/remote/oauth.rs b/rust/lancedb/src/remote/oauth.rs index 516219a14..c7cffa327 100644 --- a/rust/lancedb/src/remote/oauth.rs +++ b/rust/lancedb/src/remote/oauth.rs @@ -1,19 +1,36 @@ // SPDX-License-Identifier: Apache-2.0 // SPDX-FileCopyrightText: Copyright The LanceDB Authors +//! OAuth authentication for LanceDB Cloud connections. +//! +//! Protocol mechanics (authorization URL and CSRF state, PKCE, token +//! exchanges, refresh, device authorization and polling, standard response +//! parsing, and token-endpoint client authentication) are delegated to the +//! [`oauth2`] crate. LanceDB owns the orchestration: OIDC discovery, endpoint +//! validation, the loopback callback server, browser and terminal +//! interaction, timeouts, token caching, and the Azure managed-identity +//! (IMDS) flow. + +use std::borrow::Cow; use std::collections::HashMap; use std::net::{IpAddr, SocketAddr}; +use std::pin::Pin; use std::process::Command; use std::sync::Arc; use std::time::{Duration, Instant}; use async_trait::async_trait; -use base64::Engine; use log::{debug, warn}; -use rand::Rng; +use oauth2::basic::BasicTokenType; +use oauth2::http::{Method, StatusCode}; +use oauth2::{ + AccessToken, AuthType, AuthUrl, ClientId, ClientSecret, CsrfToken, DeviceAuthorizationUrl, + DeviceCodeErrorResponseType, EndpointNotSet, EndpointSet, HttpRequest, HttpResponse, + PkceCodeChallenge, PkceCodeVerifier, RedirectUrl, RefreshToken, RequestTokenError, Scope, + StandardDeviceAuthorizationResponse, StandardTokenIntrospectionResponse, TokenUrl, +}; use reqwest::Client; use serde::Deserialize; -use sha2::{Digest, Sha256}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::RwLock; @@ -166,6 +183,90 @@ pub enum OAuthFlow { }, } +/// How the client authenticates to the OAuth token endpoint. +/// +/// The method applies to every OAuth request that carries client +/// authentication: client-credentials, authorization-code exchange, +/// refresh-token, and device-authorization requests. The Azure managed +/// identity flow ignores this option because it uses its own IMDS protocol. +/// +/// The default (`None` in [`OAuthConfig`]) resolves to +/// [`ClientAuthMethod::ClientSecretBasic`] when a `client_secret` is +/// configured, matching [RFC 6749 section 2.3.1](https://datatracker.ietf.org/doc/html/rfc6749#section-2.3.1) +/// and the default configuration of Okta confidential applications, and to +/// [`ClientAuthMethod::None`] for public clients (no secret), such as +/// authorization-code-with-PKCE or typical device applications. +/// +/// # Example +/// +/// ``` +/// use lancedb::remote::{AuthorizationCodeOptions, ClientAuthMethod, OAuthConfig, OAuthFlow}; +/// +/// let config = OAuthConfig { +/// issuer_url: "https://idp.example.com".to_string(), +/// client_id: "client-id".to_string(), +/// client_secret: Some("secret".to_string()), +/// client_auth_method: Some(ClientAuthMethod::ClientSecretPost), +/// scopes: vec!["openid".to_string()], +/// resource: None, +/// audience: None, +/// flow: OAuthFlow::AuthorizationCode(AuthorizationCodeOptions::new()), +/// refresh_buffer_secs: None, +/// token_cache: None, +/// }; +/// ``` +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ClientAuthMethod { + /// No client authentication (`none`). For public clients such as + /// browser/CLI applications using PKCE or the device flow. Requires that + /// no `client_secret` is configured. + None, + + /// HTTP Basic authentication (`client_secret_basic`), the RFC 6749 + /// recommended method and the normal default for confidential clients, + /// including default Okta applications. Requires a `client_secret`. + ClientSecretBasic, + + /// Credentials in the request body (`client_secret_post`). Some + /// providers are configured to require this method. Requires a + /// `client_secret`. + ClientSecretPost, +} + +impl ClientAuthMethod { + fn auth_type(self) -> AuthType { + match self { + // Without a secret the crate always falls back to sending the + // client_id in the request body, which is the desired behavior + // for public clients. + Self::None | Self::ClientSecretPost => AuthType::RequestBody, + Self::ClientSecretBasic => AuthType::BasicAuth, + } + } +} + +fn resolve_client_auth_method( + method: Option, + client_secret: Option<&str>, +) -> Result { + match (method, client_secret) { + (Some(ClientAuthMethod::None), Some(_)) => Err(Error::InvalidInput { + message: "client_auth_method None cannot be combined with client_secret".to_string(), + }), + ( + Some( + method @ (ClientAuthMethod::ClientSecretBasic | ClientAuthMethod::ClientSecretPost), + ), + None, + ) => Err(Error::InvalidInput { + message: format!("client_auth_method {method:?} requires client_secret to be set"), + }), + (Some(method), _) => Ok(method), + (None, Some(_)) => Ok(ClientAuthMethod::ClientSecretBasic), + (None, None) => Ok(ClientAuthMethod::None), + } +} + /// OAuth configuration for LanceDB authentication. /// /// All token acquisition and refresh is handled in the Rust layer. @@ -201,6 +302,11 @@ pub struct OAuthConfig { /// Authentication flow to use. pub flow: OAuthFlow, + /// How the client authenticates to the token endpoint. See + /// [`ClientAuthMethod`] for the resolution rules that apply when this is + /// `None` (the default). + pub client_auth_method: Option, + /// Seconds before token expiry to trigger proactive refresh (default: 300). /// Keep this well below the token TTL; if it is greater than or equal to /// the TTL, each request refreshes the token. @@ -228,6 +334,7 @@ impl std::fmt::Debug for OAuthConfig { .field("resource", &self.resource) .field("audience", &self.audience) .field("flow", &self.flow) + .field("client_auth_method", &self.client_auth_method) .field("refresh_buffer_secs", &self.refresh_buffer_secs) .field("token_cache", &self.token_cache) .finish() @@ -245,18 +352,71 @@ struct OidcDiscovery { // -- Token Response -- +/// A token endpoint success response. +/// +/// This implements [`oauth2::TokenResponse`] so the `oauth2` crate can parse +/// provider responses directly, while keeping LanceDB's lenient field +/// handling: `expires_in` may be an integer, an integer-valued float, or a +/// numeric string, and `token_type` is optional. #[derive(Deserialize)] pub(crate) struct TokenResponse { - pub(crate) access_token: String, + pub(crate) access_token: AccessToken, #[serde(default)] - pub(crate) refresh_token: Option, + pub(crate) refresh_token: Option, /// Token lifetime in seconds. /// Some providers (Azure IMDS) return this as a string, so we accept both. #[serde(default, deserialize_with = "deserialize_optional_u64_or_string")] pub(crate) expires_in: Option, #[serde(default)] - #[allow(dead_code)] - pub(crate) token_type: Option, + pub(crate) token_type: Option, +} + +const BEARER: BasicTokenType = BasicTokenType::Bearer; + +impl oauth2::TokenResponse for TokenResponse { + type TokenType = BasicTokenType; + + fn access_token(&self) -> &AccessToken { + &self.access_token + } + + fn token_type(&self) -> &BasicTokenType { + self.token_type.as_ref().unwrap_or(&BEARER) + } + + fn expires_in(&self) -> Option { + self.expires_in.map(Duration::from_secs) + } + + fn refresh_token(&self) -> Option<&RefreshToken> { + self.refresh_token.as_ref() + } + + fn scopes(&self) -> Option<&Vec> { + None + } +} + +// The oauth2::TokenResponse trait requires Serialize; LanceDB never +// serializes token responses, so redact every credential-bearing field rather +// than risk leaking one through an accidental serialization. +impl serde::Serialize for TokenResponse { + fn serialize( + &self, + serializer: S, + ) -> std::result::Result { + use serde::ser::SerializeStruct; + + let mut state = serializer.serialize_struct("TokenResponse", 4)?; + state.serialize_field("access_token", "")?; + state.serialize_field( + "refresh_token", + &self.refresh_token.as_ref().map(|_| ""), + )?; + state.serialize_field("expires_in", &self.expires_in)?; + state.serialize_field("token_type", &self.token_type)?; + state.end() + } } impl std::fmt::Debug for TokenResponse { @@ -349,9 +509,9 @@ impl TokenState { } fn update(&mut self, resp: &TokenResponse) { - self.access_token = Some(resp.access_token.clone()); - if resp.refresh_token.is_some() { - self.refresh_token = resp.refresh_token.clone(); + self.access_token = Some(resp.access_token.secret().clone()); + if let Some(token) = resp.refresh_token.as_ref() { + self.refresh_token = Some(token.secret().clone()); } let expires_in = resp.expires_in.unwrap_or(DEFAULT_TOKEN_TTL_SECS); self.expires_at = Some(Instant::now() + Duration::from_secs(expires_in)); @@ -374,14 +534,203 @@ pub(crate) enum RefreshResult { Unsupported, } +// -- OAuth HTTP transport -- + +/// Errors raised by [`OAuthHttpClient`]. +#[derive(Debug)] +enum OAuthHttpError { + /// The request could not be built (invalid method, URL, or headers). + Build(String), + /// The request failed at the transport layer. This includes redirects + /// rejected by the hardened client redirect policy. + Transport(reqwest::Error), + /// The server reported a transient condition: HTTP 429, a 5xx status, or + /// an OAuth `temporarily_unavailable` error. The device-code poll loop + /// treats these as retryable; single-shot requests surface them as errors. + Transient(StatusCode), +} + +impl std::fmt::Display for OAuthHttpError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Build(message) => write!(f, "could not build OAuth request: {message}"), + Self::Transport(error) => { + write!(f, "OAuth HTTP request failed: {error}")?; + // Include the underlying cause (e.g. a redirect rejected by + // the hardened client policy) without ever including bodies. + if let Some(source) = std::error::Error::source(error) { + write!(f, ": {source}")?; + } + Ok(()) + } + Self::Transient(status) => { + write!(f, "OAuth server returned a transient response ({status})") + } + } + } +} + +impl std::error::Error for OAuthHttpError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Self::Transport(error) => Some(error), + _ => None, + } + } +} + +#[derive(Clone)] +struct OAuthHttpClient { + inner: Client, +} + +impl OAuthHttpClient { + fn is_retryable_status_or_body(status: StatusCode, body: &[u8]) -> bool { + if status.as_u16() == 429 || status.is_server_error() { + return true; + } + serde_json::from_slice::(body) + .map(|error| error.error == "temporarily_unavailable") + .unwrap_or(false) + } +} + +impl<'c> oauth2::AsyncHttpClient<'c> for OAuthHttpClient { + type Error = OAuthHttpError; + type Future = Pin< + Box< + dyn Future> + + Send + + Sync + + 'c, + >, + >; + + fn call(&'c self, request: HttpRequest) -> Self::Future { + Box::pin(async move { + let (parts, body) = request.into_parts(); + let method = Method::from_bytes(parts.method.as_str().as_bytes()) + .map_err(|e| OAuthHttpError::Build(e.to_string()))?; + let url: Url = parts + .uri + .to_string() + .parse() + .map_err(|e| OAuthHttpError::Build(format!("invalid request URL: {e}")))?; + + let response = self + .inner + .request(method, url) + .headers(parts.headers) + .body(body) + .send() + .await + .map_err(OAuthHttpError::Transport)?; + + let status = response.status(); + let mut builder = oauth2::http::Response::builder().status(status); + for (name, value) in response.headers().iter() { + builder = builder.header(name, value); + } + let body = response + .bytes() + .await + .map_err(OAuthHttpError::Transport)? + .to_vec(); + let response = builder + .body(body) + .map_err(|e| OAuthHttpError::Build(e.to_string()))?; + + if !status.is_success() && Self::is_retryable_status_or_body(status, response.body()) { + debug!("OAuth token endpoint returned a transient response ({status})"); + return Err(OAuthHttpError::Transient(status)); + } + Ok(response) + }) + } +} + +// -- OAuth client construction -- + +type OauthClient = oauth2::Client< + oauth2::basic::BasicErrorResponse, + TokenResponse, + StandardTokenIntrospectionResponse, + oauth2::StandardRevocableToken, + oauth2::basic::BasicErrorResponse, + HasAuthUrl, + HasDeviceAuthUrl, + EndpointNotSet, + EndpointNotSet, + HasTokenUrl, +>; + +type BaseOauthClient = OauthClient; + +type TokenEndpointClient = OauthClient; + +fn token_error_context(context: &str, response: &T) -> String { + // StandardErrorResponse's Display renders only the provider's error code, + // description, and error URI; it never includes credential material. + format!("{context} failed: {response}") +} + +fn map_token_error( + error: RequestTokenError, + context: &str, +) -> Error { + match error { + RequestTokenError::ServerResponse(response) => Error::Runtime { + message: token_error_context(context, &response), + }, + RequestTokenError::Request(error) => Error::Runtime { + message: format!("{context} failed: {error}"), + }, + // Never include the raw body: it may contain credential material. + RequestTokenError::Parse(error, _) => Error::Runtime { + message: format!("{context} response could not be parsed: {error}"), + }, + RequestTokenError::Other(message) => Error::Runtime { + message: format!("{context} failed: {message}"), + }, + } +} + +fn map_device_token_error( + error: RequestTokenError, +) -> Error { + match error { + RequestTokenError::ServerResponse(response) => match response.error() { + DeviceCodeErrorResponseType::AccessDenied => Error::Runtime { + message: "Device authorization was denied by the user".to_string(), + }, + DeviceCodeErrorResponseType::ExpiredToken => Error::Runtime { + message: "Device authorization expired before authentication completed".to_string(), + }, + _ => Error::Runtime { + message: token_error_context("Device token request", &response), + }, + }, + RequestTokenError::Request(error) => Error::Runtime { + message: format!("Device token request failed: {error}"), + }, + RequestTokenError::Parse(error, _) => Error::Runtime { + message: format!("Device token response could not be parsed: {error}"), + }, + RequestTokenError::Other(message) => Error::Runtime { + message: format!("Device token request failed: {message}"), + }, + } +} + struct OidcClient { issuer_url: String, client_id: String, client_secret: Option, + client_auth_method: ClientAuthMethod, scopes: Vec, resource: Option, audience: Option, - http_client: Client, + http_client: OAuthHttpClient, discovery: RwLock>, } @@ -394,6 +743,7 @@ impl std::fmt::Debug for OidcClient { "client_secret", &self.client_secret.as_ref().map(|_| ""), ) + .field("client_auth_method", &self.client_auth_method) .field("scopes", &self.scopes) .field("resource", &self.resource) .field("audience", &self.audience) @@ -406,6 +756,7 @@ impl OidcClient { issuer_url: String, client_id: String, client_secret: Option, + client_auth_method: ClientAuthMethod, scopes: Vec, resource: Option, audience: Option, @@ -431,10 +782,11 @@ impl OidcClient { issuer_url, client_id, client_secret, + client_auth_method, scopes, resource, audience, - http_client, + http_client: OAuthHttpClient { inner: http_client }, discovery: RwLock::new(None), }) } @@ -466,6 +818,7 @@ impl OidcClient { let resp = self .http_client + .inner .get(&discovery_url) .send() .await @@ -504,7 +857,9 @@ impl OidcClient { self.get_discovery().await.map(|disc| disc.token_endpoint) } - fn target_params(&self) -> impl Iterator { + /// Resource/audience parameters forwarded to every authorization and + /// token request. + fn target_params(&self) -> impl Iterator { self.resource .as_deref() .map(|value| ("resource", value)) @@ -512,92 +867,39 @@ impl OidcClient { .chain(self.audience.as_deref().map(|value| ("audience", value))) } - fn scopes_string(&self) -> String { - self.scopes.join(" ") + fn base_client(&self) -> BaseOauthClient { + let mut client = oauth2::Client::new(ClientId::new(self.client_id.clone())) + .set_auth_type(self.client_auth_method.auth_type()); + if let Some(secret) = self.client_secret.as_ref() { + client = client.set_client_secret(ClientSecret::new(secret.clone())); + } + client } - async fn post_token_request( - &self, - endpoint: &str, - params: &[(String, String)], - ) -> Result { - let mut params = params.to_vec(); - params.extend( - self.target_params() - .map(|(key, value)| (key.to_owned(), value.to_owned())), - ); - let resp = self - .http_client - .post(endpoint) - .form(¶ms) - .send() - .await - .map_err(|e| Error::Runtime { - message: format!("Token request to {endpoint} failed: {e}"), - })?; - - if !resp.status().is_success() { - return Err(Error::Runtime { - message: format!( - "Token request failed with status {}: {}", - resp.status(), - resp.text().await.unwrap_or_default() - ), - }); - } - - resp.json().await.map_err(|e| Error::Runtime { - message: format!("Failed to parse token response: {e}"), - }) + async fn token_client(&self) -> Result<(TokenEndpointClient, String)> { + let endpoint = self.get_token_endpoint().await?; + let token_url = TokenUrl::new(endpoint.clone()).map_err(|e| Error::InvalidInput { + message: format!("Invalid OAuth token_endpoint: {e}"), + })?; + Ok((self.base_client().set_token_uri(token_url), endpoint)) } async fn refresh_token(&self, refresh_token: &str) -> Result { - let endpoint = self.get_token_endpoint().await?; - let mut params = vec![ - ("grant_type".to_string(), "refresh_token".to_string()), - ("client_id".to_string(), self.client_id.clone()), - ("refresh_token".to_string(), refresh_token.to_string()), - ]; - if let Some(secret) = self.client_secret.as_ref() { - params.push(("client_secret".to_string(), secret.clone())); + let (client, _) = self.token_client().await?; + let refresh_token = RefreshToken::new(refresh_token.to_string()); + let mut request = client.exchange_refresh_token(&refresh_token); + for (name, value) in self.target_params() { + request = request.add_extra_param(name, value); } - params.extend( - self.target_params() - .map(|(key, value)| (key.to_owned(), value.to_owned())), - ); - let response = self - .http_client - .post(&endpoint) - .form(¶ms) - .send() - .await - .map_err(|e| Error::Runtime { - message: format!("Refresh token request to {endpoint} failed: {e}"), - })?; - if response.status().is_success() { - return response - .json() - .await - .map(RefreshResult::Refreshed) - .map_err(|e| Error::Runtime { - message: format!("Failed to parse refresh token response: {e}"), - }); + match request.request_async(&self.http_client).await { + Ok(response) => Ok(RefreshResult::Refreshed(response)), + Err(RequestTokenError::ServerResponse(response)) + if matches!(response.error().as_ref(), "invalid_grant" | "invalid_token") => + { + Ok(RefreshResult::Reauthenticate) + } + Err(error) => Err(map_token_error(error, "Refresh token request")), } - - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - let error_code = serde_json::from_str::(&body) - .ok() - .map(|error| error.error); - if matches!( - error_code.as_deref(), - Some("invalid_grant" | "invalid_token") - ) { - return Ok(RefreshResult::Reauthenticate); - } - Err(Error::Runtime { - message: format!("Refresh token request failed with status {status}: {body}"), - }) } } @@ -618,6 +920,7 @@ impl ClientCredentialsSource { issuer_url: String, client_id: String, client_secret: Option, + client_auth_method: ClientAuthMethod, scopes: Vec, resource: Option, audience: Option, @@ -632,6 +935,7 @@ impl ClientCredentialsSource { issuer_url, client_id, client_secret, + client_auth_method, scopes, resource, audience, @@ -643,18 +947,19 @@ impl ClientCredentialsSource { #[async_trait] impl TokenSource for ClientCredentialsSource { async fn fetch_token(&self) -> Result { - let token_endpoint = self.oidc.get_token_endpoint().await?; - let params = [ - ("grant_type".to_string(), "client_credentials".to_string()), - ("client_id".to_string(), self.oidc.client_id.clone()), - ( - "client_secret".to_string(), - self.oidc.client_secret.clone().expect("validated in new"), - ), - ("scope".to_string(), self.oidc.scopes_string()), - ]; + let (client, endpoint) = self.oidc.token_client().await?; + let mut request = client.exchange_client_credentials(); + for scope in &self.oidc.scopes { + request = request.add_scope(Scope::new(scope.clone())); + } + for (name, value) in self.oidc.target_params() { + request = request.add_extra_param(name, value); + } - self.oidc.post_token_request(&token_endpoint, ¶ms).await + request + .request_async(&self.oidc.http_client) + .await + .map_err(|e| map_token_error(e, &format!("Token request to {endpoint}"))) } } @@ -735,7 +1040,7 @@ impl ResolvedRedirect { struct AuthorizationRequest { url: Url, state: String, - code_verifier: Option, + code_verifier: Option, } #[derive(Debug, PartialEq)] @@ -761,10 +1066,12 @@ impl std::fmt::Debug for AuthorizationCodeSource { } impl AuthorizationCodeSource { + #[allow(clippy::too_many_arguments)] fn new( issuer_url: String, client_id: String, client_secret: Option, + client_auth_method: ClientAuthMethod, scopes: Vec, resource: Option, audience: Option, @@ -776,6 +1083,7 @@ impl AuthorizationCodeSource { issuer_url, client_id, client_secret, + client_auth_method, scopes, resource, audience, @@ -794,32 +1102,40 @@ impl AuthorizationCodeSource { .ok_or(Error::Runtime { message: "OIDC discovery did not provide authorization_endpoint".to_string(), })?; - let mut url = validate_oauth_url(&endpoint, "authorization_endpoint")?; - let state = random_urlsafe_string(32); - let code_verifier = self.options.use_pkce.then(|| random_urlsafe_string(64)); + let auth_url = AuthUrl::new(endpoint).map_err(|e| Error::InvalidInput { + message: format!("Invalid OAuth authorization_endpoint: {e}"), + })?; + let redirect_url = + RedirectUrl::new(self.redirect.uri.clone()).map_err(|e| Error::InvalidInput { + message: format!("Invalid OAuth redirect_uri: {e}"), + })?; - { - let mut query = url.query_pairs_mut(); - query - .append_pair("response_type", "code") - .append_pair("client_id", &self.oidc.client_id) - .append_pair("redirect_uri", &self.redirect.uri) - .append_pair("scope", &self.oidc.scopes_string()) - .append_pair("state", &state); - query.extend_pairs(self.oidc.target_params()); - if let Some(verifier) = code_verifier.as_ref() { - let challenge = base64::engine::general_purpose::URL_SAFE_NO_PAD - .encode(Sha256::digest(verifier.as_bytes())); - query - .append_pair("code_challenge", &challenge) - .append_pair("code_challenge_method", "S256"); - } + let pkce = self + .options + .use_pkce + .then(PkceCodeChallenge::new_random_sha256); + + let client = self + .oidc + .base_client() + .set_auth_uri(auth_url) + .set_redirect_uri(redirect_url); + let mut request = client.authorize_url(CsrfToken::new_random); + for scope in &self.oidc.scopes { + request = request.add_scope(Scope::new(scope.clone())); } + for (name, value) in self.oidc.target_params() { + request = request.add_extra_param(name, value); + } + if let Some((challenge, _)) = pkce.as_ref() { + request = request.set_pkce_challenge(challenge.clone()); + } + let (url, state) = request.url(); Ok(AuthorizationRequest { url, - state, - code_verifier, + state: state.secret().clone(), + code_verifier: pkce.map(|(_, verifier)| verifier), }) } @@ -872,22 +1188,27 @@ impl AuthorizationCodeSource { async fn exchange_code( &self, code: &str, - code_verifier: Option<&str>, + code_verifier: Option, ) -> Result { - let endpoint = self.oidc.get_token_endpoint().await?; - let mut params = vec![ - ("grant_type".to_string(), "authorization_code".to_string()), - ("client_id".to_string(), self.oidc.client_id.clone()), - ("code".to_string(), code.to_string()), - ("redirect_uri".to_string(), self.redirect.uri.clone()), - ]; + let (client, endpoint) = self.oidc.token_client().await?; + let redirect_url = + RedirectUrl::new(self.redirect.uri.clone()).map_err(|e| Error::InvalidInput { + message: format!("Invalid OAuth redirect_uri: {e}"), + })?; + let mut request = client + .exchange_code(oauth2::AuthorizationCode::new(code.to_string())) + .set_redirect_uri(Cow::Owned(redirect_url)); + for (name, value) in self.oidc.target_params() { + request = request.add_extra_param(name, value); + } if let Some(verifier) = code_verifier { - params.push(("code_verifier".to_string(), verifier.to_string())); + request = request.set_pkce_verifier(verifier); } - if let Some(secret) = self.oidc.client_secret.as_ref() { - params.push(("client_secret".to_string(), secret.clone())); - } - self.oidc.post_token_request(&endpoint, ¶ms).await + + request + .request_async(&self.oidc.http_client) + .await + .map_err(|e| map_token_error(e, &format!("Token request to {endpoint}"))) } } @@ -906,8 +1227,7 @@ impl TokenSource for AuthorizationCodeSource { show_oauth_prompt(&authorization_prompt(&request.url)); launch_browser(request.url.clone()); let code = self.wait_for_callback(&listener, &request.state).await?; - self.exchange_code(&code, request.code_verifier.as_deref()) - .await + self.exchange_code(&code, request.code_verifier).await } async fn refresh_token(&self, refresh_token: &str) -> Result { @@ -915,23 +1235,11 @@ impl TokenSource for AuthorizationCodeSource { } } -#[derive(Deserialize)] -struct DeviceAuthorizationResponse { - device_code: String, - user_code: String, - verification_uri: String, - #[serde(default)] - verification_uri_complete: Option, - expires_in: u64, - #[serde(default)] - interval: Option, -} - +/// A minimal OAuth error body, used to sniff retryable `temporarily_unavailable` +/// responses in [`OAuthHttpClient`]. Unknown fields are ignored. #[derive(Debug, Deserialize)] struct OAuthErrorResponse { error: String, - #[serde(default)] - error_description: Option, } struct DeviceCodeSource { @@ -951,6 +1259,7 @@ impl DeviceCodeSource { issuer_url: String, client_id: String, client_secret: Option, + client_auth_method: ClientAuthMethod, scopes: Vec, resource: Option, audience: Option, @@ -960,6 +1269,7 @@ impl DeviceCodeSource { issuer_url, client_id, client_secret, + client_auth_method, scopes, resource, audience, @@ -967,7 +1277,7 @@ impl DeviceCodeSource { }) } - async fn request_device_authorization(&self) -> Result { + async fn request_device_authorization(&self) -> Result { let endpoint = self .oidc .get_discovery() @@ -976,145 +1286,57 @@ impl DeviceCodeSource { .ok_or(Error::Runtime { message: "OIDC discovery did not provide device_authorization_endpoint".to_string(), })?; - let mut params = vec![ - ("client_id".to_string(), self.oidc.client_id.clone()), - ("scope".to_string(), self.oidc.scopes_string()), - ]; - if let Some(secret) = self.oidc.client_secret.as_ref() { - params.push(("client_secret".to_string(), secret.clone())); - } - params.extend( - self.oidc - .target_params() - .map(|(key, value)| (key.to_owned(), value.to_owned())), - ); - let response = self + let device_url = + DeviceAuthorizationUrl::new(endpoint.clone()).map_err(|e| Error::InvalidInput { + message: format!("Invalid OAuth device_authorization_endpoint: {e}"), + })?; + + let client = self .oidc - .http_client - .post(&endpoint) - .form(¶ms) - .send() - .await - .map_err(|e| Error::Runtime { - message: format!("Device authorization request to {endpoint} failed: {e}"), - })?; - if !response.status().is_success() { - return Err(Error::Runtime { - message: format!( - "Device authorization request failed with status {}: {}", - response.status(), - response.text().await.unwrap_or_default() - ), - }); + .base_client() + .set_device_authorization_url(device_url); + let mut request = client.exchange_device_code(); + for scope in &self.oidc.scopes { + request = request.add_scope(Scope::new(scope.clone())); } - let device: DeviceAuthorizationResponse = - response.json().await.map_err(|e| Error::Runtime { - message: format!("Failed to parse device authorization response: {e}"), + for (name, value) in self.oidc.target_params() { + request = request.add_extra_param(name, value); + } + let device: StandardDeviceAuthorizationResponse = request + .request_async(&self.oidc.http_client) + .await + .map_err(|e| { + map_token_error(e, &format!("Device authorization request to {endpoint}")) })?; - validate_oauth_url(&device.verification_uri, "verification_uri")?; - if let Some(uri) = device.verification_uri_complete.as_deref() { - validate_oauth_url(uri, "verification_uri_complete")?; + + validate_oauth_url(device.verification_uri().as_str(), "verification_uri")?; + if let Some(uri) = device.verification_uri_complete() { + validate_oauth_url(uri.secret(), "verification_uri_complete")?; } Ok(device) } - async fn poll_for_token(&self, device: &DeviceAuthorizationResponse) -> Result { - let endpoint = self.oidc.get_token_endpoint().await?; - let deadline = TokioInstant::now() + Duration::from_secs(device.expires_in); - let mut interval = Duration::from_secs(device.interval.unwrap_or(5).max(1)); - - loop { - let now = TokioInstant::now(); - if now >= deadline { - return Err(Error::Runtime { - message: "Device authorization expired before authentication completed" - .to_string(), - }); - } - tokio::time::sleep_until(std::cmp::min(now + interval, deadline)).await; - if TokioInstant::now() >= deadline { - return Err(Error::Runtime { - message: "Device authorization expired before authentication completed" - .to_string(), - }); - } - - let mut params = vec![ - ( - "grant_type".to_string(), - "urn:ietf:params:oauth:grant-type:device_code".to_string(), - ), - ("client_id".to_string(), self.oidc.client_id.clone()), - ("device_code".to_string(), device.device_code.clone()), - ]; - if let Some(secret) = self.oidc.client_secret.as_ref() { - params.push(("client_secret".to_string(), secret.clone())); - } - - params.extend( - self.oidc - .target_params() - .map(|(key, value)| (key.to_owned(), value.to_owned())), - ); - let response = match self - .oidc - .http_client - .post(&endpoint) - .form(¶ms) - .send() - .await - { - Ok(response) => response, - Err(error) => { - warn!("Device token request to {endpoint} failed; retrying: {error}"); - continue; - } - }; - if response.status().is_success() { - return response.json().await.map_err(|e| Error::Runtime { - message: format!("Failed to parse device token response: {e}"), - }); - } - - let status = response.status(); - let body = response.text().await.unwrap_or_default(); - let oauth_error = serde_json::from_str::(&body).ok(); - match oauth_error.as_ref().map(|error| error.error.as_str()) { - Some("authorization_pending") => continue, - Some("slow_down") => { - interval += Duration::from_secs(5); - continue; - } - Some("temporarily_unavailable") => continue, - Some("access_denied") => { - return Err(Error::Runtime { - message: "Device authorization was denied by the user".to_string(), - }); - } - Some("expired_token") => { - return Err(Error::Runtime { - message: "Device authorization expired before authentication completed" - .to_string(), - }); - } - _ if status == reqwest::StatusCode::TOO_MANY_REQUESTS - || status.is_server_error() => - { - warn!("Device token endpoint returned {status}; retrying"); - continue; - } - _ => { - let detail = oauth_error - .and_then(|error| error.error_description) - .unwrap_or(body); - return Err(Error::Runtime { - message: format!( - "Device token request failed with status {status}: {detail}" - ), - }); - } - } + async fn poll_for_token( + &self, + device: &StandardDeviceAuthorizationResponse, + ) -> Result { + let (client, _) = self.oidc.token_client().await?; + let mut request = client + .exchange_device_access_token(device) + .set_max_backoff_interval(Duration::from_secs(10)); + for (name, value) in self.oidc.target_params() { + request = request.add_extra_param(name, value); } + request + .request_async( + &self.oidc.http_client, + // RFC 8628: poll slowly; never spin faster than once a second + // even if a misbehaving server reports a zero interval. + |interval: Duration| tokio::time::sleep(interval.max(Duration::from_secs(1))), + None, + ) + .await + .map_err(map_device_token_error) } } @@ -1122,12 +1344,14 @@ impl DeviceCodeSource { impl TokenSource for DeviceCodeSource { async fn fetch_token(&self) -> Result { let device = self.request_device_authorization().await?; - show_oauth_prompt(&device_prompt(&device.verification_uri, &device.user_code)); + show_oauth_prompt(&device_prompt( + device.verification_uri().as_str(), + device.user_code().secret(), + )); let (browser_url, name) = device - .verification_uri_complete - .as_deref() - .map(|url| (url, "verification_uri_complete")) - .unwrap_or((&device.verification_uri, "verification_uri")); + .verification_uri_complete() + .map(|uri| (uri.secret().as_str(), "verification_uri_complete")) + .unwrap_or((device.verification_uri().as_str(), "verification_uri")); launch_browser(validate_oauth_url(browser_url, name)?); self.poll_for_token(&device).await } @@ -1137,14 +1361,6 @@ impl TokenSource for DeviceCodeSource { } } -fn random_urlsafe_string(length: usize) -> String { - const CHARSET: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-._~"; - let mut rng = rand::rng(); - (0..length) - .map(|_| CHARSET[rng.random_range(0..CHARSET.len())] as char) - .collect() -} - fn launch_browser(url: Url) { drop(tokio::task::spawn_blocking(move || { if let Some(browser) = std::env::var_os("LANCEDB_OAUTH_BROWSER") { @@ -1375,11 +1591,14 @@ pub(crate) fn build_token_source(config: &OAuthConfig) -> Result Box::new(ClientCredentialsSource::new( config.issuer_url.clone(), config.client_id.clone(), config.client_secret.clone(), + client_auth_method, config.scopes.clone(), config.resource.clone(), config.audience.clone(), @@ -1388,6 +1607,7 @@ pub(crate) fn build_token_source(config: &OAuthConfig) -> Result Result, + expires_in: Option, + ) -> TokenResponse { + TokenResponse { + access_token: AccessToken::new(access_token.to_string()), + refresh_token: refresh_token.map(|token| RefreshToken::new(token.to_string())), + expires_in, + token_type: None, + } + } + + fn basic_authorization(client_id: &str, client_secret: &str) -> String { + format!( + "Basic {}", + base64::engine::general_purpose::STANDARD + .encode(format!("{client_id}:{client_secret}")) + ) + } + + struct CapturedRequest { + line: String, + headers: String, + body: String, + } + + impl CapturedRequest { + fn header(&self, name: &str) -> Option { + self.headers.lines().find_map(|line| { + let (key, value) = line.split_once(':')?; + key.trim() + .eq_ignore_ascii_case(name) + .then(|| value.trim().to_string()) + }) + } + } + #[tokio::test] async fn test_target_parameters_across_oauth_flows() { for (resource, audience) in [ @@ -1546,15 +1807,16 @@ mod tests { // Three discovery requests and six form submissions. for _ in 0..9 { let (mut stream, _) = listener.accept().await.unwrap(); - let (line, body) = read_http_request(&mut stream).await; - let response = if line.starts_with("GET ") { + let request = read_http_request(&mut stream).await; + let response = if request.line.starts_with("GET ") { serde_json::json!({ "token_endpoint": format!("http://{addr}/token"), "authorization_endpoint": format!("http://{addr}/authorize"), "device_authorization_endpoint": format!("http://{addr}/device"), }) } else { - let params: Vec<_> = url::form_urlencoded::parse(body.as_bytes()).collect(); + let params: Vec<_> = + url::form_urlencoded::parse(request.body.as_bytes()).collect(); for (key, expected) in [("resource", resource), ("audience", audience)] { let values: Vec<_> = params .iter() @@ -1563,7 +1825,7 @@ mod tests { .collect(); assert_eq!(values, expected.into_iter().collect::>()); } - if line.starts_with("POST /device ") { + if request.line.starts_with("POST /device ") { serde_json::json!({ "device_code": "device-code", "user_code": "ABCD", "verification_uri": format!("http://{addr}/verify"), @@ -1598,6 +1860,7 @@ mod tests { issuer.clone(), "client".into(), Some("secret".into()), + ClientAuthMethod::ClientSecretBasic, vec!["scope".into()], resource.map(str::to_owned), audience.map(str::to_owned), @@ -1608,6 +1871,7 @@ mod tests { issuer.clone(), "client".into(), None, + ClientAuthMethod::None, vec!["scope".into()], resource.map(str::to_owned), audience.map(str::to_owned), @@ -1628,7 +1892,7 @@ mod tests { ); } browser - .exchange_code("code", Some("verifier")) + .exchange_code("code", Some(PkceCodeVerifier::new("verifier".to_string()))) .await .unwrap(); browser.refresh_token("refresh").await.unwrap(); @@ -1636,6 +1900,7 @@ mod tests { issuer, "client".into(), None, + ClientAuthMethod::None, vec!["scope".into()], resource.map(str::to_owned), audience.map(str::to_owned), @@ -1656,9 +1921,10 @@ mod tests { client_id: "client".into(), client_secret: None, scopes: vec!["api://app/.default".into()], - flow: OAuthFlow::AzureManagedIdentity { client_id: None }, resource: resource.map(str::to_owned), audience: audience.map(str::to_owned), + flow: OAuthFlow::AzureManagedIdentity { client_id: None }, + client_auth_method: None, refresh_buffer_secs: None, token_cache: None, }; @@ -1686,14 +1952,7 @@ mod tests { #[test] fn test_token_state_uses_default_expiry() { let mut state = TokenState::new(); - let response = TokenResponse { - access_token: "tok".to_string(), - refresh_token: None, - expires_in: None, - token_type: None, - }; - - state.update(&response); + state.update(&token_response("tok", None, None)); assert!(!state.is_expired(Duration::from_secs(DEFAULT_TOKEN_TTL_SECS - 1))); assert!(state.is_expired(Duration::from_secs(DEFAULT_TOKEN_TTL_SECS + 1))); @@ -1702,18 +1961,8 @@ mod tests { #[test] fn test_token_state_retains_refresh_token_when_not_rotated() { let mut state = TokenState::new(); - state.update(&TokenResponse { - access_token: "token-1".to_string(), - refresh_token: Some("refresh-1".to_string()), - expires_in: Some(60), - token_type: None, - }); - state.update(&TokenResponse { - access_token: "token-2".to_string(), - refresh_token: None, - expires_in: Some(60), - token_type: None, - }); + state.update(&token_response("token-1", Some("refresh-1"), Some(60))); + state.update(&token_response("token-2", None, Some(60))); assert_eq!(state.refresh_token.as_deref(), Some("refresh-1")); } @@ -1738,10 +1987,10 @@ mod tests { #[test] fn test_token_response_debug_redacts_access_token() { let response = TokenResponse { - access_token: "secret-token".to_string(), - refresh_token: Some("secret-refresh-token".to_string()), + access_token: AccessToken::new("secret-token".to_string()), + refresh_token: Some(RefreshToken::new("secret-refresh-token".to_string())), expires_in: Some(3600), - token_type: Some("Bearer".to_string()), + token_type: Some(BasicTokenType::Bearer), }; let debug = format!("{response:?}"); @@ -1751,18 +2000,53 @@ mod tests { } #[test] - fn test_scopes_string() { - let source = ClientCredentialsSource::new( - "https://login.microsoftonline.com/tenant/v2.0".to_string(), - "app-id".to_string(), - Some("secret".to_string()), - vec!["scope1".to_string(), "scope2".to_string()], - None, - None, - ) - .unwrap(); + fn test_client_auth_method_defaults() { + assert_eq!( + resolve_client_auth_method(None, Some("secret")).unwrap(), + ClientAuthMethod::ClientSecretBasic + ); + assert_eq!( + resolve_client_auth_method(None, None).unwrap(), + ClientAuthMethod::None + ); + } - assert_eq!(source.oidc.scopes_string(), "scope1 scope2"); + #[test] + fn test_client_auth_method_explicit_values() { + for method in [ + ClientAuthMethod::None, + ClientAuthMethod::ClientSecretBasic, + ClientAuthMethod::ClientSecretPost, + ] { + let secret = (method != ClientAuthMethod::None).then_some("secret"); + assert_eq!( + resolve_client_auth_method(Some(method), secret).unwrap(), + method + ); + } + } + + #[test] + fn test_client_auth_method_rejects_inconsistent_configuration() { + let err = + resolve_client_auth_method(Some(ClientAuthMethod::None), Some("secret")).unwrap_err(); + assert!(matches!( + err, + Error::InvalidInput { message } + if message == "client_auth_method None cannot be combined with client_secret" + )); + + for method in [ + ClientAuthMethod::ClientSecretBasic, + ClientAuthMethod::ClientSecretPost, + ] { + let err = resolve_client_auth_method(Some(method), None).unwrap_err(); + assert!(matches!( + err, + Error::InvalidInput { message } + if message == format!("client_auth_method {method:?} requires client_secret to be set") + )); + } } #[test] @@ -1921,6 +2205,7 @@ mod tests { "http://127.0.0.1:1".to_string(), "client-id".to_string(), None, + ClientAuthMethod::None, vec!["openid".to_string()], None, None, @@ -1999,6 +2284,7 @@ mod tests { issuer_url, "client-id".to_string(), None, + ClientAuthMethod::None, vec!["openid".to_string(), "profile".to_string()], None, None, @@ -2012,10 +2298,19 @@ mod tests { params.get("response_type").map(String::as_str), Some("code") ); + assert_eq!( + params.get("client_id").map(String::as_str), + Some("client-id") + ); + assert_eq!( + params.get("redirect_uri").map(String::as_str), + Some("http://127.0.0.1:8400/callback") + ); assert_eq!( params.get("scope").map(String::as_str), Some("openid profile") ); + assert!(params.get("state").is_some_and(|state| state.len() >= 16)); assert_eq!( params.get("code_challenge_method").map(String::as_str), Some("S256") @@ -2032,6 +2327,7 @@ mod tests { issuer_url, "client-id".to_string(), None, + ClientAuthMethod::None, vec!["openid".to_string()], None, None, @@ -2055,6 +2351,7 @@ mod tests { issuer_url, "client-id".to_string(), Some("secret".to_string()), + ClientAuthMethod::ClientSecretBasic, vec!["openid".to_string()], None, None, @@ -2071,12 +2368,13 @@ mod tests { } #[tokio::test] - async fn test_authorization_code_exchange_includes_optional_credentials() { - let (issuer_url, request_body, server) = spawn_token_exchange_server().await; + async fn test_authorization_code_exchange_public_client_sends_client_id_only() { + let (issuer_url, request, server) = spawn_captured_token_server().await; let source = AuthorizationCodeSource::new( issuer_url, "client-id".to_string(), - Some("secret".to_string()), + None, + ClientAuthMethod::None, vec!["openid".to_string()], None, None, @@ -2085,15 +2383,147 @@ mod tests { .unwrap(); let response = source - .exchange_code("auth-code", Some("verifier")) + .exchange_code( + "auth-code", + Some(PkceCodeVerifier::new("verifier".to_string())), + ) .await .unwrap(); - assert_eq!(response.access_token, "token"); - let body = request_body.lock().unwrap().clone().unwrap(); - assert!(body.contains("grant_type=authorization_code")); - assert!(body.contains("code=auth-code")); - assert!(body.contains("code_verifier=verifier")); - assert!(body.contains("client_secret=secret")); + assert_eq!(response.access_token.secret(), "token"); + + let request = request.lock().unwrap().take().unwrap(); + assert_eq!(request.header("authorization"), None); + assert!(request.body.contains("grant_type=authorization_code")); + assert!(request.body.contains("code=auth-code")); + assert!(request.body.contains("code_verifier=verifier")); + assert!(request.body.contains("client_id=client-id")); + assert!(!request.body.contains("client_secret")); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_authorization_code_exchange_uses_basic_auth_by_default() { + let (issuer_url, request, server) = spawn_captured_token_server().await; + let source = AuthorizationCodeSource::new( + issuer_url, + "client-id".to_string(), + Some("secret".to_string()), + ClientAuthMethod::ClientSecretBasic, + vec!["openid".to_string()], + None, + None, + AuthorizationCodeOptions::new(), + ) + .unwrap(); + + source + .exchange_code( + "auth-code", + Some(PkceCodeVerifier::new("verifier".to_string())), + ) + .await + .unwrap(); + + let request = request.lock().unwrap().take().unwrap(); + assert_eq!( + request.header("authorization").as_deref(), + Some(basic_authorization("client-id", "secret").as_str()) + ); + assert!(request.body.contains("grant_type=authorization_code")); + assert!(request.body.contains("code=auth-code")); + assert!(request.body.contains("code_verifier=verifier")); + assert!(!request.body.contains("client_secret")); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_authorization_code_exchange_supports_client_secret_post() { + let (issuer_url, request, server) = spawn_captured_token_server().await; + let source = AuthorizationCodeSource::new( + issuer_url, + "client-id".to_string(), + Some("secret".to_string()), + ClientAuthMethod::ClientSecretPost, + vec!["openid".to_string()], + None, + None, + AuthorizationCodeOptions::new(), + ) + .unwrap(); + + source + .exchange_code( + "auth-code", + Some(PkceCodeVerifier::new("verifier".to_string())), + ) + .await + .unwrap(); + + let request = request.lock().unwrap().take().unwrap(); + assert_eq!(request.header("authorization"), None); + assert!(request.body.contains("client_id=client-id")); + assert!(request.body.contains("client_secret=secret")); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_refresh_uses_basic_auth_by_default() { + let (issuer_url, request, server) = spawn_captured_token_server().await; + let source = AuthorizationCodeSource::new( + issuer_url, + "client-id".to_string(), + Some("secret".to_string()), + ClientAuthMethod::ClientSecretBasic, + vec!["openid".to_string()], + None, + None, + AuthorizationCodeOptions::new(), + ) + .unwrap(); + + assert!(matches!( + source.oidc.refresh_token("refresh-token").await.unwrap(), + RefreshResult::Refreshed(_) + )); + + let request = request.lock().unwrap().take().unwrap(); + assert_eq!( + request.header("authorization").as_deref(), + Some(basic_authorization("client-id", "secret").as_str()) + ); + assert!(request.body.contains("grant_type=refresh_token")); + assert!(request.body.contains("refresh_token=refresh-token")); + assert!(!request.body.contains("client_secret")); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_refresh_supports_client_secret_post_and_rotation() { + let (issuer_url, request, server) = spawn_captured_token_server().await; + let source = AuthorizationCodeSource::new( + issuer_url, + "client-id".to_string(), + Some("secret".to_string()), + ClientAuthMethod::ClientSecretPost, + vec!["openid".to_string()], + None, + None, + AuthorizationCodeOptions::new(), + ) + .unwrap(); + + let response = match source.oidc.refresh_token("old-refresh").await.unwrap() { + RefreshResult::Refreshed(response) => response, + other => panic!("expected refresh, got {other:?}"), + }; + assert_eq!(response.refresh_token().unwrap().secret(), "refresh"); + + let request = request.lock().unwrap().take().unwrap(); + assert_eq!(request.header("authorization"), None); + assert!(request.body.contains("grant_type=refresh_token")); + assert!(request.body.contains("refresh_token=old-refresh")); + assert!(request.body.contains("client_id=client-id")); + assert!(request.body.contains("client_secret=secret")); server.await.unwrap(); } @@ -2105,6 +2535,7 @@ mod tests { issuer_url, "client-id".to_string(), None, + ClientAuthMethod::None, vec!["openid".to_string()], None, None, @@ -2130,6 +2561,7 @@ mod tests { issuer_url, "client-id".to_string(), None, + ClientAuthMethod::None, vec!["openid".to_string()], None, None, @@ -2141,7 +2573,56 @@ mod tests { assert!(matches!( err, Error::Runtime { message } - if message.contains("503 Service Unavailable") + if message.contains("503 Service Unavailable") && message.contains("transient") + )); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_token_request_rejects_insecure_redirect() { + let (issuer_url, server) = spawn_redirecting_token_server().await; + let source = ClientCredentialsSource::new( + issuer_url, + "client-id".to_string(), + Some("secret".to_string()), + ClientAuthMethod::ClientSecretBasic, + vec!["scope".to_string()], + None, + None, + ) + .unwrap(); + + let err = TokenSource::fetch_token(&source).await.unwrap_err(); + let Error::Runtime { message } = &err else { + panic!("expected runtime error, got {err:?}"); + }; + assert!(message.contains("redirect")); + // The insecure redirect target must never be contacted. + assert!(!message.contains("idp.example.com")); + server.await.unwrap(); + } + + #[tokio::test] + async fn test_malformed_token_response_error_does_not_leak_body() { + let (issuer_url, server) = spawn_malformed_token_server().await; + let source = ClientCredentialsSource::new( + issuer_url, + "client-id".to_string(), + Some("secret".to_string()), + ClientAuthMethod::ClientSecretBasic, + vec!["scope".to_string()], + None, + None, + ) + .unwrap(); + + let err = TokenSource::fetch_token(&source).await.unwrap_err(); + let message = format!("{err:?}"); + assert!(!message.contains("leak-marker")); + assert!(matches!( + err, + Error::Runtime { message } + if message.contains("could not be parsed") || message.contains("Content-Type") )); server.await.unwrap(); } @@ -2153,6 +2634,7 @@ mod tests { issuer_url, "client-id".to_string(), Some("secret".to_string()), + ClientAuthMethod::ClientSecretBasic, vec!["openid".to_string()], None, None, @@ -2162,8 +2644,13 @@ mod tests { let device = source.request_device_authorization().await.unwrap(); let response = source.poll_for_token(&device).await.unwrap(); - assert_eq!(response.access_token, "device-token"); - assert_eq!(response.refresh_token.as_deref(), Some("device-refresh")); + assert_eq!(response.access_token.secret(), "device-token"); + assert_eq!( + response + .refresh_token() + .map(|token| token.secret().as_str()), + Some("device-refresh") + ); assert_eq!(token_requests.load(Ordering::SeqCst), 3); server.await.unwrap(); } @@ -2175,6 +2662,7 @@ mod tests { issuer_url, "client-id".to_string(), None, + ClientAuthMethod::None, vec!["openid".to_string()], None, None, @@ -2199,16 +2687,17 @@ mod tests { issuer_url, "client-id".to_string(), None, + ClientAuthMethod::None, vec!["openid".to_string()], None, None, ) .unwrap(); - let device = test_device_authorization_response(10, 1); + let device = test_device_authorization_response(60, 1); let response = source.poll_for_token(&device).await.unwrap(); - assert_eq!(response.access_token, "device-token"); + assert_eq!(response.access_token.secret(), "device-token"); assert_eq!(token_requests.load(Ordering::SeqCst), 4); server.await.unwrap(); } @@ -2220,6 +2709,7 @@ mod tests { issuer_url, "client-id".to_string(), None, + ClientAuthMethod::None, vec!["openid".to_string()], None, None, @@ -2243,6 +2733,7 @@ mod tests { issuer_url, "client-id".to_string(), None, + ClientAuthMethod::None, vec!["openid".to_string()], None, None, @@ -2266,12 +2757,13 @@ mod tests { issuer_url, "client-id".to_string(), None, + ClientAuthMethod::None, vec!["openid".to_string()], None, None, ) .unwrap(); - let device = test_device_authorization_response(1, 5); + let device = test_device_authorization_response(1, 1); let err = source.poll_for_token(&device).await.unwrap_err(); assert!(matches!( @@ -2292,23 +2784,17 @@ mod tests { impl TokenSource for RefreshingTokenSource { async fn fetch_token(&self) -> Result { self.fetches.fetch_add(1, Ordering::SeqCst); - Ok(TokenResponse { - access_token: "initial".to_string(), - refresh_token: Some("refresh".to_string()), - expires_in: Some(3600), - token_type: Some("Bearer".to_string()), - }) + Ok(token_response("initial", Some("refresh"), Some(3600))) } async fn refresh_token(&self, refresh_token: &str) -> Result { assert_eq!(refresh_token, "refresh"); self.refreshes.fetch_add(1, Ordering::SeqCst); - Ok(RefreshResult::Refreshed(TokenResponse { - access_token: "refreshed".to_string(), - refresh_token: None, - expires_in: Some(3600), - token_type: Some("Bearer".to_string()), - })) + Ok(RefreshResult::Refreshed(token_response( + "refreshed", + None, + Some(3600), + ))) } } @@ -2348,12 +2834,11 @@ mod tests { impl TokenSource for FailedRefreshTokenSource { async fn fetch_token(&self) -> Result { self.fetches.fetch_add(1, Ordering::SeqCst); - Ok(TokenResponse { - access_token: "reauthenticated".to_string(), - refresh_token: Some("new-refresh".to_string()), - expires_in: Some(3600), - token_type: Some("Bearer".to_string()), - }) + Ok(token_response( + "reauthenticated", + Some("new-refresh"), + Some(3600), + )) } async fn refresh_token(&self, refresh_token: &str) -> Result { @@ -2439,38 +2924,41 @@ mod tests { ); } - #[test] - fn test_oauth_config_debug_redacts_client_secret() { - let config = OAuthConfig { + fn test_config(flow: OAuthFlow, client_secret: Option) -> OAuthConfig { + OAuthConfig { issuer_url: "https://issuer.example.com".to_string(), client_id: "client-id".to_string(), - client_secret: Some("super-secret".to_string()), + client_secret, scopes: vec!["scope".to_string()], - flow: OAuthFlow::ClientCredentials, - refresh_buffer_secs: None, resource: None, audience: None, + flow, + client_auth_method: None, + refresh_buffer_secs: None, token_cache: None, - }; + } + } + + #[test] + fn test_oauth_config_debug_redacts_client_secret() { + let mut config = test_config( + OAuthFlow::ClientCredentials, + Some("super-secret".to_string()), + ); + config.client_auth_method = Some(ClientAuthMethod::ClientSecretBasic); let debug = format!("{config:?}"); assert!(!debug.contains("super-secret")); assert!(debug.contains("client_secret: Some(\"\")")); + assert!(debug.contains("client_auth_method")); } #[test] fn test_oauth_header_provider_debug_redacts_client_secret() { - let config = OAuthConfig { - issuer_url: "https://issuer.example.com".to_string(), - client_id: "client-id".to_string(), - client_secret: Some("super-secret".to_string()), - scopes: vec!["scope".to_string()], - flow: OAuthFlow::ClientCredentials, - refresh_buffer_secs: None, - resource: None, - audience: None, - token_cache: None, - }; + let config = test_config( + OAuthFlow::ClientCredentials, + Some("super-secret".to_string()), + ); let provider = OAuthHeaderProvider::new(config).unwrap(); let debug = format!("{provider:?}"); @@ -2504,10 +2992,11 @@ mod tests { "api://test-a/.default".to_string(), "api://test-b/.default".to_string(), ], - flow: OAuthFlow::AzureManagedIdentity { client_id: None }, - refresh_buffer_secs: None, resource: None, audience: None, + flow: OAuthFlow::AzureManagedIdentity { client_id: None }, + client_auth_method: None, + refresh_buffer_secs: None, token_cache: None, }; assert!(OAuthHeaderProvider::new(config).is_err()); @@ -2520,6 +3009,7 @@ mod tests { issuer_url, "client-id".to_string(), Some("secret".to_string()), + ClientAuthMethod::ClientSecretBasic, vec!["scope".to_string()], None, None, @@ -2537,33 +3027,14 @@ mod tests { #[test] fn test_client_credentials_requires_secret() { - let config = OAuthConfig { - issuer_url: "https://login.microsoftonline.com/tenant/v2.0".to_string(), - client_id: "app-id".to_string(), - client_secret: None, - scopes: vec!["scope".to_string()], - flow: OAuthFlow::ClientCredentials, - refresh_buffer_secs: None, - resource: None, - audience: None, - token_cache: None, - }; + let config = test_config(OAuthFlow::ClientCredentials, None); assert!(OAuthHeaderProvider::new(config).is_err()); } #[test] fn test_client_credentials_rejects_insecure_non_loopback_issuer() { - let config = OAuthConfig { - issuer_url: "http://issuer.example.com".to_string(), - client_id: "app-id".to_string(), - client_secret: Some("secret".to_string()), - scopes: vec!["scope".to_string()], - flow: OAuthFlow::ClientCredentials, - refresh_buffer_secs: None, - resource: None, - audience: None, - token_cache: None, - }; + let mut config = test_config(OAuthFlow::ClientCredentials, Some("secret".to_string())); + config.issuer_url = "http://issuer.example.com".to_string(); let err = OAuthHeaderProvider::new(config).unwrap_err(); assert!(matches!( @@ -2576,17 +3047,8 @@ mod tests { #[test] fn test_empty_scopes_rejected() { - let config = OAuthConfig { - issuer_url: "https://login.microsoftonline.com/tenant/v2.0".to_string(), - client_id: "app-id".to_string(), - client_secret: None, - scopes: vec![], - flow: OAuthFlow::AzureManagedIdentity { client_id: None }, - refresh_buffer_secs: None, - resource: None, - audience: None, - token_cache: None, - }; + let mut config = test_config(OAuthFlow::AzureManagedIdentity { client_id: None }, None); + config.scopes = vec![]; assert!(OAuthHeaderProvider::new(config).is_err()); } @@ -2598,10 +3060,11 @@ mod tests { client_id: "client-id".to_string(), client_secret: Some("secret".to_string()), scopes: vec!["scope".to_string()], - flow: OAuthFlow::ClientCredentials, - refresh_buffer_secs: Some(0), resource: None, audience: None, + flow: OAuthFlow::ClientCredentials, + client_auth_method: None, + refresh_buffer_secs: Some(0), token_cache: None, }; let provider = OAuthHeaderProvider::new(config).unwrap(); @@ -2624,6 +3087,34 @@ mod tests { server.await.unwrap(); } + #[tokio::test] + async fn test_client_credentials_supports_client_secret_post() { + let (issuer_url, request, server) = spawn_captured_token_server().await; + let config = OAuthConfig { + issuer_url, + client_id: "client-id".to_string(), + client_secret: Some("secret".to_string()), + scopes: vec!["scope".to_string()], + resource: None, + audience: None, + flow: OAuthFlow::ClientCredentials, + client_auth_method: Some(ClientAuthMethod::ClientSecretPost), + refresh_buffer_secs: None, + token_cache: None, + }; + let provider = OAuthHeaderProvider::new(config).unwrap(); + + provider.get_headers().await.unwrap(); + + let request = request.lock().unwrap().take().unwrap(); + assert_eq!(request.header("authorization"), None); + assert!(request.body.contains("grant_type=client_credentials")); + assert!(request.body.contains("client_id=client-id")); + assert!(request.body.contains("client_secret=secret")); + assert!(request.body.contains("scope=scope")); + server.await.unwrap(); + } + async fn spawn_discovery_server(expected_requests: usize) -> (String, JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); @@ -2632,8 +3123,12 @@ mod tests { let server = tokio::spawn(async move { for _ in 0..expected_requests { let (mut stream, _) = listener.accept().await.unwrap(); - let (request_line, _) = read_http_request(&mut stream).await; - assert!(request_line.starts_with("GET /.well-known/openid-configuration ")); + let request = read_http_request(&mut stream).await; + assert!( + request + .line + .starts_with("GET /.well-known/openid-configuration ") + ); let discovery = format!( r#"{{"token_endpoint":"http://{addr}/token","authorization_endpoint":"http://{addr}/authorize","device_authorization_endpoint":"http://{addr}/device"}}"# ); @@ -2650,8 +3145,12 @@ mod tests { let issuer_url = format!("http://{addr}"); let server = tokio::spawn(async move { let (mut stream, _) = listener.accept().await.unwrap(); - let (request_line, _) = read_http_request(&mut stream).await; - assert!(request_line.starts_with("GET /.well-known/openid-configuration ")); + let request = read_http_request(&mut stream).await; + assert!( + request + .line + .starts_with("GET /.well-known/openid-configuration ") + ); write_json_response( &mut stream, "200 OK", @@ -2670,14 +3169,17 @@ mod tests { let server = tokio::spawn(async move { for _ in 0..2 { let (mut stream, _) = listener.accept().await.unwrap(); - let (request_line, _) = read_http_request(&mut stream).await; - if request_line.starts_with("GET /.well-known/openid-configuration ") { + let request = read_http_request(&mut stream).await; + if request + .line + .starts_with("GET /.well-known/openid-configuration ") + { let discovery = format!( r#"{{"token_endpoint":"http://{addr}/token","device_authorization_endpoint":"http://{addr}/device"}}"# ); write_json_response(&mut stream, "200 OK", &discovery).await; } else { - assert!(request_line.starts_with("POST /device ")); + assert!(request.line.starts_with("POST /device ")); write_json_response( &mut stream, "200 OK", @@ -2691,32 +3193,33 @@ mod tests { (issuer_url, server) } - async fn spawn_token_exchange_server() -> ( + async fn spawn_captured_token_server() -> ( String, - Arc>>, + Arc>>, JoinHandle<()>, ) { let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); let addr = listener.local_addr().unwrap(); let issuer_url = format!("http://{addr}"); - let request_body = Arc::new(std::sync::Mutex::new(None)); - let server_request_body = Arc::clone(&request_body); + let request = Arc::new(std::sync::Mutex::new(None)); + let server_request = Arc::clone(&request); let server = tokio::spawn(async move { for _ in 0..2 { let (mut stream, _) = listener.accept().await.unwrap(); - let (request_line, body) = read_http_request(&mut stream).await; - if request_line.starts_with("GET /.well-known/openid-configuration ") { - let discovery = format!( - r#"{{"token_endpoint":"http://{addr}/token","authorization_endpoint":"http://{addr}/authorize"}}"# - ); + let captured = read_http_request(&mut stream).await; + if captured + .line + .starts_with("GET /.well-known/openid-configuration ") + { + let discovery = format!(r#"{{"token_endpoint":"http://{addr}/token"}}"#); write_json_response(&mut stream, "200 OK", &discovery).await; - } else if request_line.starts_with("POST /token ") { - *server_request_body.lock().unwrap() = Some(body); + } else if captured.line.starts_with("POST /token ") { + *server_request.lock().unwrap() = Some(captured); write_json_response( &mut stream, "200 OK", - r#"{"access_token":"token","refresh_token":"refresh","expires_in":3600}"#, + r#"{"access_token":"token","refresh_token":"refresh","expires_in":3600,"token_type":"Bearer"}"#, ) .await; } else { @@ -2725,7 +3228,63 @@ mod tests { } }); - (issuer_url, request_body, server) + (issuer_url, request, server) + } + + async fn spawn_redirecting_token_server() -> (String, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let issuer_url = format!("http://{addr}"); + + let server = tokio::spawn(async move { + for _ in 0..2 { + let (mut stream, _) = listener.accept().await.unwrap(); + let request = read_http_request(&mut stream).await; + if request + .line + .starts_with("GET /.well-known/openid-configuration ") + { + let discovery = format!(r#"{{"token_endpoint":"http://{addr}/token"}}"#); + write_json_response(&mut stream, "200 OK", &discovery).await; + } else { + assert!(request.line.starts_with("POST /token ")); + let response = "HTTP/1.1 302 Found\r\nlocation: http://idp.example.com/steal\r\ncontent-length: 0\r\nconnection: close\r\n\r\n".to_string(); + stream.write_all(response.as_bytes()).await.unwrap(); + } + } + }); + + (issuer_url, server) + } + + async fn spawn_malformed_token_server() -> (String, JoinHandle<()>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let issuer_url = format!("http://{addr}"); + + let server = tokio::spawn(async move { + for _ in 0..2 { + let (mut stream, _) = listener.accept().await.unwrap(); + let request = read_http_request(&mut stream).await; + if request + .line + .starts_with("GET /.well-known/openid-configuration ") + { + let discovery = format!(r#"{{"token_endpoint":"http://{addr}/token"}}"#); + write_json_response(&mut stream, "200 OK", &discovery).await; + } else { + assert!(request.line.starts_with("POST /token ")); + write_json_response( + &mut stream, + "200 OK", + r#"{"access_token":{"nested":"leak-marker-12345"}}"#, + ) + .await; + } + } + }); + + (issuer_url, server) } async fn spawn_refresh_error_server( @@ -2739,13 +3298,16 @@ mod tests { let server = tokio::spawn(async move { for _ in 0..2 { let (mut stream, _) = listener.accept().await.unwrap(); - let (request_line, body) = read_http_request(&mut stream).await; - if request_line.starts_with("GET /.well-known/openid-configuration ") { + let request = read_http_request(&mut stream).await; + if request + .line + .starts_with("GET /.well-known/openid-configuration ") + { let discovery = format!(r#"{{"token_endpoint":"http://{addr}/token"}}"#); write_json_response(&mut stream, "200 OK", &discovery).await; - } else if request_line.starts_with("POST /token ") { - assert!(body.contains("grant_type=refresh_token")); - assert!(body.contains("refresh_token=")); + } else if request.line.starts_with("POST /token ") { + assert!(request.body.contains("grant_type=refresh_token")); + assert!(request.body.contains("refresh_token=")); write_json_response(&mut stream, status, response_body).await; } else { write_json_response(&mut stream, "404 Not Found", "{}").await; @@ -2766,26 +3328,37 @@ mod tests { let server = tokio::spawn(async move { for _ in 0..5 { let (mut stream, _) = listener.accept().await.unwrap(); - let (request_line, body) = read_http_request(&mut stream).await; - if request_line.starts_with("GET /.well-known/openid-configuration ") { + let request = read_http_request(&mut stream).await; + if request + .line + .starts_with("GET /.well-known/openid-configuration ") + { let discovery = format!( r#"{{"token_endpoint":"http://{addr}/token","device_authorization_endpoint":"http://{addr}/device"}}"# ); write_json_response(&mut stream, "200 OK", &discovery).await; - } else if request_line.starts_with("POST /device ") { - assert!(body.contains("client_id=client-id")); - assert!(body.contains("client_secret=secret")); - assert!(body.contains("scope=openid")); + } else if request.line.starts_with("POST /device ") { + // The resolved default for a confidential client is HTTP Basic. + assert_eq!( + request.header("authorization").as_deref(), + Some(basic_authorization("client-id", "secret").as_str()) + ); + assert!(request.body.contains("scope=openid")); + assert!(!request.body.contains("client_secret")); let device = format!( r#"{{"device_code":"device-code","user_code":"ABCD-EFGH","verification_uri":"http://{addr}/verify","verification_uri_complete":"http://{addr}/verify?user_code=ABCD-EFGH","expires_in":60,"interval":1}}"# ); write_json_response(&mut stream, "200 OK", &device).await; - } else if request_line.starts_with("POST /token ") { - assert!(body.contains( + } else if request.line.starts_with("POST /token ") { + assert_eq!( + request.header("authorization").as_deref(), + Some(basic_authorization("client-id", "secret").as_str()) + ); + assert!(request.body.contains( "grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Adevice_code" )); - assert!(body.contains("device_code=device-code")); - assert!(body.contains("client_secret=secret")); + assert!(request.body.contains("device_code=device-code")); + assert!(!request.body.contains("client_secret")); let request = server_token_requests.fetch_add(1, Ordering::SeqCst); match request { 0 => { @@ -2808,7 +3381,7 @@ mod tests { write_json_response( &mut stream, "200 OK", - r#"{"access_token":"device-token","refresh_token":"device-refresh","expires_in":3600}"#, + r#"{"access_token":"device-token","refresh_token":"device-refresh","expires_in":3600,"token_type":"Bearer"}"#, ) .await; } @@ -2832,14 +3405,17 @@ mod tests { let server = tokio::spawn(async move { for _ in 0..5 { let (mut stream, _) = listener.accept().await.unwrap(); - let (request_line, _) = read_http_request(&mut stream).await; - if request_line.starts_with("GET /.well-known/openid-configuration ") { + let request = read_http_request(&mut stream).await; + if request + .line + .starts_with("GET /.well-known/openid-configuration ") + { let discovery = format!(r#"{{"token_endpoint":"http://{addr}/token"}}"#); write_json_response(&mut stream, "200 OK", &discovery).await; continue; } - assert!(request_line.starts_with("POST /token ")); + assert!(request.line.starts_with("POST /token ")); match server_token_requests.fetch_add(1, Ordering::SeqCst) { 0 => drop(stream), 1 => { @@ -2862,7 +3438,7 @@ mod tests { write_json_response( &mut stream, "200 OK", - r#"{"access_token":"device-token","expires_in":3600}"#, + r#"{"access_token":"device-token","expires_in":3600,"token_type":"Bearer"}"#, ) .await; } @@ -2876,15 +3452,11 @@ mod tests { fn test_device_authorization_response( expires_in: u64, interval: u64, - ) -> DeviceAuthorizationResponse { - DeviceAuthorizationResponse { - device_code: "device-code".to_string(), - user_code: "ABCD-EFGH".to_string(), - verification_uri: "http://127.0.0.1/verify".to_string(), - verification_uri_complete: None, - expires_in, - interval: Some(interval), - } + ) -> StandardDeviceAuthorizationResponse { + serde_json::from_str(&format!( + r#"{{"device_code":"device-code","user_code":"ABCD-EFGH","verification_uri":"http://127.0.0.1/verify","expires_in":{expires_in},"interval":{interval}}}"# + )) + .unwrap() } async fn spawn_device_error_server(error: &'static str) -> (String, JoinHandle<()>) { @@ -2895,11 +3467,14 @@ mod tests { let server = tokio::spawn(async move { for _ in 0..2 { let (mut stream, _) = listener.accept().await.unwrap(); - let (request_line, _) = read_http_request(&mut stream).await; - if request_line.starts_with("GET /.well-known/openid-configuration ") { + let request = read_http_request(&mut stream).await; + if request + .line + .starts_with("GET /.well-known/openid-configuration ") + { let discovery = format!(r#"{{"token_endpoint":"http://{addr}/token"}}"#); write_json_response(&mut stream, "200 OK", &discovery).await; - } else if request_line.starts_with("POST /token ") { + } else if request.line.starts_with("POST /token ") { write_json_response( &mut stream, "400 Bad Request", @@ -2925,16 +3500,23 @@ mod tests { let server = tokio::spawn(async move { for _ in 0..3 { let (mut stream, _) = listener.accept().await.unwrap(); - let (request_line, body) = read_http_request(&mut stream).await; + let request = read_http_request(&mut stream).await; - if request_line.starts_with("GET /.well-known/openid-configuration ") { + if request + .line + .starts_with("GET /.well-known/openid-configuration ") + { let discovery = format!(r#"{{"token_endpoint":"http://{addr}/token"}}"#); write_json_response(&mut stream, "200 OK", &discovery).await; - } else if request_line.starts_with("POST /token ") { - assert!(body.contains("grant_type=client_credentials")); - assert!(body.contains("client_id=client-id")); - assert!(body.contains("client_secret=secret")); - assert!(body.contains("scope=scope")); + } else if request.line.starts_with("POST /token ") { + assert_eq!( + request.header("authorization").as_deref(), + Some(basic_authorization("client-id", "secret").as_str()) + ); + assert!(request.body.contains("grant_type=client_credentials")); + assert!(request.body.contains("scope=scope")); + assert!(!request.body.contains("client_secret")); + assert!(!request.body.contains("client_id")); let token_num = server_token_requests.fetch_add(1, Ordering::SeqCst) + 1; let token = format!( @@ -2957,15 +3539,19 @@ mod tests { let server = tokio::spawn(async move { let (mut stream, _) = listener.accept().await.unwrap(); - let (request_line, _) = read_http_request(&mut stream).await; - assert!(request_line.starts_with("GET /.well-known/openid-configuration ")); + let request = read_http_request(&mut stream).await; + assert!( + request + .line + .starts_with("GET /.well-known/openid-configuration ") + ); write_json_response(&mut stream, "503 Service Unavailable", "{}").await; }); (issuer_url, server) } - async fn read_http_request(stream: &mut TcpStream) -> (String, String) { + async fn read_http_request(stream: &mut TcpStream) -> CapturedRequest { let mut buffer = Vec::new(); let mut header_end = None; @@ -2979,7 +3565,7 @@ mod tests { let header_end = header_end.unwrap(); let headers = String::from_utf8_lossy(&buffer[..header_end]).to_string(); - let request_line = headers.lines().next().unwrap_or_default().to_string(); + let line = headers.lines().next().unwrap_or_default().to_string(); let content_length = headers .lines() .find_map(|line| { @@ -3000,7 +3586,11 @@ mod tests { let body = String::from_utf8_lossy(&buffer[header_end..header_end + content_length]).to_string(); - (request_line, body) + CapturedRequest { + line, + headers, + body, + } } fn find_subsequence(haystack: &[u8], needle: &[u8]) -> Option { diff --git a/rust/lancedb/src/remote/token_cache.rs b/rust/lancedb/src/remote/token_cache.rs index edda93bcf..6f34ab73c 100644 --- a/rust/lancedb/src/remote/token_cache.rs +++ b/rust/lancedb/src/remote/token_cache.rs @@ -44,6 +44,7 @@ //! client_secret: None, //! scopes: vec!["openid".to_string()], //! flow: OAuthFlow::DeviceCode, +//! client_auth_method: None, //! refresh_buffer_secs: None, //! resource: Some("https://api.example.com".to_string()), //! audience: None, @@ -420,7 +421,10 @@ impl TokenCache { /// Build a record from a token response, or `None` when the response /// carries no refresh token (nothing may be persisted). fn record_from_response(&self, response: &TokenResponse) -> Option { - let refresh_token = response.refresh_token.clone()?; + let refresh_token = response + .refresh_token + .as_ref() + .map(|token| token.secret().clone())?; Some(CachedTokenRecord { version: CACHE_RECORD_VERSION, issuer_url: self.key.issuer_url.clone(), @@ -842,6 +846,7 @@ pub struct SessionLogout { /// client_secret: None, /// scopes: vec!["openid".to_string()], /// flow: OAuthFlow::DeviceCode, +/// client_auth_method: None, /// refresh_buffer_secs: None, /// resource: None, /// audience: None, @@ -983,6 +988,8 @@ mod tests { use crate::remote::HeaderProvider; use crate::remote::oauth::OAuthHeaderProvider; + use oauth2::basic::BasicTokenType; + use oauth2::{AccessToken, RefreshToken}; use serial_test::serial; /// Temp directory that satisfies the cache hardening checks. CI runners @@ -1003,6 +1010,7 @@ mod tests { issuer_url: "https://issuer.example.com".to_string(), client_id: "client-id".to_string(), client_secret: None, + client_auth_method: None, scopes: vec!["openid".to_string()], flow: OAuthFlow::DeviceCode, refresh_buffer_secs: None, @@ -1479,8 +1487,8 @@ mod tests { assert!(keys.insert(cache.key.file_stem.clone())); let record = cache .record_from_response(&TokenResponse { - access_token: "unused".into(), - refresh_token: Some("seed-refresh".into()), + access_token: AccessToken::new("unused".into()), + refresh_token: Some(RefreshToken::new("seed-refresh".into())), expires_in: Some(3600), token_type: None, }) @@ -1634,10 +1642,10 @@ mod tests { ) .unwrap(); let response = TokenResponse { - access_token: "access-token".to_string(), - refresh_token: Some("refresh-token".to_string()), + access_token: AccessToken::new("access-token".to_string()), + refresh_token: Some(RefreshToken::new("refresh-token".to_string())), expires_in: Some(3600), - token_type: Some("Bearer".to_string()), + token_type: Some(BasicTokenType::Bearer), }; let record = cache.record_from_response(&response).unwrap(); let debug = format!("{record:?}"); @@ -1678,8 +1686,8 @@ mod tests { .unwrap(); let record = cache .record_from_response(&TokenResponse { - access_token: "a".to_string(), - refresh_token: Some("r".to_string()), + access_token: AccessToken::new("a".to_string()), + refresh_token: Some(RefreshToken::new("r".to_string())), expires_in: None, token_type: None, }) @@ -1740,8 +1748,8 @@ mod tests { // A complete, well-formed record with an unknown schema version. let record = cache .record_from_response(&TokenResponse { - access_token: "a".to_string(), - refresh_token: Some("r".to_string()), + access_token: AccessToken::new("a".to_string()), + refresh_token: Some(RefreshToken::new("r".to_string())), expires_in: None, token_type: None, }) @@ -1860,7 +1868,7 @@ mod tests { ) .unwrap(); let response = TokenResponse { - access_token: "access-token".to_string(), + access_token: AccessToken::new("access-token".to_string()), refresh_token: None, expires_in: Some(3600), token_type: None, From 99ed25f7531c5f4434c620c0be912e4719756d81 Mon Sep 17 00:00:00 2001 From: Joaquin Hui <132194176+joaquinhuigomez@users.noreply.github.com> Date: Wed, 16 Sep 2026 16:31:06 +0100 Subject: [PATCH 28/43] fix: return InvalidTableName instead of panicking in open_table/create_table (#4192) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Passing an invalid table name to `open_table` or `create_table` panics instead of returning an error: thread '...' panicked at rust/lancedb/src/database/listing.rs:1155:62: called `Result::unwrap()` on an `Err` value: InvalidTableName { name: "my table", ... } Both call sites build the table URI with `request.location.clone().unwrap_or_else(|| self.table_uri(&request.name).unwrap())`, and `table_uri` is the function that validates the name — so every rejected name (empty, spaces, slashes, non-ASCII) hits the inner `unwrap`. `Error::InvalidTableName` clearly is the intended contract here: the variant exists for exactly this, and the Python binding maps it to `ValueError`. Replaced the closure with a `match` that propagates the validation error; behavior with an explicit `location` is unchanged (the name is not validated on that path, as before). Added tests asserting `InvalidTableName` for `create_table` and `open_table` over a set of rejected names — both panic without the fix. Full `cargo test -p lancedb --lib --features remote`: 1214 passed; clippy/fmt clean; `cargo check --workspace --all-targets` clean. Co-authored-by: Xuanwo --- rust/lancedb/src/database/listing.rs | 74 +++++++++++++++++++++++++--- 1 file changed, 66 insertions(+), 8 deletions(-) diff --git a/rust/lancedb/src/database/listing.rs b/rust/lancedb/src/database/listing.rs index 59b075e4f..dc1627116 100644 --- a/rust/lancedb/src/database/listing.rs +++ b/rust/lancedb/src/database/listing.rs @@ -1033,10 +1033,10 @@ impl Database for ListingDatabase { return self.namespace_database().create_table(request).await; } // Use provided location if available, otherwise derive from table name - let table_uri = request - .location - .clone() - .unwrap_or_else(|| self.table_uri(&request.name).unwrap()); + let table_uri = match request.location.clone() { + Some(location) => location, + None => self.table_uri(&request.name)?, + }; let mut write_params = request .write_options @@ -1149,10 +1149,10 @@ impl Database for ListingDatabase { return self.namespace_database().open_table(request).await; } // Use provided location if available, otherwise derive from table name - let table_uri = request - .location - .clone() - .unwrap_or_else(|| self.table_uri(&request.name).unwrap()); + let table_uri = match request.location.clone() { + Some(location) => location, + None => self.table_uri(&request.name)?, + }; // Only modify the storage options if we actually have something to // inherit. There is a difference between storage_options=None and @@ -1696,6 +1696,64 @@ mod tests { ); } + /// The names a table cannot have. A name is what the database builds the table's + /// location out of, so one it cannot build a location from is refused rather than + /// turned into some other path. + const INVALID_TABLE_NAMES: [&str; 4] = ["", "has space", "bad/name", "a!b"]; + + /// Creating a table under an invalid name is an error the caller can handle, not a + /// panic: the name comes from the caller, and the bindings turn the error into their + /// own (`ValueError` in Python). + #[tokio::test] + async fn test_create_table_rejects_invalid_names() { + let (_tempdir, db) = setup_database().await; + let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); + + for name in INVALID_TABLE_NAMES { + let result = db + .create_table(CreateTableRequest { + name: name.to_string(), + namespace_path: vec![], + data: Box::new(RecordBatch::new_empty(schema.clone())) as Box, + mode: CreateTableMode::Create, + write_options: Default::default(), + location: None, + namespace_client: None, + }) + .await; + + assert!( + matches!(result, Err(Error::InvalidTableName { .. })), + "creating {name:?} did not report an invalid table name" + ); + } + } + + /// Opening a table under an invalid name is likewise an error rather than a panic. + #[tokio::test] + async fn test_open_table_rejects_invalid_names() { + let (_tempdir, db) = setup_database().await; + + for name in INVALID_TABLE_NAMES { + let result = db + .open_table(OpenTableRequest { + name: name.to_string(), + namespace_path: vec![], + index_cache_size: None, + lance_read_params: None, + location: None, + namespace_client: None, + managed_versioning: None, + }) + .await; + + assert!( + matches!(result, Err(Error::InvalidTableName { .. })), + "opening {name:?} did not report an invalid table name" + ); + } + } + async fn setup_database() -> (tempfile::TempDir, ListingDatabase) { let tempdir = tempdir().unwrap(); let uri = tempdir.path().to_str().unwrap(); From 6a07f88980ca8c9d7c1bc6111ab06a7ffd4fd176 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 16 Sep 2026 10:51:10 -0700 Subject: [PATCH 29/43] feat: add remote catalogs and Python and TypeScript bindings (#4195) Add a `Catalog` trait and `RemoteCatalog` for managing databases, exposed through Rust, synchronous/asynchronous Python, and TypeScript. A remote catalog represents the server's root namespace, and each database is one child namespace. Create/connect return ordinary LanceDB connections, so existing table APIs work unchanged. ## Rust API `Catalog` is an object-safe async trait with `create_database`, `connect_database`, `list_databases`, and `drop_database`. Backend create/connect methods return `Arc`; the public `CatalogConnection` wraps them as `Connection` values and shares its embedding registry with those connections. `RemoteCatalog` implements the trait; `connect_catalog` is the convenience builder, available with the `remote` feature. ```rust use lancedb::catalog::{ CreateDatabaseRequest, DropDatabaseRequest, ListDatabasesRequest, }; let catalog = lancedb::connect_catalog("https://my-server.example") .api_key("my-api-key") .execute() .await?; let db = catalog.create_database( CreateDatabaseRequest::new("analytics").exist_ok(true), ).await?; let connected = catalog.connect_database("analytics").await?; let page = catalog.list_databases( ListDatabasesRequest::default().limit(20), ).await?; // page.databases: Vec; page.page_token: Option catalog.drop_database( DropDatabaseRequest::new("analytics").ignore_missing(true), ).await?; ``` Create/drop also accept a plain name for default behavior, e.g. `catalog.create_database("analytics").await?`. Existing names fail creation unless `exist_ok` is enabled; missing names fail drop unless `ignore_missing` is enabled. Drop always requires an empty database. ## Python API ```python import lancedb catalog = lancedb.connect_catalog( "https://my-server.example", api_key="my-api-key" ) db = catalog.create_database("analytics", exist_ok=True) connected = catalog.connect_database("analytics") page = catalog.list_databases(limit=20) # page.databases: list[str]; page.page_token: Optional[str] if page.page_token is not None: next_page = catalog.list_databases(limit=20, page_token=page.page_token) catalog.drop_database("analytics", ignore_missing=True) ``` `connect_catalog` returns `Catalog`; create/connect return the existing `DBConnection` API. The async equivalent is `catalog = await lancedb.connect_catalog_async(...)`, returning `AsyncCatalog`; await each of the same four methods, with create/connect returning `AsyncConnection`. ## TypeScript API ```typescript import { connectCatalog } from "@lancedb/lancedb"; const catalog = await connectCatalog("https://my-server.example", { apiKey: "my-api-key", }); const db = await catalog.createDatabase("analytics", { existOk: true }); const connected = await catalog.connectDatabase("analytics"); const page = await catalog.listDatabases({ limit: 20 }); // page.databases: string[]; page.pageToken?: string if (page.pageToken !== undefined) { const nextPage = await catalog.listDatabases({ limit: 20, pageToken: page.pageToken, }); } await catalog.dropDatabase("analytics", { ignoreMissing: true }); ``` Create/connect return the existing `Connection` API. All four methods are asynchronous. ## REST mapping All paths below are relative to the catalog endpoint. `{name}` is the logical database name encoded as one URL path component. The default namespace delimiter is `$`, so the root identifier is encoded as `%24`. | Catalog operation | Existing REST route | Request | | --- | --- | --- | | `create_database(name)` | `POST /v1/namespace/{name}/create` | `{"mode":"Create"}`; `exist_ok=true` sends `{"mode":"ExistOk"}` | | `connect_database(name)` | `POST /v1/namespace/{name}/describe` | `{}`; verifies existence before returning a scoped connection | | `list_databases(...)` | `GET /v1/namespace/%24/list` | Optional `limit` and `page_token` query parameters | | `drop_database(name)` | `POST /v1/namespace/{name}/drop` | `{"mode":"Fail","behavior":"Restrict"}`; `ignore_missing=true` changes mode to `"Skip"` | For example, database `team/search` uses `/v1/namespace/team%2Fsearch/create`. A paginated root listing can use `/v1/namespace/%24/list?limit=20&page_token=a%2Fb`. The list response retains the existing namespace wire shape, `{"namespaces":["analytics"],"page_token":"next"}`; the SDK exposes `namespaces` as `databases` and preserves the opaque continuation token. An absent or empty token ends pagination. Page limits must be between 1 and 2147483647. Create/drop accept a namespace JSON response or HTTP 204. Catalog management requests omit both `x-lancedb-database` and `x-lancedb-database-prefix`, including values supplied through static or dynamic headers. Returned database connections set `x-lancedb-database` to the exact logical name and keep independent scope. API keys, OAuth or dynamic authentication, client settings, table read consistency settings, and an optional SQL endpoint override carry over to those connections. OAuth cannot be combined with an API key or a custom header provider. For SQL through an HTTPS catalog, configure the existing SQL endpoint contract with Rust `.sql_host_override("grpc+tls://sql.example.com:10026")` or Python `sql_host_override="grpc+tls://sql.example.com:10026"`. TypeScript catalog options expose the same setting as `sqlHostOverride`. It is inherited by created/connected databases, retained by Python connection serialization, and initialized lazily when SQL is executed. Create HTTP 409 maps to `DatabaseAlreadyExists`; connect/drop HTTP 404 maps to `DatabaseNotFound`, except that `ignore_missing` suppresses a missing-database drop. Other server errors propagate. The server enforces restricted deletion; the client never requests cascading deletion. Database names preserve literal slashes as part of one name. They must be nonempty ASCII, with no control characters, surrounding whitespace, or configured namespace delimiter, and cannot be `.` or `..`. Endpoints must be HTTP(S) URLs without embedded credentials, query parameters, or fragments. ## Scope This PR adds the client API and reuses existing namespace endpoints. Local catalogs, `__catalog` storage, location generation/sanitization, and `__manifest` lifecycle support remain deferred; the Lance dependency is unchanged. The PR also runs macOS Node tests serially to avoid existing resource-contention timeouts reproduced across recent main runs. --------- Co-authored-by: Xuanwo --- .github/workflows/nodejs.yml | 2 +- docs/src/js/classes/Catalog.md | 107 ++++ docs/src/js/functions/connectCatalog.md | 32 + docs/src/js/globals.md | 4 + docs/src/js/interfaces/CatalogOptions.md | 81 +++ .../js/interfaces/ListDatabasesResponse.md | 23 + docs/src/python/python.md | 15 + nodejs/__test__/catalog.test.ts | 142 +++++ nodejs/lancedb/catalog.ts | 103 +++ nodejs/lancedb/index.ts | 7 + nodejs/src/catalog.rs | 123 ++++ nodejs/src/lib.rs | 1 + python/python/lancedb/__init__.py | 13 + python/python/lancedb/_lancedb.pyi | 24 + python/python/lancedb/catalog.py | 196 ++++++ python/python/lancedb/remote/db.py | 41 ++ python/python/tests/test_catalog.py | 140 ++++ python/src/catalog.rs | 125 ++++ python/src/error.rs | 2 + python/src/lib.rs | 3 + rust/lancedb/src/catalog.rs | 259 ++++++++ rust/lancedb/src/lib.rs | 7 + rust/lancedb/src/remote.rs | 3 + rust/lancedb/src/remote/catalog.rs | 598 ++++++++++++++++++ rust/lancedb/src/remote/client.rs | 16 +- rust/lancedb/src/remote/db.rs | 78 ++- rust/lancedb/src/remote/sql_test.rs | 62 +- 27 files changed, 2200 insertions(+), 7 deletions(-) create mode 100644 docs/src/js/classes/Catalog.md create mode 100644 docs/src/js/functions/connectCatalog.md create mode 100644 docs/src/js/interfaces/CatalogOptions.md create mode 100644 docs/src/js/interfaces/ListDatabasesResponse.md create mode 100644 nodejs/__test__/catalog.test.ts create mode 100644 nodejs/lancedb/catalog.ts create mode 100644 nodejs/src/catalog.rs create mode 100644 python/python/lancedb/catalog.py create mode 100644 python/python/tests/test_catalog.py create mode 100644 python/src/catalog.rs create mode 100644 rust/lancedb/src/catalog.rs create mode 100644 rust/lancedb/src/remote/catalog.rs diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 55c050a7d..98acccf02 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -207,4 +207,4 @@ jobs: pnpm tsc - name: Test run: | - pnpm test + pnpm test --runInBand diff --git a/docs/src/js/classes/Catalog.md b/docs/src/js/classes/Catalog.md new file mode 100644 index 000000000..deac5e3ce --- /dev/null +++ b/docs/src/js/classes/Catalog.md @@ -0,0 +1,107 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / Catalog + +# Class: Catalog + +A remote catalog manages databases through the server's root namespace. + +## Accessors + +### uri + +```ts +get uri(): string +``` + +The root namespace endpoint. + +#### Returns + +`string` + +## Methods + +### connectDatabase() + +```ts +connectDatabase(name): Promise +``` + +Connect to an existing database by its logical name. + +#### Parameters + +* **name**: `string` + +#### Returns + +`Promise`<[`Connection`](Connection.md)> + +*** + +### createDatabase() + +```ts +createDatabase(name, options): Promise +``` + +Create a database, or open an existing database when existOk is true. + +#### Parameters + +* **name**: `string` + +* **options** = `{}` + +* **options.existOk?**: `boolean` + +#### Returns + +`Promise`<[`Connection`](Connection.md)> + +*** + +### dropDatabase() + +```ts +dropDatabase(name, options): Promise +``` + +Drop an empty database. The server rejects nonempty databases. + +#### Parameters + +* **name**: `string` + +* **options** = `{}` + +* **options.ignoreMissing?**: `boolean` + +#### Returns + +`Promise`<`void`> + +*** + +### listDatabases() + +```ts +listDatabases(options): Promise +``` + +List one page of databases; pass pageToken from a response for the next page. + +#### Parameters + +* **options** = `{}` + +* **options.limit?**: `number` + +* **options.pageToken?**: `string` + +#### Returns + +`Promise`<[`ListDatabasesResponse`](../interfaces/ListDatabasesResponse.md)> diff --git a/docs/src/js/functions/connectCatalog.md b/docs/src/js/functions/connectCatalog.md new file mode 100644 index 000000000..2bdb3182f --- /dev/null +++ b/docs/src/js/functions/connectCatalog.md @@ -0,0 +1,32 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / connectCatalog + +# Function: connectCatalog() + +```ts +function connectCatalog(endpoint, options): Promise +``` + +Connect to an HTTP(S) catalog endpoint. Catalog requests omit database-selection +headers; opened database connections inherit authentication and client options. + +## Parameters + +* **endpoint**: `string` + +* **options**: [`CatalogOptions`](../interfaces/CatalogOptions.md) = `{}` + +## Returns + +`Promise`<[`Catalog`](../classes/Catalog.md)> + +## Example + +```ts +const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" }); +const db = await catalog.createDatabase("analytics", { existOk: true }); +const page = await catalog.listDatabases({ limit: 20 }); +``` diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 4e09853f6..92e7940f0 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -25,6 +25,7 @@ - [BoostQuery](classes/BoostQuery.md) - [BranchContents](classes/BranchContents.md) - [Branches](classes/Branches.md) +- [Catalog](classes/Catalog.md) - [Connection](classes/Connection.md) - [HeaderProvider](classes/HeaderProvider.md) - [Index](classes/Index.md) @@ -64,6 +65,7 @@ - [BranchIndexSummary](interfaces/BranchIndexSummary.md) - [BranchRowCountSummary](interfaces/BranchRowCountSummary.md) - [BucketStats](interfaces/BucketStats.md) +- [CatalogOptions](interfaces/CatalogOptions.md) - [CherryPickError](interfaces/CherryPickError.md) - [CherryPickPreview](interfaces/CherryPickPreview.md) - [CherryPickResult](interfaces/CherryPickResult.md) @@ -102,6 +104,7 @@ - [JobEventsOptions](interfaces/JobEventsOptions.md) - [JobFailureInfo](interfaces/JobFailureInfo.md) - [JobInfo](interfaces/JobInfo.md) +- [ListDatabasesResponse](interfaces/ListDatabasesResponse.md) - [ListNamespacesOptions](interfaces/ListNamespacesOptions.md) - [ListNamespacesResponse](interfaces/ListNamespacesResponse.md) - [ListTablesOptions](interfaces/ListTablesOptions.md) @@ -167,6 +170,7 @@ - [RecordBatchIterator](functions/RecordBatchIterator.md) - [blob](functions/blob.md) - [connect](functions/connect.md) +- [connectCatalog](functions/connectCatalog.md) - [connectNamespace](functions/connectNamespace.md) - [instrumentLanceDbMetrics](functions/instrumentLanceDbMetrics.md) - [isBlobField](functions/isBlobField.md) diff --git a/docs/src/js/interfaces/CatalogOptions.md b/docs/src/js/interfaces/CatalogOptions.md new file mode 100644 index 000000000..c66823f6c --- /dev/null +++ b/docs/src/js/interfaces/CatalogOptions.md @@ -0,0 +1,81 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / CatalogOptions + +# Interface: CatalogOptions + +Options shared by a catalog and the database connections it returns. + +## Extends + +- `Omit`<`NativeCatalogOptions`, `"oauthConfig"`> + +## Properties + +### apiKey? + +```ts +optional apiKey: string; +``` + +#### Inherited from + +`Omit.apiKey` + +*** + +### clientConfig? + +```ts +optional clientConfig: ClientConfig; +``` + +#### Inherited from + +`Omit.clientConfig` + +*** + +### headerProvider? + +```ts +optional headerProvider: HeaderProvider | () => Record | Promise>; +``` + +Called for each request to supply authentication headers. + +*** + +### oauthConfig? + +```ts +optional oauthConfig: OAuthConfig; +``` + +*** + +### readConsistencyInterval? + +```ts +optional readConsistencyInterval: number; +``` + +#### Inherited from + +`Omit.readConsistencyInterval` + +*** + +### sqlHostOverride? + +```ts +optional sqlHostOverride: string; +``` + +SQL service endpoint inherited by database connections. + +#### Inherited from + +`Omit.sqlHostOverride` diff --git a/docs/src/js/interfaces/ListDatabasesResponse.md b/docs/src/js/interfaces/ListDatabasesResponse.md new file mode 100644 index 000000000..a1a247746 --- /dev/null +++ b/docs/src/js/interfaces/ListDatabasesResponse.md @@ -0,0 +1,23 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / ListDatabasesResponse + +# Interface: ListDatabasesResponse + +## Properties + +### databases + +```ts +databases: string[]; +``` + +*** + +### pageToken? + +```ts +optional pageToken: string; +``` diff --git a/docs/src/python/python.md b/docs/src/python/python.md index d14e5cf94..96161fbac 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -28,6 +28,17 @@ is also an [asynchronous API client](#connections-asynchronous). ::: lancedb.Session +## Catalogs (Synchronous) + +Remote catalogs manage databases through a server's root namespace. Opened databases +are ordinary connections. Dropping a database requires it to be empty. + +::: lancedb.connect_catalog + +::: lancedb.catalog.Catalog + +::: lancedb.catalog.ListDatabasesResponse + ## Remote SQL Submit SQL against a remote LanceDB database through the connection. @@ -361,6 +372,10 @@ still work. Queries return descriptors. Call ## Connections (Asynchronous) +::: lancedb.connect_catalog_async + +::: lancedb.catalog.AsyncCatalog + Connections represent a connection to a LanceDb database and can be used to create, list, or open tables. diff --git a/nodejs/__test__/catalog.test.ts b/nodejs/__test__/catalog.test.ts new file mode 100644 index 000000000..d44103ab9 --- /dev/null +++ b/nodejs/__test__/catalog.test.ts @@ -0,0 +1,142 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import * as http from "http"; +import { Catalog, connectCatalog } from "../lancedb"; + +type RecordedRequest = { + url: string; + headers: http.IncomingHttpHeaders; + body: Record; +}; + +async function withCatalog( + responses: [number, unknown][], + callback: (catalog: Catalog, requests: RecordedRequest[]) => Promise, +) { + const requests: RecordedRequest[] = []; + const server = http.createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + const body = Buffer.concat(chunks).toString(); + requests.push({ + url: req.url ?? "", + headers: req.headers, + body: body ? JSON.parse(body) : {}, + }); + const [status, response] = responses.shift() ?? [ + 500, + { error: "Unexpected request" }, + ]; + res.writeHead(status, { "content-type": "application/json" }); + res.end(status === 204 ? undefined : JSON.stringify(response)); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("Missing server address"); + try { + const catalog = await connectCatalog(`http://127.0.0.1:${address.port}`, { + apiKey: "secret", + sqlHostOverride: "grpc+tls://sql.example.com:10026", + clientConfig: { + extraHeaders: { + "X-LanceDB-Database": "wrong-static", + "X-LanceDB-Database-Prefix": "wrong", + }, + }, + headerProvider: () => ({ + "X-LanceDB-Database": "wrong-dynamic", + "X-LanceDB-Database-Prefix": "wrong", + authorization: "Bearer refreshed", + }), + }); + await callback(catalog, requests); + expect(responses).toHaveLength(0); + } finally { + server.closeAllConnections(); + await new Promise((resolve) => server.close(() => resolve())); + } +} + +describe("remote catalog", () => { + it("uses root namespace routes and preserves independent database scope", async () => { + await withCatalog( + [ + [204, null], + [200, { tables: [] }], + [200, {}], + [200, { tables: [] }], + [200, { tables: [] }], + // biome-ignore lint/style/useNamingConvention: server wire format + [200, { namespaces: ["team/search"], page_token: "next" }], + [204, null], + ], + async (catalog, requests) => { + const first = await catalog.createDatabase("team/search", { + existOk: true, + }); + expect(await first.tableNames()).toEqual([]); + const second = await catalog.connectDatabase("other"); + expect(await second.tableNames()).toEqual([]); + expect(await first.tableNames()).toEqual([]); + expect( + await catalog.listDatabases({ limit: 1, pageToken: "a/b" }), + ).toEqual({ databases: ["team/search"], pageToken: "next" }); + await catalog.dropDatabase("team/search", { ignoreMissing: true }); + expect(requests[0].url).toBe("/v1/namespace/team%2Fsearch/create"); + expect(requests[0].body).toEqual({ mode: "ExistOk" }); + expect(requests[5].url).toBe( + "/v1/namespace/%24/list?limit=1&page_token=a%2Fb", + ); + expect(requests[6].body).toEqual({ + mode: "Skip", + behavior: "Restrict", + }); + for (const [i, request] of requests.entries()) { + expect(request.headers["x-lancedb-database"]).toBe( + i === 1 || i === 4 ? "team/search" : i === 3 ? "other" : undefined, + ); + expect(request.headers["x-lancedb-database-prefix"]).toBeUndefined(); + expect(request.headers.authorization).toBe("Bearer refreshed"); + } + }, + ); + }); + + it("propagates lifecycle errors and sends restricted drops", async () => { + await withCatalog( + [ + [404, {}], + [409, {}], + [400, {}], + [404, {}], + ], + async (catalog, requests) => { + await expect(catalog.connectDatabase("missing")).rejects.toThrow( + "missing", + ); + await expect(catalog.createDatabase("exists")).rejects.toThrow( + "exists", + ); + await expect(catalog.dropDatabase("full")).rejects.toThrow(); + await catalog.dropDatabase("missing", { ignoreMissing: true }); + expect(requests[2].body).toEqual({ + mode: "Fail", + behavior: "Restrict", + }); + }, + ); + }); + + it("validates endpoints and pagination", async () => { + await expect(connectCatalog("/tmp/catalog")).rejects.toThrow(); + const catalog = await connectCatalog("http://127.0.0.1:1"); + for (const limit of [0, -1, 1.5, 2147483648]) { + await expect(catalog.listDatabases({ limit })).rejects.toThrow("limit"); + } + await expect(catalog.connectDatabase("a$b")).rejects.toThrow( + "Invalid database name", + ); + }); +}); diff --git a/nodejs/lancedb/catalog.ts b/nodejs/lancedb/catalog.ts new file mode 100644 index 000000000..5cfe2c5ea --- /dev/null +++ b/nodejs/lancedb/catalog.ts @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +import { Connection, LocalConnection } from "./connection"; +import { HeaderProvider } from "./header"; +import { + JsHeaderProvider, + ListDatabasesResponse, + Catalog as NativeCatalog, + CatalogOptions as NativeCatalogOptions, +} from "./native.js"; +import { OAuthConfig } from "./oauth"; + +/** Options shared by a catalog and the database connections it returns. */ +export interface CatalogOptions + extends Omit { + oauthConfig?: OAuthConfig; + /** Called for each request to supply authentication headers. */ + headerProvider?: + | HeaderProvider + | (() => Record | Promise>); +} + +export type { ListDatabasesResponse } from "./native.js"; + +/** A remote catalog manages databases through the server's root namespace. */ +export class Catalog { + /** @hidden */ + constructor(private readonly inner: NativeCatalog) {} + + /** The root namespace endpoint. */ + get uri(): string { + return this.inner.uri; + } + + /** Create a database, or open an existing database when existOk is true. */ + async createDatabase( + name: string, + options: { existOk?: boolean } = {}, + ): Promise { + return new LocalConnection( + await this.inner.createDatabase(name, options.existOk), + ); + } + + /** Connect to an existing database by its logical name. */ + async connectDatabase(name: string): Promise { + return new LocalConnection(await this.inner.connectDatabase(name)); + } + + /** Drop an empty database. The server rejects nonempty databases. */ + async dropDatabase( + name: string, + options: { ignoreMissing?: boolean } = {}, + ): Promise { + await this.inner.dropDatabase(name, options.ignoreMissing); + } + + /** List one page of databases; pass pageToken from a response for the next page. */ + async listDatabases( + options: { limit?: number; pageToken?: string } = {}, + ): Promise { + if ( + options.limit !== undefined && + (!Number.isInteger(options.limit) || + options.limit <= 0 || + options.limit > 2147483647) + ) { + throw new Error( + "Database list limit must be an integer between 1 and 2147483647", + ); + } + return this.inner.listDatabases(options.limit, options.pageToken); + } +} + +/** + * Connect to an HTTP(S) catalog endpoint. Catalog requests omit database-selection + * headers; opened database connections inherit authentication and client options. + * + * @example + * ```ts + * const catalog = await connectCatalog("https://my-server.example", { apiKey: "secret" }); + * const db = await catalog.createDatabase("analytics", { existOk: true }); + * const page = await catalog.listDatabases({ limit: 20 }); + * ``` + */ +export async function connectCatalog( + endpoint: string, + options: CatalogOptions = {}, +): Promise { + const { headerProvider, ...nativeOptions } = options; + const provider = headerProvider + ? new JsHeaderProvider(async () => + typeof headerProvider === "function" + ? headerProvider() + : headerProvider.getHeaders(), + ) + : undefined; + return new Catalog( + await NativeCatalog.new(endpoint, nativeOptions, provider), + ); +} diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index 55078bba3..39ceec917 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -629,3 +629,10 @@ export async function connectNamespace( ); return new LocalConnection(nativeConn); } + +export { + Catalog, + CatalogOptions, + ListDatabasesResponse, + connectCatalog, +} from "./catalog"; diff --git a/nodejs/src/catalog.rs b/nodejs/src/catalog.rs new file mode 100644 index 000000000..c2288bc4a --- /dev/null +++ b/nodejs/src/catalog.rs @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +use std::sync::Arc; +use std::time::Duration; + +use lancedb::catalog::{ + CatalogConnection, CreateDatabaseRequest, DropDatabaseRequest, ListDatabasesRequest, +}; +use napi::bindgen_prelude::*; +use napi_derive::napi; + +use crate::connection::Connection; +use crate::error::NapiErrorExt; +use crate::header::JsHeaderProvider; +use crate::remote::{ClientConfig, OAuthConfig}; + +#[napi(object)] +pub struct CatalogOptions { + pub api_key: Option, + pub client_config: Option, + /// SQL service endpoint inherited by database connections. + pub sql_host_override: Option, + pub read_consistency_interval: Option, + pub oauth_config: Option, +} + +#[napi(object)] +pub struct ListDatabasesResponse { + pub databases: Vec, + pub page_token: Option, +} + +#[napi] +pub struct Catalog { + inner: CatalogConnection, +} + +#[napi] +impl Catalog { + #[napi(factory)] + pub async fn new( + endpoint: String, + options: CatalogOptions, + header_provider: Option<&JsHeaderProvider>, + ) -> Result { + let mut builder = lancedb::connect_catalog(endpoint); + if let Some(key) = options.api_key { + builder = builder.api_key(key); + } + let mut config: lancedb::remote::ClientConfig = + options.client_config.unwrap_or_default().into(); + if let Some(provider) = header_provider { + config.header_provider = Some(Arc::new(provider.clone())); + } + builder = builder.client_config(config); + if let Some(endpoint) = options.sql_host_override { + builder = builder.sql_host_override(endpoint); + } + if let Some(interval) = options.read_consistency_interval { + let interval = Duration::try_from_secs_f64(interval).map_err(|err| { + Error::from_reason(format!("Invalid read consistency interval: {err}")) + })?; + builder = builder.read_consistency_interval(interval); + } + if let Some(oauth) = options.oauth_config { + builder = builder.oauth_config(oauth.try_into().default_error()?); + } + Ok(Self { + inner: builder.execute().await.default_error()?, + }) + } + + #[napi(getter)] + pub fn uri(&self) -> String { + self.inner.uri().to_string() + } + + #[napi] + pub async fn create_database( + &self, + name: String, + exist_ok: Option, + ) -> Result { + self.inner + .create_database(CreateDatabaseRequest::new(name).exist_ok(exist_ok.unwrap_or(false))) + .await + .map(Connection::inner_new) + .default_error() + } + #[napi] + pub async fn connect_database(&self, name: String) -> Result { + self.inner + .connect_database(name) + .await + .map(Connection::inner_new) + .default_error() + } + #[napi] + pub async fn drop_database(&self, name: String, ignore_missing: Option) -> Result<()> { + self.inner + .drop_database( + DropDatabaseRequest::new(name).ignore_missing(ignore_missing.unwrap_or(false)), + ) + .await + .default_error() + } + #[napi] + pub async fn list_databases( + &self, + limit: Option, + page_token: Option, + ) -> Result { + let mut request = ListDatabasesRequest::default(); + request.limit = limit; + request.page_token = page_token; + let response = self.inner.list_databases(request).await.default_error()?; + Ok(ListDatabasesResponse { + databases: response.databases, + page_token: response.page_token, + }) + } +} diff --git a/nodejs/src/lib.rs b/nodejs/src/lib.rs index 288a2b925..1bb5f761d 100644 --- a/nodejs/src/lib.rs +++ b/nodejs/src/lib.rs @@ -11,6 +11,7 @@ use env_logger::Env; use napi_derive::*; mod blob; +mod catalog; mod connection; mod error; mod header; diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index cb9b57be3..19114e1a1 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -52,6 +52,14 @@ from .namespace import ( AsyncLanceNamespaceDBConnection, ) +from .catalog import ( + AsyncCatalog, + Catalog, + ListDatabasesResponse, + connect_catalog, + connect_catalog_async, +) + if TYPE_CHECKING: from lance.blob import BlobType as BlobType @@ -561,6 +569,11 @@ async def connect_async( __all__ = [ + "Catalog", + "AsyncCatalog", + "ListDatabasesResponse", + "connect_catalog", + "connect_catalog_async", "AsyncMaterializedView", "MaterializedView", "MaterializedViewDefinition", diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index e18657eb7..f267b3a55 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -856,3 +856,27 @@ def fts_query_to_json(query: Any) -> str: ... class PermutationReader: def __init__(self, base_table: Table, permutation_table: Table): ... + +class Catalog: + @property + def uri(self) -> str: ... + async def create_database( + self, name: str, *, exist_ok: bool = False + ) -> Connection: ... + async def connect_database(self, name: str) -> Connection: ... + async def drop_database( + self, name: str, *, ignore_missing: bool = False + ) -> None: ... + async def list_databases( + self, *, limit: Optional[int] = None, page_token: Optional[str] = None + ) -> tuple[list[str], Optional[str]]: ... + +async def connect_catalog( + endpoint: str, + *, + api_key: Optional[str] = None, + client_config: Optional[Any] = None, + sql_host_override: Optional[str] = None, + read_consistency_interval: Optional[float] = None, + oauth_config: Optional[Any] = None, +) -> Catalog: ... diff --git a/python/python/lancedb/catalog.py b/python/python/lancedb/catalog.py new file mode 100644 index 000000000..4fd941ddd --- /dev/null +++ b/python/python/lancedb/catalog.py @@ -0,0 +1,196 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""Remote catalogs manage databases through a server's root namespace.""" + +from dataclasses import dataclass +from datetime import timedelta +from typing import Any, Optional, Union + +from . import _lancedb +from .background_loop import LOOP +from .db import AsyncConnection, DBConnection +from .remote import ClientConfig, OAuthConfig +from .remote.db import RemoteDBConnection + + +@dataclass +class ListDatabasesResponse: + """A page of database names and an optional continuation token.""" + + databases: list[str] + page_token: Optional[str] = None + + +class AsyncCatalog: + """An asynchronous remote catalog returned by + [connect_catalog_async][lancedb.connect_catalog_async]. + + Create/connect return ordinary [AsyncConnection][lancedb.db.AsyncConnection] + instances. Drop uses restricted behavior: remove the database's tables first. + """ + + def __init__(self, inner: _lancedb.Catalog): + self._inner = inner + + @property + def uri(self) -> str: + """The catalog's root namespace endpoint.""" + return self._inner.uri + + async def create_database( + self, name: str, *, exist_ok: bool = False + ) -> AsyncConnection: + """Create a database, or open an existing one when ``exist_ok=True``.""" + return AsyncConnection( + await self._inner.create_database(name, exist_ok=exist_ok) + ) + + async def connect_database(self, name: str) -> AsyncConnection: + """Connect to an existing database by its logical name.""" + return AsyncConnection(await self._inner.connect_database(name)) + + async def list_databases( + self, *, limit: Optional[int] = None, page_token: Optional[str] = None + ) -> ListDatabasesResponse: + """List a page of databases. Pass the returned token for the next page.""" + names, token = await self._inner.list_databases( + limit=limit, page_token=page_token + ) + return ListDatabasesResponse(names, token) + + async def drop_database(self, name: str, *, ignore_missing: bool = False) -> None: + """Drop an empty database. A nonempty database is an error.""" + await self._inner.drop_database(name, ignore_missing=ignore_missing) + + +class Catalog: + """A synchronous remote catalog returned by + [connect_catalog][lancedb.connect_catalog]. + + Examples + -------- + ```python + catalog = lancedb.connect_catalog("https://my-server.example", api_key="secret") + db = catalog.create_database("analytics", exist_ok=True) + page = catalog.list_databases(limit=20) + ``` + """ + + def __init__( + self, + inner: AsyncCatalog, + *, + api_key=None, + client_config=None, + sql_host_override: Optional[str] = None, + oauth_config: Optional[OAuthConfig] = None, + ): + self._inner = inner + self._api_key = api_key + self._client_config = client_config + self._sql_host_override = sql_host_override + self._oauth_config = oauth_config + + @property + def uri(self) -> str: + """The catalog's root namespace endpoint.""" + return self._inner.uri + + def create_database(self, name: str, *, exist_ok: bool = False) -> DBConnection: + """Create a database, or open an existing one when ``exist_ok=True``.""" + inner = LOOP.run(self._inner.create_database(name, exist_ok=exist_ok)) + return self._wrap_database(name, inner) + + def connect_database(self, name: str) -> DBConnection: + """Connect to an existing database by its logical name.""" + return self._wrap_database(name, LOOP.run(self._inner.connect_database(name))) + + def _wrap_database(self, name: str, inner: AsyncConnection) -> DBConnection: + return RemoteDBConnection._from_catalog( + inner, + name, + self.uri, + self._api_key, + self._client_config, + self._oauth_config, + self._sql_host_override, + ) + + def list_databases( + self, *, limit: Optional[int] = None, page_token: Optional[str] = None + ) -> ListDatabasesResponse: + """List a page of databases. Pass the returned token for the next page.""" + return LOOP.run(self._inner.list_databases(limit=limit, page_token=page_token)) + + def drop_database(self, name: str, *, ignore_missing: bool = False) -> None: + """Drop an empty database. A nonempty database is an error.""" + LOOP.run(self._inner.drop_database(name, ignore_missing=ignore_missing)) + + +async def connect_catalog_async( + endpoint: str, + *, + api_key: Optional[str] = None, + client_config: Optional[Union[ClientConfig, dict[str, Any]]] = None, + sql_host_override: Optional[str] = None, + read_consistency_interval: Optional[timedelta] = None, + oauth_config: Optional[OAuthConfig] = None, +) -> AsyncCatalog: + """Connect to an HTTP(S) server's root catalog. + + Root requests omit database-selection headers. API key, client configuration, + OAuth, and table read consistency settings are inherited by opened databases. + Database names containing slashes remain single logical names. + Set ``sql_host_override`` to the SQL service endpoint to execute SQL through + returned connections when the catalog endpoint uses HTTPS. + """ + if isinstance(client_config, dict): + client_config = ClientConfig(**client_config) + if client_config is None: + client_config = ClientConfig() + inner = await _lancedb.connect_catalog( + endpoint, + api_key=api_key, + client_config=client_config, + sql_host_override=sql_host_override, + read_consistency_interval=( + read_consistency_interval.total_seconds() + if read_consistency_interval is not None + else None + ), + oauth_config=oauth_config, + ) + return AsyncCatalog(inner) + + +def connect_catalog( + endpoint: str, + *, + api_key: Optional[str] = None, + client_config: Optional[Union[ClientConfig, dict[str, Any]]] = None, + sql_host_override: Optional[str] = None, + read_consistency_interval: Optional[timedelta] = None, + oauth_config: Optional[OAuthConfig] = None, +) -> Catalog: + """Connect synchronously to an HTTP(S) server's root catalog. + + See [connect_catalog_async][lancedb.connect_catalog_async] for options. + Local filesystem and object-store catalogs are not supported. + """ + return Catalog( + LOOP.run( + connect_catalog_async( + endpoint, + api_key=api_key, + client_config=client_config, + sql_host_override=sql_host_override, + read_consistency_interval=read_consistency_interval, + oauth_config=oauth_config, + ) + ), + api_key=api_key, + client_config=client_config, + sql_host_override=sql_host_override, + oauth_config=oauth_config, + ) diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index fb4e30fdf..aceebf987 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -2,6 +2,7 @@ # SPDX-FileCopyrightText: Copyright The LanceDB Authors +from dataclasses import replace from datetime import timedelta import json import logging @@ -187,11 +188,51 @@ class RemoteDBConnection(DBConnection): ) ) + @classmethod + def _from_catalog( + cls, + inner, + name, + endpoint, + api_key, + client_config, + oauth_config, + sql_host_override, + ): + config = ( + ClientConfig(**client_config) + if isinstance(client_config, dict) + else (client_config or ClientConfig()) + ) + headers = { + key: value + for key, value in (config.extra_headers or {}).items() + if key.lower() not in ("x-lancedb-database", "x-lancedb-database-prefix") + } + headers["x-lancedb-database"] = name + result = cls.__new__(cls) + result.db_url = inner.uri + result.db_name = name + result.api_key = api_key or "" + result.region = "us-east-1" + result.host_override = endpoint + result.sql_host_override = sql_host_override + result.storage_options = None + result.client_config = replace(config, extra_headers=headers) + result._catalog_oauth = oauth_config is not None + result._conn = inner + return result + def __repr__(self) -> str: return f"RemoteConnect(name={self.db_name})" @override def serialize(self) -> str: + if getattr(self, "_catalog_oauth", False): + raise ValueError( + "Cannot serialize a catalog connection using OAuth; " + "provide a worker-side connection factory" + ) return json.dumps( { "connection_type": "remote", diff --git a/python/python/tests/test_catalog.py b/python/python/tests/test_catalog.py new file mode 100644 index 000000000..63386a998 --- /dev/null +++ b/python/python/tests/test_catalog.py @@ -0,0 +1,140 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +from threading import Thread + +import pytest + +import lancedb +from lancedb.db import AsyncConnection, DBConnection +from lancedb.remote.errors import HttpError + + +@pytest.fixture +def catalog_server(): + requests = [] + responses = [] + + class Handler(BaseHTTPRequestHandler): + def handle_request(self): + body = self.rfile.read(int(self.headers.get("Content-Length", "0"))) + requests.append( + ( + self.path, + dict(self.headers.items()), + json.loads(body) if body else None, + ) + ) + status, response = responses.pop(0) + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.end_headers() + if status != 204: + self.wfile.write(json.dumps(response).encode()) + + do_GET = handle_request + do_POST = handle_request + + with ThreadingHTTPServer(("127.0.0.1", 0), Handler) as server: + thread = Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}", requests, responses + finally: + server.shutdown() + thread.join() + + +def test_catalog_sync_scope_and_serialization(catalog_server): + endpoint, requests, responses = catalog_server + responses.extend( + [ + (204, None), + (200, {}), + (200, {"tables": []}), + (200, {"tables": []}), + (200, {"namespaces": ["team/search"], "page_token": "next"}), + (204, None), + ] + ) + catalog = lancedb.connect_catalog( + endpoint, + api_key="secret", + sql_host_override="invalid://localhost", + client_config={ + "extra_headers": { + "X-LanceDB-Database": "wrong", + "X-LanceDB-Database-Prefix": "wrong", + } + }, + ) + assert isinstance(catalog, lancedb.Catalog) + assert catalog.uri == endpoint + db = catalog.create_database("team/search", exist_ok=True) + assert isinstance(db, DBConnection) + with pytest.raises(ValueError, match="sql_host_override must use"): + db.execute_query_async("SELECT 1") + db = catalog.connect_database("team/search") + assert isinstance(db, DBConnection) + assert db.table_names() == [] + restored = lancedb.deserialize_conn(db.serialize()) + assert restored.sql_host_override == "invalid://localhost" + for connection in (db, restored): + with pytest.raises(ValueError, match="sql_host_override must use"): + connection.execute_query_async("SELECT 1") + assert restored.table_names() == [] + page = catalog.list_databases(limit=1, page_token="a/b") + assert page == lancedb.ListDatabasesResponse(["team/search"], "next") + catalog.drop_database("team/search", ignore_missing=True) + assert requests[0][0] == "/v1/namespace/team%2Fsearch/create" + assert requests[0][2] == {"mode": "ExistOk"} + assert requests[1][0] == "/v1/namespace/team%2Fsearch/describe" + assert requests[4][0] == "/v1/namespace/%24/list?limit=1&page_token=a%2Fb" + assert requests[5][2] == {"mode": "Skip", "behavior": "Restrict"} + for i, (_, headers, _) in enumerate(requests): + headers = {key.lower(): value for key, value in headers.items()} + assert headers.get("x-lancedb-database") == ( + "team/search" if i in (2, 3) else None + ) + assert "x-lancedb-database-prefix" not in headers + assert headers["x-api-key"] == "secret" + + +@pytest.mark.asyncio +async def test_catalog_async_and_errors(catalog_server): + endpoint, requests, responses = catalog_server + responses.extend( + [ + (200, {}), + (200, {"tables": []}), + (404, {"error": "missing"}), + (409, {"error": "exists"}), + (400, {"error": "not empty"}), + (404, {"error": "missing"}), + ] + ) + catalog = await lancedb.connect_catalog_async( + endpoint, sql_host_override="invalid://localhost" + ) + assert isinstance(catalog, lancedb.AsyncCatalog) + db = await catalog.connect_database("analytics") + assert isinstance(db, AsyncConnection) + with pytest.raises(ValueError, match="sql_host_override must use"): + await db.execute_query_async("SELECT 1") + assert await db.table_names() == [] + with pytest.raises(ValueError, match="missing"): + await catalog.connect_database("missing") + with pytest.raises(ValueError, match="exists"): + await catalog.create_database("exists") + with pytest.raises(HttpError): + await catalog.drop_database("full") + await catalog.drop_database("missing", ignore_missing=True) + assert requests[4][2] == {"mode": "Fail", "behavior": "Restrict"} + + +@pytest.mark.parametrize("endpoint", ["/tmp/catalog", "s3://bucket", "db://db"]) +def test_catalog_requires_remote_endpoint(endpoint): + with pytest.raises(ValueError, match="endpoint"): + lancedb.connect_catalog(endpoint) diff --git a/python/src/catalog.rs b/python/src/catalog.rs new file mode 100644 index 000000000..f61b409ff --- /dev/null +++ b/python/src/catalog.rs @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +use std::time::Duration; + +use lancedb::catalog::{ + CatalogConnection, CreateDatabaseRequest, DropDatabaseRequest, ListDatabasesRequest, +}; +use pyo3::exceptions::PyValueError; +use pyo3::{Bound, PyAny, PyRef, PyResult, Python, pyclass, pyfunction, pymethods}; + +use crate::connection::{Connection, PyClientConfig}; +use crate::error::PythonErrorExt; +use crate::runtime::future_into_py; + +#[pyclass] +pub struct Catalog { + inner: CatalogConnection, +} + +#[pymethods] +impl Catalog { + #[getter] + fn uri(&self) -> &str { + self.inner.uri() + } + + #[pyo3(signature = (name, *, exist_ok=false))] + fn create_database<'py>( + self_: PyRef<'py, Self>, + name: String, + exist_ok: bool, + ) -> PyResult> { + let inner = self_.inner.clone(); + future_into_py(self_.py(), async move { + inner + .create_database(CreateDatabaseRequest::new(name).exist_ok(exist_ok)) + .await + .map(Connection::new) + .infer_error() + }) + } + + fn connect_database<'py>(self_: PyRef<'py, Self>, name: String) -> PyResult> { + let inner = self_.inner.clone(); + future_into_py(self_.py(), async move { + inner + .connect_database(name) + .await + .map(Connection::new) + .infer_error() + }) + } + + #[pyo3(signature = (name, *, ignore_missing=false))] + fn drop_database<'py>( + self_: PyRef<'py, Self>, + name: String, + ignore_missing: bool, + ) -> PyResult> { + let inner = self_.inner.clone(); + future_into_py(self_.py(), async move { + inner + .drop_database(DropDatabaseRequest::new(name).ignore_missing(ignore_missing)) + .await + .infer_error() + }) + } + + #[pyo3(signature = (*, limit=None, page_token=None))] + fn list_databases<'py>( + self_: PyRef<'py, Self>, + limit: Option, + page_token: Option, + ) -> PyResult> { + let inner = self_.inner.clone(); + future_into_py(self_.py(), async move { + let mut request = ListDatabasesRequest::default(); + request.limit = limit; + request.page_token = page_token; + let response = inner.list_databases(request).await.infer_error()?; + Ok((response.databases, response.page_token)) + }) + } +} + +#[pyfunction] +#[pyo3(signature = (endpoint, *, api_key=None, client_config=None, sql_host_override=None, read_consistency_interval=None, oauth_config=None))] +pub fn connect_catalog( + py: Python<'_>, + endpoint: String, + api_key: Option, + client_config: Option, + sql_host_override: Option, + read_consistency_interval: Option, + oauth_config: Option, +) -> PyResult> { + let interval = read_consistency_interval + .map(Duration::try_from_secs_f64) + .transpose() + .map_err(|err| { + PyValueError::new_err(format!("Invalid read consistency interval: {err}")) + })?; + future_into_py(py, async move { + let mut builder = lancedb::connect_catalog(endpoint); + if let Some(api_key) = api_key { + builder = builder.api_key(api_key); + } + if let Some(config) = client_config { + builder = builder.client_config(config.into()); + } + if let Some(endpoint) = sql_host_override { + builder = builder.sql_host_override(endpoint); + } + if let Some(interval) = interval { + builder = builder.read_consistency_interval(interval); + } + if let Some(config) = oauth_config { + builder = builder.oauth_config(config.try_into().infer_error()?); + } + Ok(Catalog { + inner: builder.execute().await.infer_error()?, + }) + }) +} diff --git a/python/src/error.rs b/python/src/error.rs index b46fa6c83..869a5a247 100644 --- a/python/src/error.rs +++ b/python/src/error.rs @@ -31,6 +31,8 @@ impl PythonErrorExt for std::result::Result { | LanceError::TableNotFound { .. } | LanceError::NotAMaterializedView { .. } | LanceError::Schema { .. } + | LanceError::DatabaseNotFound { .. } + | LanceError::DatabaseAlreadyExists { .. } | LanceError::TableAlreadyExists { .. } => self.value_error(), LanceError::CreateDir { .. } => self.os_error(), LanceError::ObjectStore { .. } => Err(PyIOError::new_err(err.to_string())), diff --git a/python/src/lib.rs b/python/src/lib.rs index 0bd7d5d6d..fd9a23798 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -21,6 +21,7 @@ use table::{ }; pub mod arrow; +pub mod catalog; pub mod connection; pub mod error; pub mod expr; @@ -61,6 +62,8 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { .write_style("LANCEDB_LOG_STYLE"); env_logger::init_from_env(env); m.add_class::()?; + m.add_class::()?; + m.add_function(wrap_pyfunction!(catalog::connect_catalog, m)?)?; m.add_class::()?; m.add_class::
()?; m.add_class::()?; diff --git a/rust/lancedb/src/catalog.rs b/rust/lancedb/src/catalog.rs new file mode 100644 index 000000000..5eaa1d49e --- /dev/null +++ b/rust/lancedb/src/catalog.rs @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Catalogs manage databases. A remote catalog is the root namespace of a server. +//! +//! ``` +//! # #[cfg(feature = "remote")] +//! # async fn example() -> lancedb::Result<()> { +//! let catalog = lancedb::connect_catalog("https://my-server.example") +//! .api_key("my-api-key") +//! .execute().await?; +//! let database = catalog.create_database("analytics").await?; +//! # Ok(()) +//! # } +//! ``` + +use std::fmt; +use std::sync::Arc; + +use crate::Result; +use crate::connection::Connection; +use crate::database::Database; +use crate::embeddings::{EmbeddingRegistry, MemoryRegistry}; + +/// Options for creating a database. By default an existing name is an error. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct CreateDatabaseRequest { + /// Logical database name, including any literal slashes. + pub name: String, + /// Open the existing database if it is already registered. + pub exist_ok: bool, +} + +impl CreateDatabaseRequest { + /// Initialize a request with the default behavior. + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + exist_ok: false, + } + } + + /// Open an existing database instead of failing if its name already exists. + pub fn exist_ok(mut self, value: bool) -> Self { + self.exist_ok = value; + self + } +} + +impl> From for CreateDatabaseRequest { + fn from(name: T) -> Self { + Self::new(name) + } +} + +/// Options for restricted database deletion. Tables must be removed first. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct DropDatabaseRequest { + /// Logical database name, including any literal slashes. + pub name: String, + /// Succeed if the database is absent. + pub ignore_missing: bool, +} + +impl DropDatabaseRequest { + /// Initialize a request with the default behavior. + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + ignore_missing: false, + } + } + + /// Succeed when the database does not exist. + pub fn ignore_missing(mut self, value: bool) -> Self { + self.ignore_missing = value; + self + } +} + +impl> From for DropDatabaseRequest { + fn from(name: T) -> Self { + Self::new(name) + } +} + +/// Pagination options for listing databases. +#[derive(Clone, Debug, Default)] +#[non_exhaustive] +pub struct ListDatabasesRequest { + /// Maximum number of names to return. None uses the server default. + pub limit: Option, + /// Opaque continuation token from a previous response. None starts a listing. + pub page_token: Option, +} + +impl ListDatabasesRequest { + /// Set the maximum page size (1 through 2147483647). + pub fn limit(mut self, limit: u32) -> Self { + self.limit = Some(limit); + self + } + /// Resume a listing from an opaque continuation token. + pub fn page_token(mut self, token: impl Into) -> Self { + self.page_token = Some(token.into()); + self + } +} + +/// One page of database names, relative to the catalog. +#[derive(Clone, Debug, Default)] +pub struct ListDatabasesResponse { + /// Logical database names on this page. + pub databases: Vec, + /// None indicates the end of the listing. + pub page_token: Option, +} + +/// A backend that manages databases. Implementations own database lifecycle semantics. +#[async_trait::async_trait] +pub trait Catalog: Send + Sync + std::fmt::Debug + 'static { + /// Catalog endpoint or location. + fn uri(&self) -> &str; + /// Create a database, or open it when `exist_ok` permits. + async fn create_database(&self, request: CreateDatabaseRequest) -> Result>; + /// Drop an empty database. This must not cascade to tables. + async fn drop_database(&self, request: DropDatabaseRequest) -> Result<()>; + /// List a page of database names. + async fn list_databases(&self, request: ListDatabasesRequest) -> Result; + /// Connect to an existing database by its logical name. + async fn connect_database(&self, name: &str) -> Result>; +} + +/// A catalog connection that returns ordinary LanceDB database connections. +#[derive(Clone)] +pub struct CatalogConnection { + catalog: Arc, + embedding_registry: Arc, +} + +impl fmt::Debug for CatalogConnection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("CatalogConnection") + .field("uri", &self.uri()) + .finish_non_exhaustive() + } +} + +impl CatalogConnection { + /// Wrap a catalog implementation using the default in-memory embedding registry. + pub fn new(catalog: Arc) -> Self { + Self { + catalog, + embedding_registry: Arc::new(MemoryRegistry::new()), + } + } + + /// Provide the registry used by databases opened through this connection. + pub fn with_embedding_registry(mut self, registry: Arc) -> Self { + self.embedding_registry = registry; + self + } + + /// The catalog endpoint or location. + pub fn uri(&self) -> &str { + self.catalog.uri() + } + /// Access the underlying backend. + pub fn catalog(&self) -> &Arc { + &self.catalog + } + + /// Create a database; pass a name or [`CreateDatabaseRequest`] for additional options. + pub async fn create_database( + &self, + request: impl Into, + ) -> Result { + Ok(Connection::new( + self.catalog.create_database(request.into()).await?, + self.embedding_registry.clone(), + )) + } + + /// Drop an empty database; pass a name or [`DropDatabaseRequest`] for additional options. + pub async fn drop_database(&self, request: impl Into) -> Result<()> { + self.catalog.drop_database(request.into()).await + } + + /// List a page of databases. Pass [`ListDatabasesRequest::default`] for the first page. + pub async fn list_databases( + &self, + request: ListDatabasesRequest, + ) -> Result { + self.catalog.list_databases(request).await + } + + /// Connect to a database without creating it if it is missing. + pub async fn connect_database(&self, name: impl AsRef) -> Result { + Ok(Connection::new( + self.catalog.connect_database(name.as_ref()).await?, + self.embedding_registry.clone(), + )) + } +} + +/// Configure a connection to a remote catalog. +#[cfg(feature = "remote")] +#[derive(Debug)] +pub struct ConnectCatalogBuilder { + endpoint: String, + options: crate::remote::RemoteCatalogOptions, +} + +#[cfg(feature = "remote")] +impl ConnectCatalogBuilder { + /// Start configuring a remote HTTP(S) catalog connection. + pub fn new(endpoint: impl Into) -> Self { + Self { + endpoint: endpoint.into(), + options: Default::default(), + } + } + /// Authenticate with an API key. + pub fn api_key(mut self, key: impl Into) -> Self { + self.options.api_key = Some(key.into()); + self + } + /// Configure headers, TLS, timeouts, and other shared client settings. + pub fn client_config(mut self, config: crate::remote::ClientConfig) -> Self { + self.options.client_config = config; + self + } + /// Set the SQL service endpoint inherited by database connections. + /// + /// Required to execute SQL when the catalog endpoint uses HTTPS. The SQL + /// connection is initialized lazily, using the ordinary remote SQL client. + pub fn sql_host_override(mut self, endpoint: impl Into) -> Self { + self.options.sql_host_override = Some(endpoint.into()); + self + } + /// Configure table read consistency for opened databases. + pub fn read_consistency_interval(mut self, interval: std::time::Duration) -> Self { + self.options.read_consistency_interval = Some(interval); + self + } + /// Authenticate using OAuth; mutually exclusive with API keys and header providers. + pub fn oauth_config(mut self, config: crate::remote::OAuthConfig) -> Self { + self.options.oauth_config = Some(config); + self + } + /// Connect to the server's root namespace. Database-scoped headers are omitted. + pub async fn execute(self) -> Result { + Ok(CatalogConnection::new(Arc::new( + crate::remote::RemoteCatalog::try_new(&self.endpoint, self.options)?, + ))) + } +} diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs index 44c5dd616..b173a63cb 100644 --- a/rust/lancedb/src/lib.rs +++ b/rust/lancedb/src/lib.rs @@ -174,6 +174,7 @@ pub mod arrow; pub mod blob; +pub mod catalog; pub mod connection; pub mod data; pub mod database; @@ -383,3 +384,9 @@ pub use lance_io::object_store::ObjectStoreRegistry; /// declaring their own (potentially mismatched) direct `datafusion` dependency. /// See . pub use datafusion; + +/// Connect to a remote catalog through its HTTP(S) root namespace endpoint. +#[cfg(feature = "remote")] +pub fn connect_catalog(endpoint: impl Into) -> catalog::ConnectCatalogBuilder { + catalog::ConnectCatalogBuilder::new(endpoint) +} diff --git a/rust/lancedb/src/remote.rs b/rust/lancedb/src/remote.rs index 4441e01ee..bc534c0f4 100644 --- a/rust/lancedb/src/remote.rs +++ b/rust/lancedb/src/remote.rs @@ -6,6 +6,7 @@ //! building client/server applications with LanceDB or as a client for some //! other custom LanceDB service. +pub mod catalog; pub(crate) mod client; pub(crate) mod db; pub(crate) mod job; @@ -36,3 +37,5 @@ pub use oauth::{ AuthorizationCodeOptions, ClientAuthMethod, OAuthConfig, OAuthFlow, OAuthHeaderProvider, }; pub use token_cache::{OAuthSession, SessionLogout, SessionStatus, TokenCacheOptions}; + +pub use catalog::{RemoteCatalog, RemoteCatalogOptions}; diff --git a/rust/lancedb/src/remote/catalog.rs b/rust/lancedb/src/remote/catalog.rs new file mode 100644 index 000000000..098a696a7 --- /dev/null +++ b/rust/lancedb/src/remote/catalog.rs @@ -0,0 +1,598 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use http::StatusCode; +use lance_namespace::models::{ + CreateNamespaceRequest, DescribeNamespaceRequest, DropNamespaceRequest, ListNamespacesRequest, +}; + +use super::db::RemoteDatabase; +use super::{ClientConfig, HeaderProvider, OAuthConfig, OAuthHeaderProvider}; +use crate::catalog::{ + Catalog, CreateDatabaseRequest, DropDatabaseRequest, ListDatabasesRequest, + ListDatabasesResponse, +}; +use crate::database::Database; +use crate::{Error, Result}; + +/// Authentication and client settings shared by a catalog and its databases. +#[derive(Clone, Default)] +#[non_exhaustive] +pub struct RemoteCatalogOptions { + /// Optional API key for catalog and database requests. + pub api_key: Option, + /// Shared transport and authentication settings. + pub client_config: ClientConfig, + /// SQL service endpoint used by returned database connections. + /// Required for SQL when the catalog endpoint uses HTTPS. + pub sql_host_override: Option, + /// Read consistency interval for tables opened in returned databases. + pub read_consistency_interval: Option, + /// OAuth authentication, mutually exclusive with an API key or header provider. + pub oauth_config: Option, +} + +impl std::fmt::Debug for RemoteCatalogOptions { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RemoteCatalogOptions") + .field("read_consistency_interval", &self.read_consistency_interval) + .finish_non_exhaustive() + } +} + +#[derive(Clone)] +pub(crate) struct ScopedHeaderProvider { + pub provider: Option>, + pub database: Option, +} + +impl std::fmt::Debug for ScopedHeaderProvider { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ScopedHeaderProvider") + .field("database", &self.database) + .finish_non_exhaustive() + } +} + +impl ScopedHeaderProvider { + pub(crate) fn apply(&self, headers: &mut HashMap) { + headers.retain(|name, _| { + !name.eq_ignore_ascii_case("x-lancedb-database") + && !name.eq_ignore_ascii_case("x-lancedb-database-prefix") + }); + if let Some(database) = &self.database { + headers.insert("x-lancedb-database".into(), database.clone()); + } + } +} + +#[async_trait] +impl HeaderProvider for ScopedHeaderProvider { + async fn get_headers(&self) -> Result> { + let mut headers = match &self.provider { + Some(provider) => provider.get_headers().await?, + None => HashMap::new(), + }; + self.apply(&mut headers); + Ok(headers) + } +} + +/// A catalog backed by the server's root namespace APIs. +/// +/// Database management requests omit database-selection headers. Opened database +/// connections retain their own scope and authentication independently. +pub struct RemoteCatalog { + endpoint: String, + root: RemoteDatabase, + options: RemoteCatalogOptions, +} + +impl fmt::Debug for RemoteCatalog { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RemoteCatalog") + .field("uri", &self.endpoint) + .finish_non_exhaustive() + } +} + +impl RemoteCatalog { + /// Connect to an HTTP(S) root namespace endpoint. + /// + /// ``` + /// # use lancedb::remote::{RemoteCatalog, RemoteCatalogOptions}; + /// # fn example() -> lancedb::Result<()> { + /// let catalog = RemoteCatalog::try_new("https://my-server.example", RemoteCatalogOptions::default())?; + /// # Ok(()) + /// # } + /// ``` + pub fn try_new(endpoint: impl AsRef, mut options: RemoteCatalogOptions) -> Result { + let url = url::Url::parse(endpoint.as_ref()).map_err(|err| Error::InvalidInput { + message: format!("Invalid catalog endpoint: {err}"), + })?; + if !matches!(url.scheme(), "http" | "https") + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.query().is_some() + || url.fragment().is_some() + { + return Err(Error::InvalidInput { message: "Catalog endpoint must be an HTTP(S) URL without credentials, query, or fragment".into() }); + } + if options + .client_config + .id_delimiter + .as_ref() + .is_some_and(|d| d.is_empty()) + { + return Err(Error::InvalidInput { + message: "Catalog identifier delimiter cannot be empty".into(), + }); + } + if let Some(oauth) = options.oauth_config.take() { + if options.api_key.is_some() || options.client_config.header_provider.is_some() { + return Err(Error::InvalidInput { + message: "oauth_config cannot be combined with api_key or header_provider" + .into(), + }); + } + options.client_config.header_provider = + Some(Arc::new(OAuthHeaderProvider::new(oauth)?)); + } + let endpoint = url.to_string().trim_end_matches('/').to_string(); + let root = RemoteDatabase::for_catalog(&endpoint, None, &options)?; + Ok(Self { + endpoint, + root, + options, + }) + } + + fn validate_name(&self, name: &str) -> Result<()> { + let delimiter = self + .options + .client_config + .id_delimiter + .as_deref() + .unwrap_or("$"); + if name.is_empty() + || name.trim() != name + || !name.is_ascii() + || name.chars().any(char::is_control) + || name.contains(delimiter) + || matches!(name, "." | "..") + { + return Err(Error::InvalidInput { + message: format!( + "Invalid database name '{name}': expected a nonempty ASCII name without surrounding whitespace, control characters, or namespace delimiter '{delimiter}'" + ), + }); + } + Ok(()) + } + + fn database(&self, name: &str) -> Result> { + Ok(Arc::new(RemoteDatabase::for_catalog( + &self.endpoint, + Some(name), + &self.options, + )?)) + } + + fn map_missing(name: &str, err: Error) -> Error { + match err { + Error::Http { + status_code: Some(StatusCode::NOT_FOUND), + .. + } => Error::DatabaseNotFound { name: name.into() }, + err => err, + } + } +} + +#[async_trait] +impl Catalog for RemoteCatalog { + fn uri(&self) -> &str { + &self.endpoint + } + + async fn create_database(&self, request: CreateDatabaseRequest) -> Result> { + self.validate_name(&request.name)?; + self.root + .create_namespace(CreateNamespaceRequest { + id: Some(vec![request.name.clone()]), + mode: Some( + if request.exist_ok { + "ExistOk" + } else { + "Create" + } + .into(), + ), + ..Default::default() + }) + .await + .map_err(|err| match err { + Error::Http { + status_code: Some(StatusCode::CONFLICT), + .. + } => Error::DatabaseAlreadyExists { + name: request.name.clone(), + }, + err => err, + })?; + self.database(&request.name) + } + + async fn drop_database(&self, request: DropDatabaseRequest) -> Result<()> { + self.validate_name(&request.name)?; + let result = self + .root + .drop_namespace(DropNamespaceRequest { + id: Some(vec![request.name.clone()]), + mode: Some( + if request.ignore_missing { + "Skip" + } else { + "Fail" + } + .into(), + ), + behavior: Some("Restrict".into()), + ..Default::default() + }) + .await; + match result { + Ok(_) => Ok(()), + Err(Error::Http { + status_code: Some(StatusCode::NOT_FOUND), + .. + }) if request.ignore_missing => Ok(()), + Err(err) => Err(Self::map_missing(&request.name, err)), + } + } + + async fn list_databases(&self, request: ListDatabasesRequest) -> Result { + let limit = request + .limit + .map(|limit| { + i32::try_from(limit) + .ok() + .filter(|limit| *limit > 0) + .ok_or_else(|| Error::InvalidInput { + message: "Database list limit must be between 1 and 2147483647".into(), + }) + }) + .transpose()?; + let response = self + .root + .list_namespaces(ListNamespacesRequest { + id: Some(vec![]), + limit, + page_token: request.page_token, + ..Default::default() + }) + .await?; + Ok(ListDatabasesResponse { + databases: response.namespaces, + page_token: response.page_token.filter(|token| !token.is_empty()), + }) + } + + async fn connect_database(&self, name: &str) -> Result> { + self.validate_name(name)?; + self.root + .describe_namespace(DescribeNamespaceRequest { + id: Some(vec![name.into()]), + ..Default::default() + }) + .await + .map_err(|err| Self::map_missing(name, err))?; + self.database(name) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::catalog::CatalogConnection; + use serde_json::{Value, json}; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + use tokio::task::JoinHandle; + + #[derive(Debug)] + struct Request { + line: String, + headers: HashMap, + body: Value, + } + + async fn server(responses: Vec<(u16, Value)>) -> (String, JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + let task = tokio::spawn(async move { + let mut requests = Vec::new(); + for (status, body) in responses { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut bytes = Vec::new(); + let header_end = loop { + let mut buf = [0; 4096]; + let n = socket.read(&mut buf).await.unwrap(); + assert!(n > 0); + bytes.extend_from_slice(&buf[..n]); + if let Some(pos) = bytes.windows(4).position(|b| b == b"\r\n\r\n") { + break pos + 4; + } + }; + let header = String::from_utf8(bytes[..header_end].to_vec()).unwrap(); + let mut lines = header.lines(); + let line = lines.next().unwrap().to_string(); + let headers: HashMap<_, _> = lines + .filter_map(|line| line.split_once(':')) + .map(|(name, value)| (name.to_ascii_lowercase(), value.trim().to_string())) + .collect(); + let length: usize = headers + .get("content-length") + .map(|s| s.parse().unwrap()) + .unwrap_or(0); + while bytes.len() < header_end + length { + let mut buf = [0; 4096]; + let n = socket.read(&mut buf).await.unwrap(); + assert!(n > 0); + bytes.extend_from_slice(&buf[..n]); + } + let request_body = if length == 0 { + Value::Null + } else { + serde_json::from_slice(&bytes[header_end..header_end + length]).unwrap() + }; + requests.push(Request { + line, + headers, + body: request_body, + }); + let body = if status == 204 { + String::new() + } else { + body.to_string() + }; + socket.write_all(format!("HTTP/1.1 {status} Response\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len()).as_bytes()).await.unwrap(); + } + requests + }); + (endpoint, task) + } + + #[derive(Debug)] + struct AuthProvider; + + #[async_trait] + impl HeaderProvider for AuthProvider { + async fn get_headers(&self) -> Result> { + Ok(HashMap::from([ + ("Authorization".into(), "Bearer refreshed".into()), + ("X-LanceDB-Database".into(), "wrong-dynamic".into()), + ("X-LanceDB-Database-Prefix".into(), "wrong-prefix".into()), + ])) + } + } + + #[tokio::test] + async fn catalog_routes_root_and_independent_database_scopes() { + let (endpoint, task) = server(vec![ + ( + 200, + json!({"namespaces": ["team/search"], "page_token": "next"}), + ), + (204, Value::Null), + (200, json!({"namespaces": []})), + (200, json!({})), + (200, json!({"namespaces": []})), + (200, json!({"namespaces": []})), + (200, json!({"namespaces": [], "page_token": ""})), + (204, Value::Null), + ]) + .await; + let mut options = RemoteCatalogOptions { + api_key: Some("test-key".into()), + ..Default::default() + }; + options.client_config.extra_headers = HashMap::from([ + ("x-lancedb-database".into(), "wrong-static".into()), + ( + "x-lancedb-database-prefix".into(), + "wrong-static-prefix".into(), + ), + ]); + options.client_config.header_provider = Some(Arc::new(AuthProvider)); + let catalog = CatalogConnection::new(Arc::new( + RemoteCatalog::try_new(&endpoint, options).unwrap(), + )); + let page = catalog + .list_databases(ListDatabasesRequest::default().limit(1).page_token("a/b")) + .await + .unwrap(); + assert_eq!(page.databases, ["team/search"]); + assert_eq!(page.page_token.as_deref(), Some("next")); + let first = catalog + .create_database(CreateDatabaseRequest::new("team/search").exist_ok(true)) + .await + .unwrap(); + first + .database() + .list_namespaces(ListNamespacesRequest::default()) + .await + .unwrap(); + let second = catalog.connect_database("other").await.unwrap(); + second + .database() + .list_namespaces(ListNamespacesRequest::default()) + .await + .unwrap(); + first + .database() + .list_namespaces(ListNamespacesRequest::default()) + .await + .unwrap(); + assert!( + catalog + .list_databases(ListDatabasesRequest::default()) + .await + .unwrap() + .page_token + .is_none() + ); + catalog + .drop_database(DropDatabaseRequest::new("team/search").ignore_missing(true)) + .await + .unwrap(); + let requests = task.await.unwrap(); + for (i, request) in requests.iter().enumerate() { + let database = match i { + 2 | 5 => Some("team/search"), + 4 => Some("other"), + _ => None, + }; + assert_eq!( + request + .headers + .get("x-lancedb-database") + .map(String::as_str), + database + ); + assert!(!request.headers.contains_key("x-lancedb-database-prefix")); + assert_eq!(request.headers["authorization"], "Bearer refreshed"); + } + assert_eq!( + requests[0].line, + "GET /v1/namespace/%24/list?limit=1&page_token=a%2Fb HTTP/1.1" + ); + assert_eq!( + requests[1].line, + "POST /v1/namespace/team%2Fsearch/create HTTP/1.1" + ); + assert_eq!(requests[1].body, json!({"mode": "ExistOk"})); + assert_eq!( + requests[3].line, + "POST /v1/namespace/other/describe HTTP/1.1" + ); + assert_eq!( + requests[7].body, + json!({"mode": "Skip", "behavior": "Restrict"}) + ); + } + + #[tokio::test] + async fn catalog_preserves_errors_and_never_cascades() { + let (endpoint, task) = server(vec![ + (404, json!({"error": "missing"})), + (409, json!({"error": "exists"})), + (400, json!({"error": "not empty"})), + (404, json!({"error": "missing"})), + (404, json!({"error": "missing"})), + (401, json!({"error": "unauthorized"})), + ]) + .await; + let catalog = RemoteCatalog::try_new(endpoint, RemoteCatalogOptions::default()).unwrap(); + assert!(matches!( + catalog.connect_database("missing").await, + Err(Error::DatabaseNotFound { .. }) + )); + assert!(matches!( + catalog.create_database("exists".into()).await, + Err(Error::DatabaseAlreadyExists { .. }) + )); + assert!(matches!( + catalog.drop_database("full".into()).await, + Err(Error::Http { + status_code: Some(StatusCode::BAD_REQUEST), + .. + }) + )); + assert!(matches!( + catalog.drop_database("missing".into()).await, + Err(Error::DatabaseNotFound { .. }) + )); + catalog + .drop_database(DropDatabaseRequest::new("missing").ignore_missing(true)) + .await + .unwrap(); + assert!( + catalog + .list_databases(ListDatabasesRequest::default()) + .await + .is_err() + ); + let requests = task.await.unwrap(); + assert_eq!(requests[1].body, json!({"mode": "Create"})); + assert_eq!( + requests[2].body, + json!({"mode": "Fail", "behavior": "Restrict"}) + ); + } + + #[tokio::test] + async fn catalog_validates_before_sending_requests() { + for endpoint in [ + "/tmp/catalog", + "s3://bucket", + "db://database", + "https://user:pass@host", + "https://host?q=1", + "https://host#fragment", + ] { + assert!(RemoteCatalog::try_new(endpoint, RemoteCatalogOptions::default()).is_err()); + } + let catalog = + RemoteCatalog::try_new("http://127.0.0.1:1", RemoteCatalogOptions::default()).unwrap(); + for name in ["", "a$b", "\r\ninjected", "..", "café", " padded "] { + assert!(matches!( + catalog.create_database(name.into()).await, + Err(Error::InvalidInput { .. }) + )); + assert!(matches!( + catalog.connect_database(name).await, + Err(Error::InvalidInput { .. }) + )); + assert!(matches!( + catalog.drop_database(name.into()).await, + Err(Error::InvalidInput { .. }) + )); + } + for limit in [0, u32::MAX] { + assert!(matches!( + catalog + .list_databases(ListDatabasesRequest::default().limit(limit)) + .await, + Err(Error::InvalidInput { .. }) + )); + } + } + + #[test] + fn catalog_debug_redacts_credentials() { + let mut options = RemoteCatalogOptions { + api_key: Some("catalog-secret-key".into()), + ..Default::default() + }; + options + .client_config + .extra_headers + .insert("authorization".into(), "Bearer catalog-secret-token".into()); + let catalog = RemoteCatalog::try_new("https://catalog.example", options).unwrap(); + let catalog_debug = format!("{catalog:?}"); + let connection = CatalogConnection::new(Arc::new(catalog)); + for debug in [catalog_debug, format!("{connection:?}")] { + assert!(debug.contains("https://catalog.example")); + assert!(!debug.contains("catalog-secret-key")); + assert!(!debug.contains("catalog-secret-token")); + } + } +} diff --git a/rust/lancedb/src/remote/client.rs b/rust/lancedb/src/remote/client.rs index 6e4764ad6..55d272b51 100644 --- a/rust/lancedb/src/remote/client.rs +++ b/rust/lancedb/src/remote/client.rs @@ -379,7 +379,11 @@ pub fn parse_db_url(db_url: &str) -> Result { message: format!("Invalid database URL (missing host) '{}'", db_url), }); } - let db_name = parsed_url.host_str().unwrap().to_string(); + let db_name = urlencoding::decode(parsed_url.host_str().unwrap()) + .map_err(|err| Error::InvalidInput { + message: format!("Invalid encoded database name: {err}"), + })? + .into_owned(); let db_prefix = { let prefix = parsed_url.path().trim_start_matches('/'); if prefix.is_empty() { @@ -1098,6 +1102,16 @@ mod tests { ENV_MUTEX.lock().unwrap_or_else(|e| e.into_inner()) } + #[test] + fn test_parse_catalog_database_uri() { + let parsed = parse_db_url("db://team%2Fsearch").unwrap(); + assert_eq!(parsed.db_name, "team/search"); + assert!(parsed.db_prefix.is_none()); + let parsed = parse_db_url("db://db/prefix").unwrap(); + assert_eq!(parsed.db_name, "db"); + assert_eq!(parsed.db_prefix.as_deref(), Some("prefix")); + } + #[test] fn test_timeout_config_default() { let config = TimeoutConfig::default(); diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index c24ede4e1..e7ae78108 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -289,6 +289,66 @@ impl RemoteDatabase { read_consistency_interval: Option, ) -> Result { let parsed = super::client::parse_db_url(uri)?; + Self::try_new_with_identity( + uri, + api_key, + region, + host_overrides, + client_config, + options, + read_consistency_interval, + parsed, + ) + } + + pub(crate) fn for_catalog( + endpoint: &str, + name: Option<&str>, + options: &super::catalog::RemoteCatalogOptions, + ) -> Result { + let scope = super::catalog::ScopedHeaderProvider { + provider: options.client_config.header_provider.clone(), + database: name.map(str::to_string), + }; + let mut config = options.client_config.clone(); + scope.apply(&mut config.extra_headers); + config.header_provider = Some(Arc::new(scope)); + let uri = name + .map(|name| format!("db://{}", urlencoding::encode(name))) + .unwrap_or_else(|| endpoint.to_string()); + let mut db = Self::try_new_with_identity( + &uri, + options.api_key.as_deref().unwrap_or(""), + "us-east-1", + RemoteHostOverrides { + rest: Some(endpoint.to_string()), + sql: options.sql_host_override.clone(), + }, + config, + RemoteOptions::default(), + options.read_consistency_interval, + super::client::ParsedDbUrl { + db_name: name.unwrap_or("").to_string(), + db_prefix: None, + }, + )?; + if name.is_none() { + db.sql_client = None; + } + Ok(db) + } + + #[allow(clippy::too_many_arguments)] + fn try_new_with_identity( + uri: &str, + api_key: &str, + region: &str, + host_overrides: RemoteHostOverrides, + client_config: ClientConfig, + options: RemoteOptions, + read_consistency_interval: Option, + parsed: super::client::ParsedDbUrl, + ) -> Result { let sql_client = SqlClient::new( parsed.db_name.clone(), parsed.db_prefix.clone(), @@ -301,7 +361,7 @@ impl RemoteDatabase { api_key, region, &parsed.db_name, - host_overrides.rest.is_some(), + host_overrides.rest.is_some() && !parsed.db_name.is_empty(), &options, parsed.db_prefix.as_deref(), &client_config, @@ -1220,6 +1280,7 @@ impl Database for RemoteDatabase { ) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); let namespace_id = build_namespace_identifier(namespace_parts, &self.client.id_delimiter); + let namespace_id = urlencoding::encode(&namespace_id); let mut req = self .client .get(&format!("/v1/namespace/{}/list", namespace_id)); @@ -1242,6 +1303,7 @@ impl Database for RemoteDatabase { ) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); let namespace_id = build_namespace_identifier(namespace_parts, &self.client.id_delimiter); + let namespace_id = urlencoding::encode(&namespace_id); let mut req = self .client .post(&format!("/v1/namespace/{}/create", namespace_id)); @@ -1256,7 +1318,7 @@ impl Database for RemoteDatabase { } let body = CreateNamespaceRequestBody { - mode: request.mode.as_ref().map(|m| format!("{:?}", m)), + mode: request.mode, properties: request.properties, }; @@ -1264,12 +1326,16 @@ impl Database for RemoteDatabase { let (request_id, resp) = self.client.send(req).await?; let resp = self.client.check_response(&request_id, resp).await?; + if resp.status() == StatusCode::NO_CONTENT { + return Ok(CreateNamespaceResponse::default()); + } resp.json().await.err_to_http(request_id) } async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); let namespace_id = build_namespace_identifier(namespace_parts, &self.client.id_delimiter); + let namespace_id = urlencoding::encode(&namespace_id); let mut req = self .client .post(&format!("/v1/namespace/{}/drop", namespace_id)); @@ -1284,14 +1350,17 @@ impl Database for RemoteDatabase { } let body = DropNamespaceRequestBody { - mode: request.mode.as_ref().map(|m| format!("{:?}", m)), - behavior: request.behavior.as_ref().map(|b| format!("{:?}", b)), + mode: request.mode, + behavior: request.behavior, }; req = req.json(&body); let (request_id, resp) = self.client.send(req).await?; let resp = self.client.check_response(&request_id, resp).await?; + if resp.status() == StatusCode::NO_CONTENT { + return Ok(DropNamespaceResponse::default()); + } resp.json().await.err_to_http(request_id) } @@ -1301,6 +1370,7 @@ impl Database for RemoteDatabase { ) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); let namespace_id = build_namespace_identifier(namespace_parts, &self.client.id_delimiter); + let namespace_id = urlencoding::encode(&namespace_id); let req = self .client .post(&format!("/v1/namespace/{}/describe", namespace_id)) diff --git a/rust/lancedb/src/remote/sql_test.rs b/rust/lancedb/src/remote/sql_test.rs index a05a11259..d20ba495f 100644 --- a/rust/lancedb/src/remote/sql_test.rs +++ b/rust/lancedb/src/remote/sql_test.rs @@ -18,7 +18,10 @@ use futures::{StreamExt, TryStreamExt}; use tonic::{Request, Response, Status, Streaming}; use super::*; +use crate::database::Database; +use crate::remote::RemoteCatalogOptions; use crate::remote::client::HeaderProvider; +use crate::remote::db::RemoteDatabase; #[derive(Debug, Default)] struct DelayedHeaderProvider { @@ -173,7 +176,10 @@ impl FlightService for TestSqlService { namespace_path: header("namespace-path"), request_id: header("x-request-id"), api_key: header("x-api-key"), - database_prefix: header("x-lancedb-database-prefix"), + database_prefix: metadata + .get("x-lancedb-database-prefix") + .map(|value| value.to_str().unwrap().to_string()) + .unwrap_or_default(), }); let command = Any::decode(request.get_ref().cmd.as_ref()) @@ -386,6 +392,60 @@ impl FlightService for TestSqlService { } } +#[tokio::test] +async fn catalog_connections_use_explicit_sql_endpoint_and_database_scope() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let incoming = futures::stream::try_unfold(listener, |listener| async { + let (socket, _) = listener.accept().await?; + Ok::<_, std::io::Error>(Some((socket, listener))) + }); + let service = TestSqlService::default(); + let headers = service.headers.clone(); + let expected = service.result.clone(); + let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel(); + let server = tokio::spawn( + tonic::transport::Server::builder() + .add_service(FlightServiceServer::new(service)) + .serve_with_incoming_shutdown(incoming, async { + let _ = shutdown_rx.await; + }), + ); + let options = RemoteCatalogOptions { + api_key: Some("catalog-key".into()), + sql_host_override: Some(format!("grpc://{address}")), + ..Default::default() + }; + for name in ["analytics", "team/search"] { + let database = + RemoteDatabase::for_catalog("https://catalog.example", Some(name), &options).unwrap(); + let query = database + .execute_query_async("SELECT 42", &[]) + .await + .unwrap(); + assert_eq!( + collect_result(&query).await.unwrap(), + vec![expected.clone()] + ); + } + { + let headers = headers.lock().unwrap(); + assert!(headers.iter().any(|header| header.database == "analytics")); + assert!( + headers + .iter() + .any(|header| header.database == "team/search") + ); + for header in headers.iter() { + assert_eq!(header.api_key, "catalog-key"); + assert_eq!(header.namespace_path, "public"); + assert!(header.database_prefix.is_empty()); + } + } + shutdown_tx.send(()).unwrap(); + server.await.unwrap().unwrap(); +} + #[tokio::test] async fn submits_polls_fetches_cancels_and_reuses_client() { let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); From a4f66afc6998c9cb557609c36848f91e3631399b Mon Sep 17 00:00:00 2001 From: Bruno Ramirez Date: Wed, 16 Sep 2026 12:54:22 -0600 Subject: [PATCH 30/43] feat(rust): add zonemap index builder (#4199) Lance supports ZoneMap scalar indexes, but the LanceDB Rust API did not expose a first-class way to request one through `Table::create_index`. Users had builders for the other scalar index families, while ZoneMap was missing from the public `Index` model and remote create-index serialization. This PR adds ZoneMap as a supported scalar index option in LanceDB. This was accomplished with the following changes: - Added `ZoneMapIndexBuilder` in `rust/lancedb/src/index/scalar.rs`. - Added `Index::ZoneMap` and `IndexType::ZoneMap`, including display/from-string aliases for `ZONEMAP` and `ZONE_MAP`. - Mapped local index creation to `ScalarIndexParams::for_builtin(BuiltinIndexType::ZoneMap)` and Lance `IndexType::ZoneMap` in `rust/lancedb/src/table/create_index.rs`. - Serialized remote create-index requests as `index_type: "ZONEMAP"` in `rust/lancedb/src/remote/table.rs`. - Added coverage for both local ZoneMap index creation and remote request serialization. Example: ```rust table .create_index(&["my_column"], Index::ZoneMap(Default::default())) .execute() .await?; ``` ### Testing Added `test_create_zonemap_index` for local index creation and extended the remote request body test matrix for `ZONEMAP`. --- rust/lancedb/src/index.rs | 15 ++++++- rust/lancedb/src/index/scalar.rs | 22 ++++++++++ rust/lancedb/src/remote/table.rs | 2 + rust/lancedb/src/table/create_index.rs | 57 +++++++++++++++++++++++++- 4 files changed, 94 insertions(+), 2 deletions(-) diff --git a/rust/lancedb/src/index.rs b/rust/lancedb/src/index.rs index c693dc056..0611f95b1 100644 --- a/rust/lancedb/src/index.rs +++ b/rust/lancedb/src/index.rs @@ -13,7 +13,10 @@ use crate::index::vector::IvfRqIndexBuilder; use crate::{DistanceType, Error, Result, job::Job, table::BaseTable}; use self::{ - scalar::{BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder, LabelListIndexBuilder}, + scalar::{ + BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder, LabelListIndexBuilder, + ZoneMapIndexBuilder, + }, vector::{ IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder, IvfSqIndexBuilder, @@ -54,6 +57,12 @@ pub enum Index { /// substrings of the raw bytes, unlike the tokenized [`Index::FTS`] index. Fm(FmIndexBuilder), + /// A `ZoneMap` index stores min/max summaries for ranges of rows. + /// + /// It can accelerate range filters by skipping zones whose min/max values + /// prove they cannot match the predicate. + ZoneMap(ZoneMapIndexBuilder), + /// Full text search index using BM25. /// /// The posting block size defaults to 128. Supported values are 128 and 256; @@ -341,6 +350,8 @@ pub enum IndexType { LabelList, #[serde(alias = "FM", alias = "FMINDEX", alias = "FMIndex")] Fm, + #[serde(alias = "ZONEMAP", alias = "ZONE_MAP")] + ZoneMap, // FTS #[serde(alias = "INVERTED", alias = "Inverted")] FTS, @@ -362,6 +373,7 @@ impl std::fmt::Display for IndexType { Self::Bitmap => write!(f, "BITMAP"), Self::LabelList => write!(f, "LABEL_LIST"), Self::Fm => write!(f, "FM"), + Self::ZoneMap => write!(f, "ZONEMAP"), Self::FTS => write!(f, "FTS"), Self::Unknown => write!(f, "UNKNOWN"), } @@ -377,6 +389,7 @@ impl std::str::FromStr for IndexType { "BITMAP" => Ok(Self::Bitmap), "LABEL_LIST" | "LABELLIST" => Ok(Self::LabelList), "FM" | "FMINDEX" => Ok(Self::Fm), + "ZONEMAP" | "ZONE_MAP" => Ok(Self::ZoneMap), "FTS" | "INVERTED" => Ok(Self::FTS), "IVF_FLAT" => Ok(Self::IvfFlat), "IVF_SQ" => Ok(Self::IvfSq), diff --git a/rust/lancedb/src/index/scalar.rs b/rust/lancedb/src/index/scalar.rs index dba05b776..af7407a21 100644 --- a/rust/lancedb/src/index/scalar.rs +++ b/rust/lancedb/src/index/scalar.rs @@ -60,6 +60,28 @@ pub struct LabelListIndexBuilder {} #[derive(Debug, Clone, Default, serde::Serialize)] pub struct FmIndexBuilder {} +/// Builder for a ZoneMap index. +/// +/// A ZoneMap index stores min/max summaries for ranges of rows and can +/// accelerate range predicates by pruning zones that cannot match. +/// +/// ``` +/// use lancedb::{ +/// index::{scalar::ZoneMapIndexBuilder, Index}, +/// Table, +/// }; +/// +/// # async fn create_zonemap_index(table: &Table) -> lancedb::Result<()> { +/// table +/// .create_index(&["timestamp"], Index::ZoneMap(ZoneMapIndexBuilder::default())) +/// .execute() +/// .await?; +/// # Ok(()) +/// # } +/// ``` +#[derive(Debug, Clone, Default, serde::Serialize)] +pub struct ZoneMapIndexBuilder {} + pub use lance_index::scalar::FullTextSearchQuery; pub use lance_index::scalar::InvertedIndexParams as FtsIndexBuilder; pub use lance_index::scalar::InvertedIndexParams; diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 89061c393..c1182f996 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -587,6 +587,7 @@ impl RemoteTable { Index::Bitmap(p) => ("BITMAP", Some(to_json(p)?)), Index::LabelList(p) => ("LABEL_LIST", Some(to_json(p)?)), Index::Fm(p) => ("FM", Some(to_json(p)?)), + Index::ZoneMap(p) => ("ZONEMAP", Some(to_json(p)?)), Index::FTS(p) => { let mut params = to_json(p)?; if p.get_document_granularity().is_list_element() { @@ -6376,6 +6377,7 @@ mod tests { // HNSW_PQ isn't yet supported on SaaS ("BTREE", json!({}), Index::BTree(Default::default())), ("BITMAP", json!({}), Index::Bitmap(Default::default())), + ("ZONEMAP", json!({}), Index::ZoneMap(Default::default())), ( "LABEL_LIST", json!({}), diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs index e30c310ac..2cc66b8cd 100644 --- a/rust/lancedb/src/table/create_index.rs +++ b/rust/lancedb/src/table/create_index.rs @@ -259,6 +259,12 @@ impl NativeTable { BuiltinIndexType::Fm, ))) } + Index::ZoneMap(_) => { + Self::validate_index_type(field, "ZoneMap", supported_btree_data_type)?; + Ok(Box::new(ScalarIndexParams::for_builtin( + BuiltinIndexType::ZoneMap, + ))) + } Index::FTS(fts_opts) => { Self::validate_index_type(field, "FTS", supported_fts_data_type)?; Ok(Box::new(fts_opts)) @@ -418,6 +424,7 @@ impl NativeTable { Index::Bitmap(_) => IndexType::Bitmap, Index::LabelList(_) => IndexType::LabelList, Index::Fm(_) => IndexType::Fm, + Index::ZoneMap(_) => IndexType::ZoneMap, Index::FTS(_) => IndexType::Inverted, Index::IvfFlat(_) | Index::IvfSq(_) @@ -451,7 +458,8 @@ mod tests { use crate::connection::ConnectBuilder; use crate::index::Index; use crate::index::scalar::{ - BTreeIndexBuilder, BitmapIndexBuilder, DocumentGranularity, FmIndexBuilder, FtsIndexBuilder, + BTreeIndexBuilder, BitmapIndexBuilder, DocumentGranularity, FmIndexBuilder, + FtsIndexBuilder, ZoneMapIndexBuilder, }; use crate::index::vector::{ IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, @@ -1197,6 +1205,53 @@ mod tests { assert_eq!(stats.distance_type, None); } + #[tokio::test] + async fn test_create_zonemap_index() { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!(("i", Int32, [1, 2, 3, 4, 5])).unwrap(); + let table = conn + .create_table("zonemap_table", batch) + .execute() + .await + .unwrap(); + + table + .create_index(&["i"], Index::ZoneMap(ZoneMapIndexBuilder::default())) + .execute() + .await + .unwrap(); + table + .wait_for_index(&["i_idx"], Duration::from_millis(10)) + .await + .unwrap(); + + let index_configs = table.list_indices().await.unwrap(); + assert_eq!(index_configs.len(), 1); + let index = index_configs.into_iter().next().unwrap(); + assert_eq!(index.index_type, crate::index::IndexType::ZoneMap); + assert_eq!(index.columns, vec!["i".to_string()]); + + let count = table + .query() + .only_if("i >= 2 AND i < 5") + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap() + .iter() + .map(|b| b.num_rows()) + .sum::(); + assert_eq!(count, 3); + + let stats = table.index_stats("i_idx").await.unwrap().unwrap(); + assert_eq!(stats.num_indexed_rows, 5); + assert_eq!(stats.num_unindexed_rows, 0); + assert_eq!(stats.index_type, crate::index::IndexType::ZoneMap); + assert_eq!(stats.distance_type, None); + } + #[tokio::test] async fn test_create_index_nested_field_paths() { let tmp_dir = tempdir().unwrap(); From 86835da5dbd980b9f70ce435f5cdc559b0cbea5d Mon Sep 17 00:00:00 2001 From: Lance Release Date: Wed, 16 Sep 2026 18:59:58 +0000 Subject: [PATCH 31/43] =?UTF-8?q?Bump=20version:=200.39.0-beta.10=20?= =?UTF-8?q?=E2=86=92=200.40.0-beta.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 87b0f1f7c..8a9eb735d 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.39.0-beta.10" +current_version = "0.40.0-beta.0" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 4142b4cf1..36878f1a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5463,7 +5463,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.39.0-beta.10" +version = "0.40.0-beta.0" dependencies = [ "ahash", "anyhow", @@ -5560,7 +5560,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.39.0-beta.10" +version = "0.40.0-beta.0" dependencies = [ "arrow-array", "arrow-buffer", @@ -5585,7 +5585,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.39.0-beta.10" +version = "0.40.0-beta.0" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index f08277fff..a52dbbd31 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.39.0-beta.10 + 0.40.0-beta.0 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index def010a76..dc280b499 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.39.0-beta.10 + 0.40.0-beta.0 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index ccdbf285b..95dcb6850 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.39.0-beta.10 + 0.40.0-beta.0 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 3a37a9f5d..76455a0e1 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.39.0-beta.10" +version = "0.40.0-beta.0" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index 40562d1b1..bf6122591 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.39.0-beta.10", + "version": "0.40.0-beta.0", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index aec5307b0..c9bb6aeaa 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.39.0-beta.10", + "version": "0.40.0-beta.0", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 5fade6079..38b05288b 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.39.0-beta.10", + "version": "0.40.0-beta.0", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index ee3ff460a..23ca47f1a 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.39.0-beta.10", + "version": "0.40.0-beta.0", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 64e9471e9..de5879031 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.39.0-beta.10", + "version": "0.40.0-beta.0", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 3f0123a45..8474e8c23 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.39.0-beta.10", + "version": "0.40.0-beta.0", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index e3d4ab7eb..e735f97c2 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.39.0-beta.10", + "version": "0.40.0-beta.0", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index 0de36ad95..0f3551ba2 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.39.0-beta.10", + "version": "0.40.0-beta.0", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index d9dc7f1f8..9937c2d7e 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.39.0-beta.10" +version = "0.40.0-beta.0" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index fd7803d4a..da8253cb3 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.39.0-beta.10" +version = "0.40.0-beta.0" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 32871e97a7fb046dcb8104e4d6b242eb3968c280 Mon Sep 17 00:00:00 2001 From: Bruno Ramirez Date: Wed, 16 Sep 2026 14:10:19 -0600 Subject: [PATCH 32/43] fix: widen zonemap index type support (#4206) ZoneMap indexes were added to the Rust API in #4199, but LanceDB reused the BTree type validation when creating them. That made the public builder reject some types that Lance ZoneMap can support, including `LargeUtf8`, `Binary`, and `LargeBinary`. This PR gives ZoneMap its own validation helper so it can accept the broader scalar set while keeping the rest of the create-index path unchanged. --- rust/lancedb/src/table/create_index.rs | 73 +++++++++++++++++++++++++- rust/lancedb/src/utils/mod.rs | 8 +++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs index 2cc66b8cd..a5fb59b0b 100644 --- a/rust/lancedb/src/table/create_index.rs +++ b/rust/lancedb/src/table/create_index.rs @@ -30,7 +30,7 @@ use crate::index::vector::{VectorIndex, suggested_num_sub_vectors}; use crate::utils::{ resolve_lance_fts_field_path, supported_bitmap_data_type, supported_btree_data_type, supported_fm_data_type, supported_fts_data_type, supported_label_list_data_type, - supported_vector_data_type, + supported_vector_data_type, supported_zonemap_data_type, }; use super::NativeTable; @@ -260,7 +260,7 @@ impl NativeTable { ))) } Index::ZoneMap(_) => { - Self::validate_index_type(field, "ZoneMap", supported_btree_data_type)?; + Self::validate_index_type(field, "ZoneMap", supported_zonemap_data_type)?; Ok(Box::new(ScalarIndexParams::for_builtin( BuiltinIndexType::ZoneMap, ))) @@ -1252,6 +1252,75 @@ mod tests { assert_eq!(stats.distance_type, None); } + #[tokio::test] + async fn test_create_zonemap_index_on_wider_scalar_types() { + let conn = connect("memory://").execute().await.unwrap(); + let schema = Arc::new(Schema::new(vec![ + Field::new("large_text", DataType::LargeUtf8, true), + Field::new("binary", DataType::Binary, true), + Field::new("large_binary", DataType::LargeBinary, true), + ])); + let batch = RecordBatch::try_new( + schema, + vec![ + Arc::new(LargeStringArray::from(vec![ + Some("alpha"), + None, + Some("omega"), + ])) as ArrayRef, + Arc::new(BinaryArray::from(vec![ + Some(b"aa".as_slice()), + None, + Some(b"zz".as_slice()), + ])) as ArrayRef, + Arc::new(LargeBinaryArray::from(vec![ + Some(b"left".as_slice()), + None, + Some(b"right".as_slice()), + ])) as ArrayRef, + ], + ) + .unwrap(); + let table = conn + .create_table("zonemap_wider_scalar_table", batch) + .execute() + .await + .unwrap(); + + for column in ["large_text", "binary", "large_binary"] { + table + .create_index(&[column], Index::ZoneMap(ZoneMapIndexBuilder::default())) + .execute() + .await + .unwrap(); + let index_name = format!("{column}_idx"); + table + .wait_for_index(&[&index_name], Duration::from_millis(10)) + .await + .unwrap(); + } + + let index_configs = table.list_indices().await.unwrap(); + assert_eq!(index_configs.len(), 3); + for index in index_configs { + assert_eq!(index.index_type, crate::index::IndexType::ZoneMap); + } + + let null_count = table + .query() + .only_if("large_text IS NULL") + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap() + .iter() + .map(|b| b.num_rows()) + .sum::(); + assert_eq!(null_count, 1); + } + #[tokio::test] async fn test_create_index_nested_field_paths() { let tmp_dir = tempdir().unwrap(); diff --git a/rust/lancedb/src/utils/mod.rs b/rust/lancedb/src/utils/mod.rs index 1bfd2d042..a66e758f7 100644 --- a/rust/lancedb/src/utils/mod.rs +++ b/rust/lancedb/src/utils/mod.rs @@ -406,6 +406,14 @@ pub fn supported_btree_data_type(dtype: &DataType) -> bool { ) } +pub fn supported_zonemap_data_type(dtype: &DataType) -> bool { + supported_btree_data_type(dtype) + || matches!( + dtype, + DataType::LargeUtf8 | DataType::Binary | DataType::LargeBinary + ) +} + pub fn supported_bitmap_data_type(dtype: &DataType) -> bool { dtype.is_integer() || matches!( From 3309c71a6ebc674f34e178aaf7e00070007d766a Mon Sep 17 00:00:00 2001 From: Lance Release Date: Wed, 16 Sep 2026 20:15:06 +0000 Subject: [PATCH 33/43] =?UTF-8?q?Bump=20version:=200.40.0-beta.0=20?= =?UTF-8?q?=E2=86=92=200.40.0-beta.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 8a9eb735d..c28f65ec1 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.40.0-beta.0" +current_version = "0.40.0-beta.1" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 36878f1a4..598f8e8a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5463,7 +5463,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.40.0-beta.0" +version = "0.40.0-beta.1" dependencies = [ "ahash", "anyhow", @@ -5560,7 +5560,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.40.0-beta.0" +version = "0.40.0-beta.1" dependencies = [ "arrow-array", "arrow-buffer", @@ -5585,7 +5585,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.40.0-beta.0" +version = "0.40.0-beta.1" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index a52dbbd31..4a798c770 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.40.0-beta.0 + 0.40.0-beta.1 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index dc280b499..0981f4bb2 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.0 + 0.40.0-beta.1 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 95dcb6850..b15c9dfc1 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.0 + 0.40.0-beta.1 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 76455a0e1..6962c5802 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.40.0-beta.0" +version = "0.40.0-beta.1" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index bf6122591..b98d8ec41 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.40.0-beta.0", + "version": "0.40.0-beta.1", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index c9bb6aeaa..be28e90a8 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.40.0-beta.0", + "version": "0.40.0-beta.1", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 38b05288b..cc8ff6163 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.40.0-beta.0", + "version": "0.40.0-beta.1", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 23ca47f1a..1a5992ec9 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.40.0-beta.0", + "version": "0.40.0-beta.1", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index de5879031..e838efc17 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.40.0-beta.0", + "version": "0.40.0-beta.1", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 8474e8c23..e0543a3a6 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.40.0-beta.0", + "version": "0.40.0-beta.1", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index e735f97c2..93d520269 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.40.0-beta.0", + "version": "0.40.0-beta.1", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index 0f3551ba2..3e8725c35 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.40.0-beta.0", + "version": "0.40.0-beta.1", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 9937c2d7e..705eb4a0a 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.40.0-beta.0" +version = "0.40.0-beta.1" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index da8253cb3..899600bf9 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.40.0-beta.0" +version = "0.40.0-beta.1" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From 04150b3b826c54d6469abaa6bc07cde324058e30 Mon Sep 17 00:00:00 2001 From: Colin Patrick McCabe Date: Wed, 16 Sep 2026 15:03:06 -0700 Subject: [PATCH 34/43] feat: index builders for NGram, BloomFilter, RTree (#4205) Add index builders in the lancedb library for NGram, BloomFilter, and RTree indexes. --- Cargo.lock | 297 +++++++++++++++++++++++++ rust/lancedb/Cargo.toml | 2 + rust/lancedb/src/index.rs | 57 ++++- rust/lancedb/src/index/scalar.rs | 170 ++++++++++++++ rust/lancedb/src/remote/table.rs | 30 +++ rust/lancedb/src/table/create_index.rs | 209 +++++++++++++++++ 6 files changed, 763 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 598f8e8a2..7037168c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3151,6 +3151,16 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d926b4d407d372f141f93bb444696142c29d32962ccbd3531117cf3aa0bfa9" +[[package]] +name = "earcutr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79127ed59a85d7687c409e9978547cffb7dc79675355ed22da6b66fd5f6ead01" +dependencies = [ + "itertools 0.11.0", + "num-traits", +] + [[package]] name = "ecdsa" version = "0.14.8" @@ -3450,6 +3460,12 @@ dependencies = [ "rand_distr 0.5.1", ] +[[package]] +name = "float_next_after" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf7cc16383c4b8d58b9905a8509f02926ce3058053c056376248d958c9df1e8" + [[package]] name = "fnv" version = "1.0.7" @@ -3776,6 +3792,129 @@ dependencies = [ "version_check", ] +[[package]] +name = "geo" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc1a1678e54befc9b4bcab6cd43b8e7f834ae8ea121118b0fd8c42747675b4a" +dependencies = [ + "earcutr", + "float_next_after", + "geo-types", + "geographiclib-rs", + "i_overlay", + "log", + "num-traits", + "robust", + "rstar", + "spade", +] + +[[package]] +name = "geo-traits" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e7c353d12a704ccfab1ba8bfb1a7fe6cb18b665bf89d37f4f7890edcd260206" +dependencies = [ + "geo-types", +] + +[[package]] +name = "geo-types" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94776032c45f950d30a13af6113c2ad5625316c9abfbccee4dd5a6695f8fe0f5" +dependencies = [ + "approx", + "num-traits", + "rayon", + "rstar", + "serde", +] + +[[package]] +name = "geoarrow-array" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dafe7b7de3fab1a8b7099fd6a6434ca955fa65065f9c19f0f8a133693f3c2b0e" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-schema", + "geo-traits", + "geoarrow-schema", + "num-traits", + "wkb", + "wkt", +] + +[[package]] +name = "geoarrow-expr-geo" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e4a62ac19c86827c6ec81ea584594b3ee96db5a8119b9774d3466c6b373c434" +dependencies = [ + "arrow-array", + "arrow-buffer", + "geo", + "geo-traits", + "geoarrow-array", + "geoarrow-schema", +] + +[[package]] +name = "geoarrow-schema" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d4a7edb2a1d87024a93805332a9c8184a0354836271d42c0d18cf628a5e3cd0" +dependencies = [ + "arrow-schema", + "geo-traits", + "serde", + "serde_json", + "thiserror 1.0.69", +] + +[[package]] +name = "geodatafusion" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecbdd00d0fff2b04635c1b1e4129c217908f0c2d17539e0a2275308afce2552" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-schema", + "datafusion", + "geo", + "geo-traits", + "geoarrow-array", + "geoarrow-expr-geo", + "geoarrow-schema", + "geohash", + "thiserror 1.0.69", + "wkt", +] + +[[package]] +name = "geographiclib-rs" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5a7f08910fd98737a6eda7568e7c5e645093e073328eeef49758cfe8b0489c7" +dependencies = [ + "libm", +] + +[[package]] +name = "geohash" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f58890382f70caccc5fa388981f7ac80c913795042afce9f3e065695d8f7464" +dependencies = [ + "geo-types", + "libm", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -3954,6 +4093,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -4010,6 +4158,16 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0049b265b7f201ca9ab25475b22b47fe444060126a51abe00f77d986fc5cc52e" +[[package]] +name = "heapless" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bfb9eb618601c89945a70e254898da93b13be0388091d42117462b265bb3fad" +dependencies = [ + "hash32", + "stable_deref_trait", +] + [[package]] name = "heck" version = "0.4.1" @@ -4327,6 +4485,49 @@ dependencies = [ "serde", ] +[[package]] +name = "i_float" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "010025c2c532c8d82e42d0b8bb5184afa449fa6f06c709ea9adcb16c49ae405b" +dependencies = [ + "libm", +] + +[[package]] +name = "i_key_sort" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9190f86706ca38ac8add223b2aed8b1330002b5cdbbce28fb58b10914d38fc27" + +[[package]] +name = "i_overlay" +version = "4.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413183068e6e0289e18d7d0a1f661b81546e6918d5453a44570b9ab30cbed1b3" +dependencies = [ + "i_float", + "i_key_sort", + "i_shape", + "i_tree", + "rayon", +] + +[[package]] +name = "i_shape" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ea154b742f7d43dae2897fcd5ead86bc7b5eefcedd305a7ebf9f69d44d61082" +dependencies = [ + "i_float", +] + +[[package]] +name = "i_tree" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e6d558e6d4c7b82bc51d9c771e7a927862a161a7d87bf2b0541450e0e20915" + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -5061,6 +5262,7 @@ dependencies = [ "jsonb", "lance-arrow", "lance-core", + "lance-geo", "log", "pin-project", "prost", @@ -5164,6 +5366,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "lance-geo" +version = "13.0.0-beta.3" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +dependencies = [ + "datafusion", + "geo-traits", + "geo-types", + "geoarrow-array", + "geoarrow-schema", + "geodatafusion", + "lance-core", + "serde", +] + [[package]] name = "lance-index" version = "13.0.0-beta.3" @@ -5190,6 +5407,9 @@ dependencies = [ "dirs", "fst", "futures", + "geo-types", + "geoarrow-array", + "geoarrow-schema", "half", "itertools 0.14.0", "jieba-rs", @@ -5201,6 +5421,7 @@ dependencies = [ "lance-datafusion", "lance-encoding", "lance-file", + "lance-geo", "lance-index-core", "lance-io", "lance-linalg", @@ -6414,6 +6635,28 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26" +dependencies = [ + "num_enum_derive", + "rustversion", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "number_prefix" version = "0.4.0" @@ -8771,6 +9014,12 @@ dependencies = [ "byteorder", ] +[[package]] +name = "robust" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e27ee8bb91ca0adcf0ecb116293afa12d393f9c2b9b9cd54d33e8078fe19839" + [[package]] name = "rsa" version = "0.9.10" @@ -8792,6 +9041,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rstar" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "421400d13ccfd26dfa5858199c30a5d76f9c54e0dba7575273025b43c5175dbb" +dependencies = [ + "heapless", + "num-traits", + "smallvec", +] + [[package]] name = "rstest" version = "0.23.0" @@ -9530,6 +9790,18 @@ dependencies = [ "winapi", ] +[[package]] +name = "spade" +version = "2.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9699399fd9349b00b184f5635b074f9ec93afffef30c853f8c875b32c0f8c7fa" +dependencies = [ + "hashbrown 0.16.1", + "num-traits", + "robust", + "smallvec", +] + [[package]] name = "spin" version = "0.9.9" @@ -11353,6 +11625,31 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wkb" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a120b336c7ad17749026d50427c23d838ecb50cd64aaea6254b5030152f890a9" +dependencies = [ + "byteorder", + "geo-traits", + "num_enum", + "thiserror 1.0.69", +] + +[[package]] +name = "wkt" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efb2b923ccc882312e559ffaa832a055ba9d1ac0cc8e86b3e25453247e4b81d7" +dependencies = [ + "geo-traits", + "geo-types", + "log", + "num-traits", + "thiserror 1.0.69", +] + [[package]] name = "writeable" version = "0.6.3" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 899600bf9..bde33283d 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -128,6 +128,8 @@ pprof = { version = "0.14", features = ["flamegraph"] } [features] default = [] +# Native spatial indexes and queries. Remote RTree requests do not require this feature. +geo = ["lance/geo"] aws = [ "dep:aws-smithy-types", "lance/aws", diff --git a/rust/lancedb/src/index.rs b/rust/lancedb/src/index.rs index 0611f95b1..cdc973044 100644 --- a/rust/lancedb/src/index.rs +++ b/rust/lancedb/src/index.rs @@ -14,8 +14,8 @@ use crate::{DistanceType, Error, Result, job::Job, table::BaseTable}; use self::{ scalar::{ - BTreeIndexBuilder, BitmapIndexBuilder, FmIndexBuilder, LabelListIndexBuilder, - ZoneMapIndexBuilder, + BTreeIndexBuilder, BitmapIndexBuilder, BloomFilterIndexBuilder, FmIndexBuilder, + LabelListIndexBuilder, NGramIndexBuilder, RTreeIndexBuilder, ZoneMapIndexBuilder, }, vector::{ IvfHnswFlatIndexBuilder, IvfHnswPqIndexBuilder, IvfHnswSqIndexBuilder, IvfPqIndexBuilder, @@ -63,6 +63,16 @@ pub enum Index { /// prove they cannot match the predicate. ZoneMap(ZoneMapIndexBuilder), + /// An NGram index accelerates substring and pattern filters on UTF-8 strings. + NGram(NGramIndexBuilder), + + /// A Bloom filter index skips groups of scalar values that cannot match a filter. + BloomFilter(BloomFilterIndexBuilder), + + /// An R-tree index accelerates spatial intersection filters on GeoArrow geometries. + /// Native creation requires the `geo` feature. + RTree(RTreeIndexBuilder), + /// Full text search index using BM25. /// /// The posting block size defaults to 128. Supported values are 128 and 256; @@ -352,6 +362,12 @@ pub enum IndexType { Fm, #[serde(alias = "ZONEMAP", alias = "ZONE_MAP")] ZoneMap, + #[serde(alias = "NGRAM", alias = "N_GRAM")] + NGram, + #[serde(alias = "BLOOM_FILTER", alias = "BLOOMFILTER")] + BloomFilter, + #[serde(alias = "RTREE", alias = "R_TREE")] + RTree, // FTS #[serde(alias = "INVERTED", alias = "Inverted")] FTS, @@ -374,6 +390,9 @@ impl std::fmt::Display for IndexType { Self::LabelList => write!(f, "LABEL_LIST"), Self::Fm => write!(f, "FM"), Self::ZoneMap => write!(f, "ZONEMAP"), + Self::NGram => write!(f, "NGRAM"), + Self::BloomFilter => write!(f, "BLOOM_FILTER"), + Self::RTree => write!(f, "RTREE"), Self::FTS => write!(f, "FTS"), Self::Unknown => write!(f, "UNKNOWN"), } @@ -390,6 +409,9 @@ impl std::str::FromStr for IndexType { "LABEL_LIST" | "LABELLIST" => Ok(Self::LabelList), "FM" | "FMINDEX" => Ok(Self::Fm), "ZONEMAP" | "ZONE_MAP" => Ok(Self::ZoneMap), + "NGRAM" | "N_GRAM" => Ok(Self::NGram), + "BLOOM_FILTER" | "BLOOMFILTER" => Ok(Self::BloomFilter), + "RTREE" | "R_TREE" => Ok(Self::RTree), "FTS" | "INVERTED" => Ok(Self::FTS), "IVF_FLAT" => Ok(Self::IvfFlat), "IVF_SQ" => Ok(Self::IvfSq), @@ -495,3 +517,34 @@ pub struct IndexStatistics { /// The number of parts this index is split into. pub num_indices: Option, } + +#[cfg(test)] +mod tests { + use super::IndexType; + + #[test] + fn builtin_scalar_index_type_names() { + for (index_type, canonical, aliases) in [ + (IndexType::NGram, "NGRAM", ["NGram", "NGRAM", "N_GRAM"]), + ( + IndexType::BloomFilter, + "BLOOM_FILTER", + ["BloomFilter", "BLOOMFILTER", "BLOOM_FILTER"], + ), + (IndexType::RTree, "RTREE", ["RTree", "RTREE", "R_TREE"]), + ] { + assert_eq!(index_type.to_string(), canonical); + for alias in aliases { + assert_eq!(alias.parse::().unwrap(), index_type); + assert_eq!( + alias.to_lowercase().parse::().unwrap(), + index_type + ); + assert_eq!( + serde_json::from_value::(serde_json::json!(alias)).unwrap(), + index_type + ); + } + } + } +} diff --git a/rust/lancedb/src/index/scalar.rs b/rust/lancedb/src/index/scalar.rs index af7407a21..5bfc2b4e6 100644 --- a/rust/lancedb/src/index/scalar.rs +++ b/rust/lancedb/src/index/scalar.rs @@ -82,8 +82,178 @@ pub struct FmIndexBuilder {} #[derive(Debug, Clone, Default, serde::Serialize)] pub struct ZoneMapIndexBuilder {} +/// Builder for an NGram index over UTF-8 strings. +/// +/// This index accelerates certain substring, `LIKE`, and regular-expression filters. +/// It uses Lance's default trigram parameters. +/// +/// ``` +/// use lancedb::index::{Index, scalar::NGramIndexBuilder}; +/// # async fn example(table: &lancedb::Table) -> lancedb::Result<()> { +/// table.create_index(&["text"], Index::NGram(NGramIndexBuilder::default())) +/// .execute().await?; +/// # Ok(()) +/// # } +/// ``` +#[derive(Debug, Clone, Default, serde::Serialize)] +pub struct NGramIndexBuilder {} + +/// Builder for a Bloom filter index on scalar values. +/// +/// Bloom filters accelerate equality and membership filters by skipping groups of rows that cannot +/// match. Candidate rows are checked to remove false positives. A bloom filter is much smaller than +/// a btree or bitmap index, but not as precise. It is also limited to equality queries. +/// +/// Unset parameters use Lance's defaults. +/// +/// ``` +/// use lancedb::index::{Index, scalar::BloomFilterIndexBuilder}; +/// # async fn example(table: &lancedb::Table) -> lancedb::Result<()> { +/// let params = BloomFilterIndexBuilder::default() +/// .number_of_items(4096)? +/// .probability(0.01)?; +/// table.create_index(&["id"], Index::BloomFilter(params)) +/// .execute().await?; +/// # Ok(()) +/// # } +/// ``` +#[derive(Debug, Clone, Default, serde::Serialize)] +pub struct BloomFilterIndexBuilder { + #[serde(skip_serializing_if = "Option::is_none")] + number_of_items: Option, + #[serde(skip_serializing_if = "Option::is_none")] + probability: Option, +} + +impl BloomFilterIndexBuilder { + /// Set the number of rows covered by each Bloom filter. + /// + /// Must be greater than zero. Defaults to 8192, unless overridden by Lance's + /// `LANCE_BLOOMFILTER_DEFAULT_NUMBER_OF_ITEMS` environment variable. + pub fn number_of_items(mut self, number_of_items: u64) -> crate::Result { + if number_of_items == 0 { + return Err(crate::Error::InvalidInput { + message: "BloomFilter number_of_items must be greater than zero".into(), + }); + } + self.number_of_items = Some(number_of_items); + Ok(self) + } + + /// Set the desired false-positive probability for each Bloom filter. + /// + /// Must be finite and strictly between zero and one. Lower values use more + /// space. Defaults to 0.00057, unless overridden by Lance's + /// `LANCE_BLOOMFILTER_DEFAULT_PROBABILITY` environment variable. + pub fn probability(mut self, probability: f64) -> crate::Result { + if !probability.is_finite() || probability <= 0.0 || probability >= 1.0 { + return Err(crate::Error::InvalidInput { + message: "BloomFilter probability must be finite and strictly between zero and one" + .into(), + }); + } + self.probability = Some(probability); + Ok(self) + } +} + +/// Builder for an R-tree index on GeoArrow geometry columns. +/// +/// This index accelerates spatial intersection filters using geometry bounding +/// boxes. Unset parameters use Lance's defaults. Native creation requires +/// the `geo` feature; remote creation requires server support. +/// +/// ``` +/// use lancedb::index::{Index, scalar::RTreeIndexBuilder}; +/// # async fn example(table: &lancedb::Table) -> lancedb::Result<()> { +/// let params = RTreeIndexBuilder::default().page_size(1024)?; +/// table.create_index(&["geometry"], Index::RTree(params)) +/// .execute().await?; +/// # Ok(()) +/// # } +/// ``` +#[derive(Debug, Clone, Default, serde::Serialize)] +pub struct RTreeIndexBuilder { + #[serde(skip_serializing_if = "Option::is_none")] + page_size: Option, +} + +impl RTreeIndexBuilder { + /// Set the maximum number of entries in each R-tree page. + /// + /// Must be at least 2. Defaults to 4096. + pub fn page_size(mut self, page_size: u32) -> crate::Result { + if page_size < 2 { + return Err(crate::Error::InvalidInput { + message: "RTree page_size must be at least 2".into(), + }); + } + self.page_size = Some(page_size); + Ok(self) + } +} + pub use lance_index::scalar::FullTextSearchQuery; pub use lance_index::scalar::InvertedIndexParams as FtsIndexBuilder; pub use lance_index::scalar::InvertedIndexParams; pub use lance_index::scalar::inverted::DocumentGranularity; pub use lance_index::scalar::inverted::query::*; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn scalar_index_parameters() { + assert_eq!( + serde_json::to_value(BloomFilterIndexBuilder::default()).unwrap(), + serde_json::json!({}) + ); + assert_eq!( + serde_json::to_value(RTreeIndexBuilder::default()).unwrap(), + serde_json::json!({}) + ); + assert_eq!( + serde_json::to_value( + BloomFilterIndexBuilder::default() + .number_of_items(1) + .unwrap() + ) + .unwrap(), + serde_json::json!({"number_of_items": 1}) + ); + assert_eq!( + serde_json::to_value( + BloomFilterIndexBuilder::default() + .probability(0.01) + .unwrap() + ) + .unwrap(), + serde_json::json!({"probability": 0.01}) + ); + assert!( + BloomFilterIndexBuilder::default() + .number_of_items(0) + .is_err() + ); + for probability in [ + f64::NAN, + f64::INFINITY, + f64::NEG_INFINITY, + -0.1, + 0.0, + 1.0, + 1.1, + ] { + assert!( + BloomFilterIndexBuilder::default() + .probability(probability) + .is_err() + ); + } + for page_size in [0, 1] { + assert!(RTreeIndexBuilder::default().page_size(page_size).is_err()); + } + assert!(RTreeIndexBuilder::default().page_size(2).is_ok()); + } +} diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index c1182f996..75cf8cd00 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -588,6 +588,9 @@ impl RemoteTable { Index::LabelList(p) => ("LABEL_LIST", Some(to_json(p)?)), Index::Fm(p) => ("FM", Some(to_json(p)?)), Index::ZoneMap(p) => ("ZONEMAP", Some(to_json(p)?)), + Index::NGram(p) => ("NGRAM", Some(to_json(p)?)), + Index::BloomFilter(p) => ("BLOOM_FILTER", Some(to_json(p)?)), + Index::RTree(p) => ("RTREE", Some(to_json(p)?)), Index::FTS(p) => { let mut params = to_json(p)?; if p.get_document_granularity().is_list_element() { @@ -6378,6 +6381,33 @@ mod tests { ("BTREE", json!({}), Index::BTree(Default::default())), ("BITMAP", json!({}), Index::Bitmap(Default::default())), ("ZONEMAP", json!({}), Index::ZoneMap(Default::default())), + ("NGRAM", json!({}), Index::NGram(Default::default())), + ( + "BLOOM_FILTER", + json!({}), + Index::BloomFilter(Default::default()), + ), + ("RTREE", json!({}), Index::RTree(Default::default())), + ( + "BLOOM_FILTER", + json!({"number_of_items": 4096, "probability": 0.01}), + Index::BloomFilter( + crate::index::scalar::BloomFilterIndexBuilder::default() + .number_of_items(4096) + .unwrap() + .probability(0.01) + .unwrap(), + ), + ), + ( + "RTREE", + json!({"page_size": 1024}), + Index::RTree( + crate::index::scalar::RTreeIndexBuilder::default() + .page_size(1024) + .unwrap(), + ), + ), ( "LABEL_LIST", json!({}), diff --git a/rust/lancedb/src/table/create_index.rs b/rust/lancedb/src/table/create_index.rs index a5fb59b0b..0a0d00da1 100644 --- a/rust/lancedb/src/table/create_index.rs +++ b/rust/lancedb/src/table/create_index.rs @@ -265,6 +265,26 @@ impl NativeTable { BuiltinIndexType::ZoneMap, ))) } + Index::NGram(_) => Ok(Box::new(ScalarIndexParams::for_builtin( + BuiltinIndexType::NGram, + ))), + Index::BloomFilter(params) => { + let params = serde_json::to_value(params).map_err(|e| Error::InvalidInput { + message: format!("failed to serialize index params: {e}"), + })?; + Ok(Box::new( + ScalarIndexParams::for_builtin(BuiltinIndexType::BloomFilter) + .with_params(¶ms), + )) + } + Index::RTree(params) => { + let params = serde_json::to_value(params).map_err(|e| Error::InvalidInput { + message: format!("failed to serialize index params: {e}"), + })?; + Ok(Box::new( + ScalarIndexParams::for_builtin(BuiltinIndexType::RTree).with_params(¶ms), + )) + } Index::FTS(fts_opts) => { Self::validate_index_type(field, "FTS", supported_fts_data_type)?; Ok(Box::new(fts_opts)) @@ -425,6 +445,9 @@ impl NativeTable { Index::LabelList(_) => IndexType::LabelList, Index::Fm(_) => IndexType::Fm, Index::ZoneMap(_) => IndexType::ZoneMap, + Index::NGram(_) => IndexType::NGram, + Index::BloomFilter(_) => IndexType::BloomFilter, + Index::RTree(_) => IndexType::RTree, Index::FTS(_) => IndexType::Inverted, Index::IvfFlat(_) | Index::IvfSq(_) @@ -439,6 +462,7 @@ impl NativeTable { #[cfg(test)] mod tests { + use lance::index::DatasetIndexExt; use std::sync::Arc; use std::time::Duration; @@ -1321,6 +1345,191 @@ mod tests { assert_eq!(null_count, 1); } + #[tokio::test] + async fn test_create_builtin_scalar_index() { + for (index, index_type, predicate, invalid_column) in [ + ( + Index::NGram(Default::default()), + crate::index::IndexType::NGram, + "contains(text, 'abc')", + "id", + ), + ( + Index::BloomFilter(Default::default()), + crate::index::IndexType::BloomFilter, + "text = 'abc'", + "flag", + ), + ( + Index::BloomFilter( + crate::index::scalar::BloomFilterIndexBuilder::default() + .number_of_items(2) + .unwrap() + .probability(0.01) + .unwrap(), + ), + crate::index::IndexType::BloomFilter, + "text = 'abc'", + "flag", + ), + ] { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!( + ("id", Int32, [1, 2, 3, 4]), + ("text", Utf8, [Some("abc"), Some("xyz"), None, Some("")]), + ("flag", Boolean, [true, false, true, false]) + ) + .unwrap(); + let table = conn + .create_table("scalar", batch.clone()) + .execute() + .await + .unwrap(); + table.add(batch).execute().await.unwrap(); + + table + .create_index(&["text"], index.clone()) + .name("text_search".into()) + .train(false) + .execute() + .await + .unwrap(); + assert_eq!( + table.list_indices().await.unwrap()[0].index_type, + index_type + ); + assert!( + table + .create_index(&["text"], index.clone()) + .name("text_search".into()) + .replace(false) + .execute() + .await + .is_err() + ); + table + .create_index(&["text"], index.clone()) + .name("text_search".into()) + .execute() + .await + .unwrap(); + + let stats = table.index_stats("text_search").await.unwrap().unwrap(); + assert_eq!(stats.index_type, index_type); + assert_eq!(stats.num_indexed_rows, 8); + if let Index::BloomFilter(params) = &index { + let expected = serde_json::to_value(params).unwrap(); + let dataset = table.as_native().unwrap().dataset.get().await.unwrap(); + let stats: serde_json::Value = + serde_json::from_str(&dataset.index_statistics("text_search").await.unwrap()) + .unwrap(); + for (key, value) in expected.as_object().unwrap() { + assert_eq!(&stats["indices"][0][key], value); + } + } + for (filter, expected_rows) in [(predicate, 2), ("text IS NULL", 2), ("text = ''", 2)] { + let query = table.query().only_if(filter); + let plan = query.explain_plan(false).await.unwrap(); + if filter == predicate { + assert!(plan.contains("ScalarIndexQuery"), "{plan}"); + } + let batches = query + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!( + batches.iter().map(RecordBatch::num_rows).sum::(), + expected_rows + ); + } + let error = table + .create_index(&[invalid_column], index) + .execute() + .await + .unwrap_err(); + assert!( + error.to_string().to_lowercase().contains( + if index_type == crate::index::IndexType::NGram { + "ngram" + } else { + "bloom" + } + ), + "{error}" + ); + } + } + + #[cfg(feature = "geo")] + #[tokio::test] + async fn test_create_rtree_index() { + let points = StructArray::new( + vec![ + Field::new("x", DataType::Float64, false), + Field::new("y", DataType::Float64, false), + ] + .into(), + vec![ + Arc::new(arrow_array::Float64Array::from(vec![0.0, 10.0, 0.0])), + Arc::new(arrow_array::Float64Array::from(vec![0.0, 10.0, 0.0])), + ], + Some(arrow_buffer::NullBuffer::from(vec![true, true, false])), + ); + let field = Field::new("geometry", points.data_type().clone(), true).with_metadata( + std::collections::HashMap::from([ + ("ARROW:extension:name".into(), "geoarrow.point".into()), + ("ARROW:extension:metadata".into(), "{}".into()), + ]), + ); + let batch = + RecordBatch::try_new(Arc::new(Schema::new(vec![field])), vec![Arc::new(points)]) + .unwrap(); + let conn = connect("memory://").execute().await.unwrap(); + let table = conn.create_table("spatial", batch).execute().await.unwrap(); + table + .create_index( + &["geometry"], + Index::RTree( + crate::index::scalar::RTreeIndexBuilder::default() + .page_size(2) + .unwrap(), + ), + ) + .name("spatial_idx".into()) + .execute() + .await + .unwrap(); + let indices = table.list_indices().await.unwrap(); + assert_eq!(indices.len(), 1); + assert_eq!(indices[0].index_type, crate::index::IndexType::RTree); + let stats = table.index_stats("spatial_idx").await.unwrap().unwrap(); + assert_eq!(stats.index_type, crate::index::IndexType::RTree); + assert_eq!(stats.num_indexed_rows, 3); + let dataset = table.as_native().unwrap().dataset.get().await.unwrap(); + let stats: serde_json::Value = + serde_json::from_str(&dataset.index_statistics("spatial_idx").await.unwrap()).unwrap(); + assert_eq!(stats["indices"][0]["page_size"], 2); + for predicate in [ + "ST_Intersects(geometry, ST_GeomFromText('POLYGON ((-1 -1, 1 -1, 1 1, -1 1, -1 -1))'))", + "geometry IS NULL", + ] { + let query = table.query().only_if(predicate); + let plan = query.explain_plan(false).await.unwrap(); + assert!(plan.contains("ScalarIndexQuery"), "{plan}"); + let batches = query + .execute() + .await + .unwrap() + .try_collect::>() + .await + .unwrap(); + assert_eq!(batches.iter().map(RecordBatch::num_rows).sum::(), 1); + } + } + #[tokio::test] async fn test_create_index_nested_field_paths() { let tmp_dir = tempdir().unwrap(); From f8d73b34470a9b9074ed5c08670fbfdb8d3bd936 Mon Sep 17 00:00:00 2001 From: LanceDB Robot Date: Wed, 16 Sep 2026 16:11:15 -0700 Subject: [PATCH 35/43] chore: update lance dependency to v13.0.0-beta.4 (#4207) Updates the Rust workspace Lance dependencies, Cargo lockfile, and Java lance-core from v13.0.0-beta.3 to [v13.0.0-beta.4](https://github.com/lance-format/lance/releases/tag/v13.0.0-beta.4); no compatibility fixes were required. Validation passed: `cargo clippy --quiet --workspace --tests --all-features -- -D warnings`, `cargo fmt --all --quiet`, and `git diff --check`. --- Cargo.lock | 84 ++++++++++++++++++++++++++-------------------------- Cargo.toml | 28 +++++++++--------- java/pom.xml | 2 +- 3 files changed, 57 insertions(+), 57 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7037168c6..0d6a28878 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3523,8 +3523,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5073,8 +5073,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arc-swap", "arrow", @@ -5146,8 +5146,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow-array", "arrow-buffer", @@ -5169,7 +5169,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow-array", "arrow-buffer", @@ -5183,7 +5183,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "58.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow-array", "arrow-schema", @@ -5192,8 +5192,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrayref", "crunchy", @@ -5203,8 +5203,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow-array", "arrow-buffer", @@ -5241,8 +5241,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow", "arrow-array", @@ -5273,8 +5273,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow", "arrow-array", @@ -5291,8 +5291,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "proc-macro2", "quote", @@ -5301,8 +5301,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow-arith", "arrow-array", @@ -5335,8 +5335,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow-arith", "arrow-array", @@ -5383,8 +5383,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arc-swap", "arrow", @@ -5452,8 +5452,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow-array", "arrow-schema", @@ -5475,8 +5475,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow", "arrow-array", @@ -5516,8 +5516,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow-array", "arrow-schema", @@ -5531,8 +5531,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow", "async-trait", @@ -5546,8 +5546,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow", "arrow-ipc", @@ -5600,8 +5600,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow-array", "arrow-buffer", @@ -5615,8 +5615,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow", "arrow-array", @@ -5656,8 +5656,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "arrow-array", "arrow-schema", @@ -5670,8 +5670,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 897bdbe40..35b5a7d60 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,20 +13,20 @@ categories = ["database-implementations"] rust-version = "1.91.0" [workspace.dependencies] -lance = { "version" = "=13.0.0-beta.3", default-features = false, "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=13.0.0-beta.3", default-features = false, "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=13.0.0-beta.3", default-features = false, "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=13.0.0-beta.3", "tag" = "v13.0.0-beta.3", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=13.0.0-beta.4", default-features = false, "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=13.0.0-beta.4", default-features = false, "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=13.0.0-beta.4", default-features = false, "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=13.0.0-beta.4", "tag" = "v13.0.0-beta.4", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" # Note that this one does not include pyarrow diff --git a/java/pom.xml b/java/pom.xml index b15c9dfc1..53de36b8e 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -28,7 +28,7 @@ UTF-8 15.0.0 - 13.0.0-beta.3 + 13.0.0-beta.4 false 2.30.0 1.7 From 60a1b4c2191194896cb6920e91bce392c1093674 Mon Sep 17 00:00:00 2001 From: Jonathan Hsieh Date: Wed, 16 Sep 2026 17:00:23 -0700 Subject: [PATCH 36/43] feat(secrets): named Secrets, bindings, and namespace addressing (#4150) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the client half of database-scoped named Secrets: a Secret is a name and an opaque value stored by the service, and a Function binds one to the environment variable its library already reads. Secrets are addressed by a namespace path plus a name. The UDF body is unchanged and stays portable — it reads `OPENAI_API_KEY` the way it always did, and the binding is what puts a value there: ```python db.create_secret("openai-prod", os.environ["OPENAI_API_KEY"]) function = db.create_function( analyze_caption, secrets=[ EnvVarSecret(secret_name="openai-prod", env_variable="OPENAI_API_KEY") ], ) function.secret_bindings # the Secret's name, never its value ``` - `create_secret` / `alter_secret` / `list_secrets` / `describe_secret` / `drop_secret` on sync, async and remote connections, with the pyo3 binding and the Rust client behind them. Each takes `namespace_path` keyword-only, defaulting to the root. - **There is no read API, by construction rather than by policy** — no code path returns a stored credential, and `describe_secret` answers with metadata only. - `EnvVarSecret` is a pure local constructor: it contacts no server, so it cannot fail on a Secret that does not exist. It exists so that a bare string in that position — which would be a credential — is a `TypeError` rather than a plausible-looking mistake that reads identically in a diff. - `create_function(..., secrets=[...])` carries the bindings as `secret_bindings`: a list of `SecretBinding` tagged by `kind`, so a later delivery mode is a variant rather than a sibling field. The value never travels — it is resolved by the service when the Function runs, which is what lets a rotation reach columns already pinned to an older FunctionVersion. - A binding names its Secret as a `SecretReference` of `{name, namespace_path}` rather than one joined string, so no delimiter has to be excluded from every name and segment forever, and `ClientConfig.id_delimiter` cannot contradict an identity built on a fixed separator. - A root namespace is omitted from the request body rather than sent empty, so a root request is byte-identical to one from a client that predates namespaces. Tests pin it. This is the client surface the design's §4 describes; the service side lives in sophon. **Previously split across two PRs.** Namespace addressing was #4151, stacked on this one; it is folded in here so the Secret identity contract — name, namespace path, and the binding that carries both — is reviewable as one piece rather than as a shape introduced and then replaced. ## Identifier safety, merged from #4189 **#4189 is merged into this branch**, so the client half of Secrets and the guards on the identity it puts in the URL are one PR. What it added: - Components are checked where the identifier is built, before a request is constructed. `create_secret("../jobs", value)` no longer resolves to `/v1/jobs/create` and delivers a credential-bearing body to a route with none of this one's body suppression. - Each component is percent-encoded and joined by the delimiter, so nothing inside a component can end the path segment or add one. - A component may not be empty, a relative segment (`.`, `..`, and their `%2e` spellings), or the delimiter itself — the three ways a component erases a boundary the split has to recover. `["prod", ""]` joined to `prod$`, which reads back as `["prod"]`. - `$` is the only accepted `id_delimiter`, refused at client construction. `ClientConfig.id_delimiter` remains, since the identifier grammar comes from the Lance REST catalog standard, but a value that would produce identifiers no service splits the caller's way is now an error where it was written. - One `build_object_identifier` and one character set serve tables, namespaces, Secrets, Functions and materialized views. Components are checked for *addressability*, not a character set: the name's own grammar stays each object's own, so a catalog database keeps the `/` that `RemoteCatalog::validate_name` allows. ## Known shortcoming `secret_bindings` is omitted from a registration body when empty, so a client that binds nothing sends what a client without bindings sends. When a client does bind a Secret and the service does not know the field, the field is ignored: registration succeeds, the returned version carries no bindings, and the Function fails at execution with the variable unset, far from the call that asked for it. `ServerVersion` is how this codebase refuses a feature the service is too old for, and it gates five features already. It does not gate this one: it is held per table, and registering a Function is a database-level call. Noted at the field in `remote/db.rs`; wiring the gate is follow-up work. **Tests:** lancedb lib 1340 passed, `first_class_function_slice1` 9, `first_class_function_slice2` 3, plus Python tests across both slices. Rebased onto `main` after #4176 (OCI Function identity), #4191 (`.`/`..` table names) and #4195 (remote catalogs). 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01UfmeJ533rQDnPBkMtjerV6 --------- Co-authored-by: Claude Opus 5 (1M context) --- docs/src/js/interfaces/ClientConfig.md | 4 + docs/src/python/python.md | 4 + nodejs/__test__/catalog.test.ts | 2 +- nodejs/src/remote.rs | 3 + python/python/lancedb/__init__.py | 2 + python/python/lancedb/_lancedb.pyi | 15 + python/python/lancedb/db.py | 223 +++++- python/python/lancedb/functions.py | 112 ++- python/python/lancedb/remote/__init__.py | 5 +- python/python/lancedb/remote/db.py | 50 +- python/python/lancedb/secrets.py | 226 ++++++ python/python/tests/test_catalog.py | 2 +- .../tests/test_first_class_function_slice1.py | 48 ++ .../tests/test_first_class_function_slice2.py | 414 ++++++++++- python/src/connection.rs | 79 ++ rust/lancedb/src/connection.rs | 85 +++ rust/lancedb/src/database.rs | 45 ++ rust/lancedb/src/function.rs | 102 ++- rust/lancedb/src/lib.rs | 1 + rust/lancedb/src/remote/catalog.rs | 2 +- rust/lancedb/src/remote/client.rs | 194 ++++- rust/lancedb/src/remote/db.rs | 702 ++++++++++++++++-- rust/lancedb/src/secrets.rs | 181 +++++ rust/lancedb/src/utils/mod.rs | 137 ++-- .../tests/first_class_function_slice1.rs | 100 +++ .../tests/first_class_function_slice2.rs | 57 ++ .../v1/remote_function_job.json | 11 +- ...secret_registration_request.canonical.json | 1 + ..._function_secret_registration_request.json | 52 ++ .../v1/remote_function_version.canonical.json | 2 +- 30 files changed, 2671 insertions(+), 190 deletions(-) create mode 100644 python/python/lancedb/secrets.py create mode 100644 rust/lancedb/src/secrets.rs create mode 100644 rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.canonical.json create mode 100644 rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.json diff --git a/docs/src/js/interfaces/ClientConfig.md b/docs/src/js/interfaces/ClientConfig.md index 94cee5b5e..2b4984b8d 100644 --- a/docs/src/js/interfaces/ClientConfig.md +++ b/docs/src/js/interfaces/ClientConfig.md @@ -22,6 +22,10 @@ optional extraHeaders: Record; optional idDelimiter: string; ``` +The delimiter joining a namespace path and a name into one object +identifier. `"$"` is the only supported value, and leaving this unset is +how to get it; anything else is rejected when the connection is created. + *** ### retryConfig? diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 96161fbac..80899b964 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -136,6 +136,10 @@ listing a storage directory. ::: lancedb.functions.UdfDefinition +::: lancedb.secrets.EnvVarSecret + +::: lancedb.secrets.SecretInfo + ::: lancedb.functions.FunctionRegistrationRequest ::: lancedb.functions.FunctionArtifactRequest diff --git a/nodejs/__test__/catalog.test.ts b/nodejs/__test__/catalog.test.ts index d44103ab9..f9158c94f 100644 --- a/nodejs/__test__/catalog.test.ts +++ b/nodejs/__test__/catalog.test.ts @@ -87,7 +87,7 @@ describe("remote catalog", () => { expect(requests[0].url).toBe("/v1/namespace/team%2Fsearch/create"); expect(requests[0].body).toEqual({ mode: "ExistOk" }); expect(requests[5].url).toBe( - "/v1/namespace/%24/list?limit=1&page_token=a%2Fb", + "/v1/namespace/$/list?limit=1&page_token=a%2Fb", ); expect(requests[6].body).toEqual({ mode: "Skip", diff --git a/nodejs/src/remote.rs b/nodejs/src/remote.rs index 784631267..c2d671735 100644 --- a/nodejs/src/remote.rs +++ b/nodejs/src/remote.rs @@ -93,6 +93,9 @@ pub struct ClientConfig { pub retry_config: Option, pub timeout_config: Option, pub extra_headers: Option>, + /// The delimiter joining a namespace path and a name into one object + /// identifier. `"$"` is the only supported value, and leaving this unset is + /// how to get it; anything else is rejected when the connection is created. pub id_delimiter: Option, pub tls_config: Option, /// User identifier for tracking purposes. diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 19114e1a1..05ad664e1 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -37,6 +37,8 @@ from .functions import ( UdfDefinition as UdfDefinition, udf as udf, ) +from .secrets import EnvVarSecret as EnvVarSecret +from .secrets import SecretInfo as SecretInfo from .materialized_view import ( AsyncMaterializedView, MaterializedView, diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index f267b3a55..a0039df05 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -153,6 +153,21 @@ class Connection(object): async def get_function(self, name: str, version: str) -> str: ... async def list_functions(self) -> List[str]: ... async def drop_function(self, name: str, version: str) -> bool: ... + async def create_secret( + self, name: str, value: str, namespace_path: Optional[List[str]] = None + ) -> None: ... + async def alter_secret( + self, name: str, value: str, namespace_path: Optional[List[str]] = None + ) -> None: ... + async def list_secrets( + self, namespace_path: Optional[List[str]] = None + ) -> List[str]: ... + async def drop_secret( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> None: ... + async def describe_secret( + self, name: str, namespace_path: Optional[List[str]] = None + ) -> Tuple[str, int, int]: ... async def list_jobs(self) -> List[JobInfo]: ... async def cancel_job(self, job_id: str) -> bool: ... async def execute_query_async( diff --git a/python/python/lancedb/db.py b/python/python/lancedb/db.py index 4a8820cef..d17e897cf 100644 --- a/python/python/lancedb/db.py +++ b/python/python/lancedb/db.py @@ -17,6 +17,7 @@ from typing import ( List, Literal, Optional, + Sequence, Union, ) from uuid import UUID @@ -57,6 +58,12 @@ from .materialized_view import ( SelectArg, normalize_select, ) +from .secrets import ( + EnvVarSecret, + SecretInfo, + validate_namespace_path, + validate_secret_name, +) from .table import ( AsyncTable, LanceTable, @@ -742,16 +749,50 @@ class DBConnection(EnforceOverrides): """ raise NotImplementedError("serialize is not supported for this connection type") - def create_function(self, definition: UdfDefinition) -> FunctionVersion: + def create_function( + self, + definition: UdfDefinition, + *, + secrets: Optional[Sequence[EnvVarSecret]] = None, + ) -> FunctionVersion: """Build and register a scalar Python UDF, then return its version. The server builds the OCI image and registers the completed artifact. This is the blocking counterpart of :meth:`create_function_async`. Local connections raise ``NotImplementedError``. - """ - return self.create_function_async(definition).wait() - def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]: + Parameters + ---------- + definition : UdfDefinition + A callable decorated with [udf][lancedb.udf]. + secrets : sequence of EnvVarSecret, optional + One [EnvVarSecret][lancedb.secrets.EnvVarSecret] per credential the + Function needs, each naming a Secret and the environment variable + its value arrives in. The Function's source is unchanged by this; + it reads the variable the way it already did. + + Examples + -------- + ```python + db.create_secret("openai-prod", os.environ["OPENAI_API_KEY"]) + db.create_function( + analyze_caption, + secrets=[ + EnvVarSecret( + secret_name="openai-prod", env_variable="OPENAI_API_KEY" + ) + ], + ) + ``` + """ + return self.create_function_async(definition, secrets=secrets).wait() + + def create_function_async( + self, + definition: UdfDefinition, + *, + secrets: Optional[Sequence[EnvVarSecret]] = None, + ) -> Job[FunctionVersion]: """Submit a scalar Python UDF for building and registration. The server-side job builds the OCI image, then registers the completed @@ -798,6 +839,70 @@ class DBConnection(EnforceOverrides): "Function catalog operations are not supported for this connection type" ) + def create_secret( + self, name: str, value: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + """Create a named Secret in this database. + + Fails if the name is taken, so a create never silently becomes a + rotation. Nothing reads the value back: it is bound to a Function by + name and resolved by the service when that Function runs. Local + connections raise ``NotImplementedError``. + """ + raise NotImplementedError( + "Secret operations are not supported for this connection type" + ) + + def alter_secret( + self, name: str, value: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + """Replace the credential behind an existing Secret. + + Fails if it does not exist. Every Function bound to the Secret uses the + new value from its next job, and no new Function version is created -- + which is how a rotation reaches columns pinned to a version registered + before it. Local connections raise ``NotImplementedError``. + """ + raise NotImplementedError( + "Secret operations are not supported for this connection type" + ) + + def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]: + """The names of every Secret in this database. + + Names only. No method returns a stored credential, by construction + rather than by policy. Local connections raise ``NotImplementedError``. + """ + raise NotImplementedError( + "Secret operations are not supported for this connection type" + ) + + def drop_secret( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + """Drop a Secret. + + Functions bound to it fail at their next job, naming the Secret; that + is the revocation path. The name becomes free to reuse, and a new + Secret under it is picked up by everything still bound to that name. + Local connections raise ``NotImplementedError``. + """ + raise NotImplementedError( + "Secret operations are not supported for this connection type" + ) + + def describe_secret( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> SecretInfo: + """What this database records about a Secret: name and timestamps. + + Never the value -- there is no code path that could return one. Local + connections raise ``NotImplementedError``. + """ + raise NotImplementedError( + "Secret operations are not supported for this connection type" + ) + def open_job(self, job_id: str) -> Job: """Open a server-side job by id, returning a handle with its record already populated. @@ -1557,8 +1662,13 @@ class LanceDBConnection(DBConnection): return Job(LOOP.run(self._conn.open_job(job_id))) @override - def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]: - job = LOOP.run(self._conn.create_function_async(definition)) + def create_function_async( + self, + definition: UdfDefinition, + *, + secrets: Optional[Sequence[EnvVarSecret]] = None, + ) -> Job[FunctionVersion]: + job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets)) return Job(job) @override @@ -1573,6 +1683,34 @@ class LanceDBConnection(DBConnection): def drop_function(self, name: str, *, version: str) -> bool: return LOOP.run(self._conn.drop_function(name, version=version)) + @override + def create_secret( + self, name: str, value: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + LOOP.run(self._conn.create_secret(name, value, namespace_path=namespace_path)) + + @override + def alter_secret( + self, name: str, value: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + LOOP.run(self._conn.alter_secret(name, value, namespace_path=namespace_path)) + + @override + def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]: + return LOOP.run(self._conn.list_secrets(namespace_path=namespace_path)) + + @override + def drop_secret( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + LOOP.run(self._conn.drop_secret(name, namespace_path=namespace_path)) + + @override + def describe_secret( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> SecretInfo: + return LOOP.run(self._conn.describe_secret(name, namespace_path=namespace_path)) + @override def list_jobs(self) -> List[JobInfo]: """List server-side jobs across the database's tables.""" @@ -2422,19 +2560,24 @@ class AsyncConnection(object): return AsyncJob(await self._inner.open_job(job_id)) async def create_function_async( - self, definition: UdfDefinition + self, + definition: UdfDefinition, + *, + secrets: Optional[Sequence[EnvVarSecret]] = None, ) -> AsyncJob[FunctionVersion]: """Submit a scalar Python UDF for building and registration. The server-side job builds the OCI image, then registers the completed artifact. Waiting on the job returns the immutable Function version. - Local connections raise ``NotImplementedError``. + ``secrets`` is a sequence of + [EnvVarSecret][lancedb.secrets.EnvVarSecret], each naming a Secret and + the environment variable its value arrives in. Local connections raise + ``NotImplementedError``. """ if not isinstance(definition, UdfDefinition): raise TypeError("create_function_async requires a @udf definition") - inner = await self._inner.create_function_async( - definition.registration_request.to_canonical_json() - ) + request = definition.bind_secrets(secrets) + inner = await self._inner.create_function_async(request.to_canonical_json()) return _typed_job(inner, FunctionVersion.from_json) async def get_function(self, name: str, *, version: str) -> FunctionVersion: @@ -2456,6 +2599,64 @@ class AsyncConnection(object): """Remove the current name binding, retaining the object and its history.""" return await self._inner.drop_function(name, version) + async def create_secret( + self, name: str, value: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + """Create a named Secret in this database. + + Fails if the name is taken, so a create never silently becomes a + rotation. Nothing reads the value back. + """ + await self._inner.create_secret( + validate_secret_name(name), + value, + list(validate_namespace_path(namespace_path)), + ) + + async def alter_secret( + self, name: str, value: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + """Replace the credential behind an existing Secret. + + Fails if it does not exist. Bound Functions use the new value from + their next job, with no new Function version. + """ + await self._inner.alter_secret( + validate_secret_name(name), + value, + list(validate_namespace_path(namespace_path)), + ) + + async def list_secrets( + self, *, namespace_path: Optional[List[str]] = None + ) -> List[str]: + """The names of every Secret in this database. Names only.""" + return await self._inner.list_secrets( + list(validate_namespace_path(namespace_path)) + ) + + async def drop_secret( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + """Drop a Secret. Bound Functions fail at their next job.""" + await self._inner.drop_secret( + validate_secret_name(name), list(validate_namespace_path(namespace_path)) + ) + + async def describe_secret( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> SecretInfo: + """What this database records about a Secret. Never the value.""" + name, created_at_millis, updated_at_millis = await self._inner.describe_secret( + validate_secret_name(name), + list(validate_namespace_path(namespace_path)), + ) + return SecretInfo( + name=name, + created_at_millis=created_at_millis, + updated_at_millis=updated_at_millis, + ) + async def list_jobs(self) -> List[JobInfo]: """List server-side jobs across the database's tables.""" return await self._inner.list_jobs() diff --git a/python/python/lancedb/functions.py b/python/python/lancedb/functions.py index 692bc5756..8c6864cc3 100644 --- a/python/python/lancedb/functions.py +++ b/python/python/lancedb/functions.py @@ -4,7 +4,7 @@ """Canonical Function values exchanged with LanceDB Enterprise services. These immutable models contain client/wire state only. Catalog persistence, -environment bake, and execution are owned by Sophon. +environment bake, secret resolution, and execution are owned by Sophon. ``RefreshColumnResult`` is also the backend-neutral result of a local expression-backed refresh job. """ @@ -25,7 +25,7 @@ import re import sys import textwrap import types -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import date, datetime from typing import ( Annotated, @@ -51,6 +51,7 @@ from pydantic import ( ) from .schema import is_blob_v2_field as _is_blob_v2_field +from .secrets import EnvVarSecret _Int32 = conint(strict=True, ge=-(2**31), le=2**31 - 1) _UInt32 = conint(strict=True, ge=0, le=2**32 - 1) @@ -227,6 +228,33 @@ class FunctionOutput(_OpenRemoteValue): fields: tuple[FunctionResultField, ...] = () +class SecretReference(_RemoteValue): + """Where a Secret lives, carried as its parts rather than as one string. + + A joined id would need a delimiter, and a delimiter has to be excluded from + every name and segment forever, agreed on by both sides, and re-agreed each + time either grows a new way to be configured. Naming the parts settles all + of that: nothing here is parsed, so nothing can parse two ways. + """ + + name: str + namespace_path: tuple[str, ...] = () + + +class SecretBinding(_RemoteValue): + """How a Secret reaches the Function that binds it. + + One list rather than a field per delivery mode: a binding is the concept, + and how it arrives is a property of one. ``kind`` is open, so a binding a + newer service introduces decodes here instead of failing the whole + FunctionVersion. + """ + + kind: str + variable: Optional[str] = None + secret_ref: Optional[SecretReference] = None + + class FunctionSignature(_RemoteValue): inputs: tuple[FunctionParameter, ...] output: FunctionOutput @@ -326,6 +354,7 @@ class FunctionVersion(_RemoteValue): version: _ObjectVersion image: FunctionImage signature: FunctionSignature + secret_bindings: tuple[SecretBinding, ...] = () created_at: str metadata: Mapping[str, str] disabled: bool @@ -395,12 +424,18 @@ class FunctionVersion(_RemoteValue): class FunctionRegistrationRequest(_RemoteValue): - """Stable remote registration envelope produced by :func:`udf`.""" + """Stable remote registration envelope produced by :func:`udf`. + + Credential values deliberately have no field here. The only secret-shaped + thing a client sends is ``secret_bindings``: the name of a Secret the + database already holds, which the remote service resolves at execution. + """ name: str artifact: FunctionArtifactRequest signature: FunctionSignature runtime: PythonRuntimeSpec + secret_bindings: tuple[SecretBinding, ...] = () class FunctionVersionRef(_OpenRemoteValue): @@ -552,6 +587,7 @@ class RefreshColumnResult(_RemoteValue): _FUNCTION_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_.-]*$") + _FUNCTION_BLOB_V2_TYPE = "blob_v2" _ARROW_EXTENSION_NAME_KEY = "ARROW:extension:name" _BLOB_V2_EXTENSION_NAME = "lance.blob.v2" @@ -1293,9 +1329,73 @@ class UdfDefinition: @property def registration_request(self) -> FunctionRegistrationRequest: - """The immutable request sent by ``create_function_async``.""" + """The immutable request sent by ``create_function_async``. + + Carries no secret bindings. Binding is a registration-time decision, + so a Function bound to Secrets is registered through :meth:`bind_secrets`, + which is what ``create_function`` calls. + """ return self._request + def bind_secrets( + self, secrets: Optional[Sequence[EnvVarSecret]] + ) -> FunctionRegistrationRequest: + """The registration request for this definition bound to ``secrets``. + + Binding does not change the Function's source: each + [EnvVarSecret][lancedb.secrets.EnvVarSecret] names a Secret and the + environment variable its value should arrive in, and the Function reads + that variable the way it already did. Whether the named Secrets exist is + the server's answer, not this one. + """ + bindings = () if secrets is None else tuple(secrets) + wrong_type = [ + binding for binding in bindings if not isinstance(binding, EnvVarSecret) + ] + if wrong_type: + kinds = sorted({type(binding).__name__ for binding in wrong_type}) + raise TypeError( + f"Function secrets must be EnvVarSecret values, not {kinds!r}; a " + "credential value is never sent to this API" + ) + variables = [binding.env_variable for binding in bindings] + duplicates = sorted({name for name in variables if variables.count(name) > 1}) + if duplicates: + raise ValueError( + "a Function binds each environment variable once; duplicated: " + f"{duplicates!r}" + ) + # `env` is ordinary configuration carried in the definition, so a name in + # both would have a value visible in the Function's record and a value + # that is not. Refuse rather than pick. + environment = self._request.runtime.env or {} + overlap = sorted(set(environment) & set(variables)) + if overlap: + raise ValueError( + f"Function env and secret bindings must be disjoint: {overlap!r}" + ) + if not bindings: + return self._request + # Sorted, because the list is carried in the FunctionVersion hash and a + # caller's argument order is not part of what a Function is. + resolved = tuple( + sorted( + ( + SecretBinding( + kind="env", + variable=binding.env_variable, + secret_ref=SecretReference( + name=binding.secret_name, + namespace_path=tuple(binding.secret_namespace_path), + ), + ) + for binding in bindings + ), + key=lambda binding: (binding.kind, binding.variable or ""), + ) + ) + return self._request._copy(update={"secret_bindings": resolved}) + def __call__(self, *args, **kwargs): return self._function(*args, **kwargs) @@ -1360,7 +1460,9 @@ def udf( conda_channels : sequence of str, optional Conda channels in priority order; requires ``conda``. env : mapping of str to str, optional - Environment variables included in the Function definition. + Environment variables included in the Function definition. Not for + credentials -- these are ordinary configuration, stored with the + Function and visible wherever it is. python_version : str, optional Remote Python major/minor version. Defaults to the client version. gpu : bool, default False diff --git a/python/python/lancedb/remote/__init__.py b/python/python/lancedb/remote/__init__.py index b70a869a3..dbcaa1b64 100644 --- a/python/python/lancedb/remote/__init__.py +++ b/python/python/lancedb/remote/__init__.py @@ -173,7 +173,10 @@ class ClientConfig: extra_headers: Optional[dict] Additional headers to include in requests. id_delimiter: Optional[str] - The delimiter to use when constructing object identifiers. + The delimiter joining a namespace path and a name into one object + identifier. ``"$"`` is the only supported value, and leaving this + unset is how to get it; anything else is rejected when the connection + is created. tls_config: Optional[TlsConfig] TLS/mTLS configuration for secure connections. header_provider: Optional[HeaderProvider] diff --git a/python/python/lancedb/remote/db.py b/python/python/lancedb/remote/db.py index aceebf987..c9e857495 100644 --- a/python/python/lancedb/remote/db.py +++ b/python/python/lancedb/remote/db.py @@ -8,7 +8,16 @@ import json import logging from concurrent.futures import ThreadPoolExecutor import sys -from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, Union +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Iterable, + List, + Optional, + Sequence, + Union, +) from urllib.parse import urlparse from uuid import UUID import warnings @@ -30,6 +39,7 @@ from ..job import AsyncJob, Job from ..sql import Query as SqlQuery from ..sql import QueryDescription from ..materialized_view import MaterializedView, SelectArg +from ..secrets import EnvVarSecret, SecretInfo if TYPE_CHECKING: from .._lancedb import JobInfo @@ -845,8 +855,14 @@ class RemoteDBConnection(DBConnection): return Job(LOOP.run(self._conn.open_job(job_id))) @override - def create_function_async(self, definition: UdfDefinition) -> Job[FunctionVersion]: - return Job(LOOP.run(self._conn.create_function_async(definition))) + def create_function_async( + self, + definition: UdfDefinition, + *, + secrets: Optional[Sequence[EnvVarSecret]] = None, + ) -> Job[FunctionVersion]: + job = LOOP.run(self._conn.create_function_async(definition, secrets=secrets)) + return Job(job) @override def get_function(self, name: str, *, version: str) -> FunctionVersion: @@ -860,6 +876,34 @@ class RemoteDBConnection(DBConnection): def drop_function(self, name: str, *, version: str) -> bool: return LOOP.run(self._conn.drop_function(name, version=version)) + @override + def create_secret( + self, name: str, value: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + LOOP.run(self._conn.create_secret(name, value, namespace_path=namespace_path)) + + @override + def alter_secret( + self, name: str, value: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + LOOP.run(self._conn.alter_secret(name, value, namespace_path=namespace_path)) + + @override + def describe_secret( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> SecretInfo: + return LOOP.run(self._conn.describe_secret(name, namespace_path=namespace_path)) + + @override + def list_secrets(self, *, namespace_path: Optional[List[str]] = None) -> List[str]: + return LOOP.run(self._conn.list_secrets(namespace_path=namespace_path)) + + @override + def drop_secret( + self, name: str, *, namespace_path: Optional[List[str]] = None + ) -> None: + LOOP.run(self._conn.drop_secret(name, namespace_path=namespace_path)) + @override def list_jobs(self) -> List["JobInfo"]: """List server-side jobs across the database's tables.""" diff --git a/python/python/lancedb/secrets.py b/python/python/lancedb/secrets.py new file mode 100644 index 000000000..75ac9e340 --- /dev/null +++ b/python/python/lancedb/secrets.py @@ -0,0 +1,226 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + +"""Named Secrets, and the bindings that deliver them to Functions. + +A Secret is a database-scoped named credential. Nothing in this module holds a +value: :class:`EnvVarSecret` names one and says which environment variable it +should arrive in, and the value is resolved by the remote service when a +Function bound to it runs. No API returns a stored credential, by construction +rather than by policy -- there is no code path that could. +""" + +from __future__ import annotations + +import re + +# The same characters LanceDB already admits in a namespace or table name, and +# no positional rule on top of them: a segment may begin with `_`, `-` or `.` +# today, so anything narrower would put Secrets out of reach inside namespaces +# that already exist. Matches the service, which admits the same set. +_SECRET_NAME = re.compile(r"^[A-Za-z0-9_.-]{1,255}$") +_ENV_VARIABLE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +def validate_secret_name(name: str) -> str: + """Check a Secret name locally and return it unchanged.""" + if not isinstance(name, str): + raise TypeError(f"Secret name must be a string, not {type(name).__name__}") + if not _SECRET_NAME.fullmatch(name): + raise ValueError(f"invalid Secret name: {name!r}") + return name + + +def validate_namespace_path(namespace_path=None): + """Check a namespace path locally and return it as a tuple. + + ``None`` and ``[]`` both mean the root namespace. Segments follow the same + rule as Secret names: a binding carries the path and the name as separate + fields, so neither is ever parsed out of the other. + """ + if namespace_path is None: + return () + if isinstance(namespace_path, str): + raise TypeError( + "namespace_path must be a list of segments, not a string; " + f"did you mean [{namespace_path!r}]?" + ) + segments = tuple(namespace_path) + for segment in segments: + if not isinstance(segment, str): + raise TypeError( + f"namespace path segment must be a string, not {type(segment).__name__}" + ) + if not _SECRET_NAME.fullmatch(segment): + raise ValueError(f"invalid namespace path segment: {segment!r}") + return segments + + +def validate_env_variable(name: str) -> str: + """Check an environment variable name locally and return it unchanged.""" + if not isinstance(name, str): + raise TypeError( + f"environment variable name must be a string, not {type(name).__name__}" + ) + if not _ENV_VARIABLE.fullmatch(name): + raise ValueError(f"invalid environment variable name: {name!r}") + return name + + +class EnvVarSecret: + """A Secret bound to the environment variable a Function's library reads. + + Pass these in the ``secrets`` sequence of + [DBConnection.create_function][lancedb.db.DBConnection.create_function]. The + Function's source is unchanged by binding: it reads ``OPENAI_API_KEY`` the + way it always did, and the binding is what puts a value there. + + This is a local value. Constructing it contacts no server, so it always + succeeds and says nothing about whether the Secret exists; that is checked + at registration, where a mistyped Secret name surfaces as a clear "does not + exist" naming both the Secret and the variable bound to it. A mistyped + *variable* name cannot be caught anywhere -- nothing knows which variables a + Function reads -- so it surfaces on the first rows instead. + + The type exists so a credential cannot be passed by accident. A bare string + in the same position is a plausible-looking mistake with the opposite + meaning, and it reads identically in a diff. + + Parameters + ---------- + secret_name : str + The Secret's database-scoped name. + env_variable : str + The environment variable the Function reads it from. + secret_namespace_path : list of str, optional + The namespace the Secret is addressed within. ``None`` and ``[]`` both + mean the root namespace. Carried beside the name rather than joined + into it, so neither is ever parsed back out of the other. + + Examples + -------- + >>> from lancedb import EnvVarSecret + >>> binding = EnvVarSecret( + ... secret_name="openai-prod", env_variable="OPENAI_API_KEY" + ... ) + >>> binding.secret_name, binding.env_variable + ('openai-prod', 'OPENAI_API_KEY') + """ + + __slots__ = ("_secret_name", "_env_variable", "_secret_namespace_path") + + def __init__( + self, secret_name: str, env_variable: str, *, secret_namespace_path=None + ): + self._secret_name = validate_secret_name(secret_name) + self._env_variable = validate_env_variable(env_variable) + self._secret_namespace_path = validate_namespace_path(secret_namespace_path) + + @property + def secret_name(self) -> str: + """The Secret's database-scoped name.""" + return self._secret_name + + @property + def env_variable(self) -> str: + """The environment variable the value is delivered in.""" + return self._env_variable + + @property + def secret_namespace_path(self): + """The namespace path the Secret is addressed within, root when empty.""" + return list(self._secret_namespace_path) + + def __repr__(self) -> str: + path = ( + f", secret_namespace_path={list(self._secret_namespace_path)!r}" + if self._secret_namespace_path + else "" + ) + return ( + f"EnvVarSecret(secret_name={self._secret_name!r}, " + f"env_variable={self._env_variable!r}{path})" + ) + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, EnvVarSecret) + and other._secret_name == self._secret_name + and other._env_variable == self._env_variable + and other._secret_namespace_path == self._secret_namespace_path + ) + + def __hash__(self) -> int: + return hash( + ( + EnvVarSecret, + self._secret_name, + self._env_variable, + self._secret_namespace_path, + ) + ) + + +class SecretInfo: + """What a database records about a Secret. Never its value. + + Returned by + [DBConnection.describe_secret][lancedb.db.DBConnection.describe_secret]. + """ + + __slots__ = ("_name", "_created_at_millis", "_updated_at_millis") + + def __init__(self, name: str, created_at_millis: int, updated_at_millis: int): + self._name = name + self._created_at_millis = created_at_millis + self._updated_at_millis = updated_at_millis + + @property + def name(self) -> str: + """The Secret's database-scoped name.""" + return self._name + + @property + def created_at_millis(self) -> int: + """When the Secret was created, in milliseconds since the Unix epoch.""" + return self._created_at_millis + + @property + def updated_at_millis(self) -> int: + """When the Secret's value was last rotated, in epoch milliseconds. + + The only observable that a rotation landed: no API returns a credential, + so a caller confirms ``alter_secret`` took effect by watching this move. + """ + return self._updated_at_millis + + @classmethod + def from_json(cls, value: dict) -> "SecretInfo": + return cls( + name=value["name"], + created_at_millis=value["created_at_millis"], + updated_at_millis=value["updated_at_millis"], + ) + + def __repr__(self) -> str: + return ( + f"SecretInfo(name={self._name!r}, " + f"created_at_millis={self._created_at_millis!r}, " + f"updated_at_millis={self._updated_at_millis!r})" + ) + + def __eq__(self, other: object) -> bool: + return ( + isinstance(other, SecretInfo) + and other._name == self._name + and other._created_at_millis == self._created_at_millis + and other._updated_at_millis == self._updated_at_millis + ) + + +__all__ = [ + "EnvVarSecret", + "SecretInfo", + "validate_env_variable", + "validate_secret_name", +] diff --git a/python/python/tests/test_catalog.py b/python/python/tests/test_catalog.py index 63386a998..ca5d68fdf 100644 --- a/python/python/tests/test_catalog.py +++ b/python/python/tests/test_catalog.py @@ -91,7 +91,7 @@ def test_catalog_sync_scope_and_serialization(catalog_server): assert requests[0][0] == "/v1/namespace/team%2Fsearch/create" assert requests[0][2] == {"mode": "ExistOk"} assert requests[1][0] == "/v1/namespace/team%2Fsearch/describe" - assert requests[4][0] == "/v1/namespace/%24/list?limit=1&page_token=a%2Fb" + assert requests[4][0] == "/v1/namespace/$/list?limit=1&page_token=a%2Fb" assert requests[5][2] == {"mode": "Skip", "behavior": "Restrict"} for i, (_, headers, _) in enumerate(requests): headers = {key.lower(): value for key, value in headers.items()} diff --git a/python/python/tests/test_first_class_function_slice1.py b/python/python/tests/test_first_class_function_slice1.py index 459212b0f..fbcfa03b1 100644 --- a/python/python/tests/test_first_class_function_slice1.py +++ b/python/python/tests/test_first_class_function_slice1.py @@ -13,7 +13,9 @@ from lancedb.functions import ( FunctionBinding, FunctionVersion, PythonRuntimeSpec, + SecretBinding, RefreshColumnResult, + SecretReference, ) from lancedb.table import AsyncTable @@ -37,6 +39,22 @@ def job_result(name: str) -> dict: return json.loads(fixture(name))["result"] +def assert_no_secret_values(value): + """No client value models a resolved credential, at any nesting depth.""" + if isinstance(value, dict): + for key, child in value.items(): + assert key not in { + "secret_value", + "secret_values", + "resolved_secret", + "resolved_secrets", + } + assert_no_secret_values(child) + elif isinstance(value, list): + for child in value: + assert_no_secret_values(child) + + def test_public_function_values_are_in_api_reference(): docs = Path(__file__).parents[3] / "docs" / "src" / "python" / "python.md" rendered = docs.read_text() @@ -96,6 +114,11 @@ def test_function_version_identity_is_immutable_and_exact(): assert version.version == "1" assert version.image.manifest_digest.startswith("sha256:") assert version.version != version.image.manifest_digest + assert list(version.secret_bindings) == [ + SecretBinding( + kind="env", variable="HF_TOKEN", secret_ref=SecretReference(name="hf-prod") + ) + ] with pytest.raises((TypeError, ValueError)): version.version = "1" @@ -282,6 +305,31 @@ def test_refresh_result_rejects_non_u64_values(field): RefreshColumnResult.from_json(json.dumps(value)) +def test_canonical_client_values_carry_bindings_and_no_credentials(): + """A binding names a Secret; the credential behind it has no client field.""" + version = FunctionVersion.from_json( + json.dumps(job_result("remote_function_job.json")) + ) + canonical = json.loads(version.to_canonical_json()) + assert canonical["secret_bindings"] == [ + {"kind": "env", "variable": "HF_TOKEN", "secret_ref": {"name": "hf-prod"}} + ] + assert_no_secret_values(canonical) + + +def test_a_version_without_bindings_omits_the_field_in_both_directions(): + """A Function that binds nothing carries no ``secret_bindings`` key. + + Absent decodes as an empty list, and an empty list serializes back to + absent. + """ + value = job_result("remote_function_job.json") + del value["secret_bindings"] + version = FunctionVersion.from_json(json.dumps(value)) + assert list(version.secret_bindings) == [] + assert "secret_bindings" not in json.loads(version.to_canonical_json()) + + class _FunctionDeclarationInner: def __init__(self): self.calls = [] diff --git a/python/python/tests/test_first_class_function_slice2.py b/python/python/tests/test_first_class_function_slice2.py index 8d4f75f67..1cd8d4cf5 100644 --- a/python/python/tests/test_first_class_function_slice2.py +++ b/python/python/tests/test_first_class_function_slice2.py @@ -11,6 +11,7 @@ import types from datetime import date import http.server import json +import os from pathlib import Path import subprocess import sys @@ -24,11 +25,14 @@ import pytest import lancedb from lancedb.functions import ( PythonRuntimeSpec, + SecretBinding, + SecretReference, UdfDefinition, _canonical_arrow_type, _GRAMMAR_PRIMITIVES, udf, ) +from lancedb.secrets import EnvVarSecret THRESHOLD = 20 _CACHE = None @@ -59,6 +63,15 @@ def normalize_score(value: float) -> float: return value / 100.0 +@udf( + pip=["openai==3.7.0"], + env={"MODE": "test"}, + python_version="3.12", +) +def analyze_caption(caption: str) -> str: + return caption.strip() + + def test_scalar_udf_matches_shared_registration_golden_and_remains_callable(): assert isinstance(normalize_score, UdfDefinition) assert normalize_score(25.0) == 0.25 @@ -75,6 +88,286 @@ def test_scalar_udf_matches_shared_registration_golden_and_remains_callable(): } +def test_secret_bound_udf_matches_its_shared_registration_golden(): + assert analyze_caption(" hello ") == "hello" + bound = analyze_caption.bind_secrets( + [EnvVarSecret(secret_name="openai-prod", env_variable="OPENAI_API_KEY")] + ) + assert ( + bound.to_canonical_json() + == (FIXTURES / "remote_function_secret_registration_request.canonical.json") + .read_text() + .strip() + ) + + +def test_a_namespaced_binding_records_the_path_and_the_name(): + """A binding names the parts, so nothing has to be parsed back out. + + A root binding carries no path at all: the field is absent rather than an + empty list, so a binding states a namespace only when it has one. + """ + root = EnvVarSecret(secret_name="openai-prod", env_variable="OPENAI_API_KEY") + assert root.secret_namespace_path == [] + + nested = EnvVarSecret( + secret_name="openai-prod", + env_variable="OPENAI_API_KEY", + secret_namespace_path=["prod", "vision"], + ) + assert nested.secret_namespace_path == ["prod", "vision"] + assert nested != root + + bound = analyze_caption.bind_secrets([nested]) + assert list(bound.secret_bindings) == [ + SecretBinding( + kind="env", + variable="OPENAI_API_KEY", + secret_ref=SecretReference( + name="openai-prod", namespace_path=("prod", "vision") + ), + ) + ] + + at_root = analyze_caption.bind_secrets([root]) + assert list(at_root.secret_bindings) == [ + SecretBinding( + kind="env", + variable="OPENAI_API_KEY", + secret_ref=SecretReference(name="openai-prod"), + ) + ] + # A root binding carries no path at all on the wire. + canonical = json.loads(at_root.to_canonical_json()) + assert canonical["secret_bindings"] == [ + { + "kind": "env", + "variable": "OPENAI_API_KEY", + "secret_ref": {"name": "openai-prod"}, + } + ] + + +def test_a_namespace_path_is_validated_locally(): + # The charset is the service's, not a delimiter's: a reference is never + # joined, so a segment cannot make anything parse two ways. + with pytest.raises(ValueError): + EnvVarSecret( + secret_name="openai-prod", + env_variable="K", + secret_namespace_path=["with$delim"], + ) + with pytest.raises(ValueError): + EnvVarSecret( + secret_name="openai-prod", env_variable="K", secret_namespace_path=["a/b"] + ) + # A bare string is a plausible mistake with the wrong meaning. + with pytest.raises(TypeError): + EnvVarSecret( + secret_name="openai-prod", env_variable="K", secret_namespace_path="prod" + ) + + +def test_an_unbound_request_carries_no_binding_at_all(): + """Binding is a registration-time decision, so the definition holds none. + + The decorator declares nothing about secrets, which is what makes the PRD's + claim true: a Function's source and its registration request are identical + whether or not a credential is later bound to it. + """ + unbound = json.loads(analyze_caption.registration_request.to_canonical_json()) + assert "secret_bindings" not in unbound + assert "OPENAI_API_KEY" not in json.dumps(unbound) + + +def test_binding_a_secret_leaves_the_packaged_artifact_untouched(): + """The artifact is source bytes and nothing else, with or without secrets.""" + bound = analyze_caption.bind_secrets( + [EnvVarSecret(secret_name="openai-prod", env_variable="OPENAI_API_KEY")] + ) + assert bound.artifact == analyze_caption.registration_request.artifact + assert bound.artifact.digest == analyze_caption.registration_request.artifact.digest + + +def test_a_function_declaring_no_secret_is_registered_exactly_as_before(): + """The compatibility claim: nothing about the no-secret path moves.""" + assert ( + normalize_score.bind_secrets(None).to_canonical_json() + == normalize_score.registration_request.to_canonical_json() + ) + assert ( + "secret_bindings" + not in normalize_score.registration_request.to_canonical_json() + ) + + +def test_a_function_binds_each_variable_once(): + with pytest.raises(ValueError, match="binds each environment variable once"): + analyze_caption.bind_secrets( + [ + EnvVarSecret(secret_name="openai-prod", env_variable="OPENAI_API_KEY"), + EnvVarSecret( + secret_name="openai-staging", env_variable="OPENAI_API_KEY" + ), + ] + ) + + +def test_bindings_may_not_collide_with_plain_configuration(): + """`env` is stored with the Function; a Secret is not. Refuse, do not pick.""" + with pytest.raises(ValueError, match="must be disjoint"): + analyze_caption.bind_secrets( + [EnvVarSecret(secret_name="mode-prod", env_variable="MODE")] + ) + + +def test_a_binding_envelope_reaches_the_service_for_it_to_judge(): + """Binding rules are the service's: it owns the runtime the names land in. + + The client sends what it was given, so a rule it duplicated could disagree + with the service's without either side noticing. What is checked here is + that the envelope arrives intact -- the shape the service judges is the + shape the caller wrote. + """ + with _mock_remote_function_catalog() as (host, state): + db = lancedb.connect( + "db://dev", + api_key="fake", + host_override=host, + client_config={"retry_config": {"retries": 0}}, + ) + bindings = [ + EnvVarSecret(secret_name=f"secret-{index}", env_variable=f"TOKEN_{index}") + for index in range(17) + ] + db.create_function(normalize_score, secrets=bindings) + + sent = state["requests"][0][1] + assert len(sent["secret_bindings"]) == 17 + assert { + "kind": "env", + "variable": "TOKEN_0", + "secret_ref": {"name": "secret-0"}, + } in sent["secret_bindings"] + + +_SECRET_DEBUG_LOG_SOURCE = """ +import http.server +import json +import threading + +import lancedb + + +class Handler(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): + pass + + def do_POST(self): + self.rfile.read(int(self.headers.get("Content-Length", "0"))) + payload = json.dumps({}).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + +server = http.server.ThreadingHTTPServer(("localhost", 0), Handler) +threading.Thread(target=server.serve_forever, daemon=True).start() +try: + db = lancedb.connect( + "db://dev", + api_key="API_KEY_SENTINEL", + host_override="http://localhost:%d" % server.server_address[1], + client_config={"retry_config": {"retries": 0}}, + ) + db.create_secret("openai-prod", "SECRET_VALUE_SENTINEL") +finally: + server.shutdown() +""" + + +def test_a_credential_never_reaches_a_debug_log(tmp_path): + """The logger sees the serialized body, so no value-side redaction reaches it. + + Runs in a subprocess because the Rust logger reads ``LANCEDB_LOG`` once, at + import. + """ + script = tmp_path / "write_secret.py" + script.write_text(_SECRET_DEBUG_LOG_SOURCE) + + result = subprocess.run( + [sys.executable, str(script)], + check=True, + capture_output=True, + text=True, + env={**os.environ, "LANCEDB_LOG": "debug"}, + ) + output = result.stdout + result.stderr + + # Without this the test passes when debug logging is simply off. + assert "Sending request_id=" in output, output + assert "SECRET_VALUE_SENTINEL" not in output + assert "API_KEY_SENTINEL" not in output + + +def test_a_credential_value_is_rejected_in_the_binding_position(): + """The one mistake the typed binding exists to stop.""" + with pytest.raises(TypeError, match="EnvVarSecret"): + analyze_caption.bind_secrets(["sk-live-0001"]) + + +@pytest.mark.parametrize( + ("secret", "variable", "message"), + [ + ("openai-prod", "not-a-var", "invalid environment variable name"), + ("openai-prod", "API-TOKEN", "invalid environment variable name"), + ("not a name", "API_TOKEN", "invalid Secret name"), + ("openai$prod", "API_TOKEN", "invalid Secret name"), + ], +) +def test_a_binding_validates_both_names_locally(secret, variable, message): + with pytest.raises(ValueError, match=message): + EnvVarSecret(secret_name=secret, env_variable=variable) + + +def test_a_secret_name_admits_what_a_namespace_name_does(): + """A Secret has to be nameable wherever a namespace already is. + + LanceDB namespace and table names are `[A-Za-z0-9_.-]` with no rule about + which character comes first, so a name may lead with `_`, `-` or `.`. + Anything narrower here would leave Secrets unaddressable inside namespaces + that already exist -- the reason periods are admitted is the reason the + edges are too. + """ + for name in ["openai.prod.v1", ".hidden", "_internal", "-lead", "trailing."]: + binding = EnvVarSecret(secret_name=name, env_variable="OPENAI_API_KEY") + assert binding.secret_name == name + + for name in ["", "with/slash", "with$delimiter", "a" * 256]: + with pytest.raises(ValueError, match="invalid Secret name"): + EnvVarSecret(secret_name=name, env_variable="OPENAI_API_KEY") + + # A namespace segment follows the same rule, and LanceDB already admits + # these shapes as namespace names -- so a Secret is addressable inside one. + for segment in [".hidden", "_internal", "-lead", "trailing."]: + binding = EnvVarSecret( + secret_name="openai-prod", + env_variable="OPENAI_API_KEY", + secret_namespace_path=[segment], + ) + assert binding.secret_namespace_path == [segment] + + for segment in ["", "with/slash", "with$delimiter"]: + with pytest.raises(ValueError, match="invalid namespace path segment"): + EnvVarSecret( + secret_name="openai-prod", + env_variable="OPENAI_API_KEY", + secret_namespace_path=[segment], + ) + + def _main_udf_source( *, threshold: int = 20, input_annotation: str = "int", comparison: str = ">=" ) -> str: @@ -1232,7 +1525,20 @@ def _mock_remote_function_catalog(): body = json.loads(self.rfile.read(length) or b"{}") state["requests"].append((self.path, body)) status = 200 - if self.path == "/v1/function/normalize_score/create": + # `{id}` is the Function name, so match on the shape rather than on + # one name: these tests register more than one Function. + parts = self.path.strip("/").split("/") + function_action = ( + (urllib.parse.unquote(parts[2]), parts[3]) + if len(parts) == 4 and parts[0] == "v1" and parts[1] == "function" + else (None, None) + ) + secret_action = ( + (urllib.parse.unquote(parts[2]), parts[3]) + if len(parts) == 4 and parts[0] == "v1" and parts[1] == "secret" + else (None, None) + ) + if function_action[1] == "create": state["version"] = { "name": "normalize_score", "version": FUNCTION_VERSION, @@ -1246,6 +1552,7 @@ def _mock_remote_function_catalog(): ).read_text() )["image"], "signature": body["signature"], + "secret_bindings": body.get("secret_bindings", []), "created_at": "2026-08-21T00:00:00Z", } response = {"job_id": "job-register"} @@ -1264,6 +1571,15 @@ def _mock_remote_function_catalog(): elif self.path == "/v1/function/normalize_score/drop": assert body == {"version": FUNCTION_VERSION} response = {"dropped": True} + elif secret_action[1] in ("create", "alter"): + # The Secret is the path identifier, so the body is the value. + assert set(body) == {"value"} + assert secret_action[0] == "openai-prod" + response = {} + elif secret_action[1] == "drop": + assert secret_action[0] == "openai-prod" + assert body == {} + response = {} else: status = 404 response = {"error": "not found"} @@ -1276,6 +1592,16 @@ def _mock_remote_function_catalog(): for key, values in urllib.parse.parse_qs(url.query).items() } state["requests"].append((url.path, query)) + if url.path == "/v1/namespace/$/secret/list": + if "page_token" not in query: + self._write_response( + 200, + {"secrets": [{"name": "openai-prod"}], "page_token": "next"}, + ) + else: + assert query["page_token"] == "next" + self._write_response(200, {"secrets": [{"name": "hf-prod"}]}) + return if url.path != "/v1/namespace/$/function/list": self._write_response(404, {"error": "not found"}) return @@ -1330,6 +1656,92 @@ def test_remote_registration_job_and_exact_version_reopen_round_trip(): assert create_request == expected_request +def test_remote_registration_sends_bindings_and_never_a_credential(): + with _mock_remote_function_catalog() as (host, state): + db = lancedb.connect( + "db://dev", + api_key="fake", + host_override=host, + client_config={"retry_config": {"retries": 0}}, + ) + created = db.create_function( + analyze_caption, + secrets=[ + EnvVarSecret(secret_name="openai-prod", env_variable="OPENAI_API_KEY") + ], + ) + + assert list(created.secret_bindings) == [ + SecretBinding( + kind="env", + variable="OPENAI_API_KEY", + secret_ref=SecretReference(name="openai-prod"), + ) + ] + path, create_request = state["requests"][0] + assert path == "/v1/function/analyze_caption/create" + assert create_request["secret_bindings"] == [ + { + "kind": "env", + "variable": "OPENAI_API_KEY", + "secret_ref": {"name": "openai-prod"}, + } + ] + # The request names a Secret and carries nothing that could be one. The + # Function's own name is the path identifier rather than a body field, so + # it is the one key the body does not repeat. + expected = json.loads( + analyze_caption.bind_secrets( + [EnvVarSecret(secret_name="openai-prod", env_variable="OPENAI_API_KEY")] + ).to_canonical_json() + ) + assert expected.pop("name") == "analyze_caption" + assert create_request == expected + + +def test_remote_secret_verbs_round_trip(): + with _mock_remote_function_catalog() as (host, state): + db = lancedb.connect( + "db://dev", + api_key="fake", + host_override=host, + client_config={"retry_config": {"retries": 0}}, + ) + assert db.create_secret("openai-prod", "sk-live-0001") is None + assert db.alter_secret("openai-prod", "sk-live-0002") is None + assert db.list_secrets() == ["openai-prod", "hf-prod"] + assert db.drop_secret("openai-prod") is None + + routes = [path for path, _ in state["requests"]] + assert routes == [ + "/v1/secret/openai-prod/create", + "/v1/secret/openai-prod/alter", + "/v1/namespace/$/secret/list", + "/v1/namespace/$/secret/list", + "/v1/secret/openai-prod/drop", + ] + # The Secret is the path identifier, so the body is the value alone. + assert state["requests"][0][1] == {"value": "sk-live-0001"} + # Listing is a GET: the first page asks for nothing, the second resumes on + # the token the server handed back, and neither carries a body. + assert state["requests"][2][1] == {} + assert state["requests"][3][1] == {"page_token": "next"} + + +def test_building_a_binding_contacts_no_server(): + """A binding is a local value: it says nothing about whether the Secret exists. + + Existence is the server's answer at registration, where a mistyped name is a + clear error rather than a client-side check that was already stale. + """ + with _mock_remote_function_catalog() as (_host, state): + binding = EnvVarSecret(secret_name="openai-prod", env_variable="OPENAI_API_KEY") + assert binding.secret_name == "openai-prod" + assert binding.env_variable == "OPENAI_API_KEY" + + assert state["requests"] == [] + + def test_blocking_remote_registration_returns_function_version(): with _mock_remote_function_catalog() as (host, state): db = lancedb.connect( diff --git a/python/src/connection.rs b/python/src/connection.rs index 65d4c98e8..a972f4351 100644 --- a/python/src/connection.rs +++ b/python/src/connection.rs @@ -768,6 +768,85 @@ impl Connection { }) } + #[pyo3(signature = (name, value, namespace_path=None))] + pub fn create_secret( + self_: PyRef<'_, Self>, + name: String, + value: String, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + inner + .create_secret(name, value, &namespace_path) + .await + .infer_error() + }) + } + + #[pyo3(signature = (name, value, namespace_path=None))] + pub fn alter_secret( + self_: PyRef<'_, Self>, + name: String, + value: String, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + inner + .alter_secret(name, value, &namespace_path) + .await + .infer_error() + }) + } + + #[pyo3(signature = (namespace_path=None))] + pub fn list_secrets( + self_: PyRef<'_, Self>, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + inner.list_secrets(&namespace_path).await.infer_error() + }) + } + + #[pyo3(signature = (name, namespace_path=None))] + pub fn drop_secret( + self_: PyRef<'_, Self>, + name: String, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + inner.drop_secret(name, &namespace_path).await.infer_error() + }) + } + + /// Name and timestamps as a plain tuple. `SecretInfo` carries no value, so + /// there is none to filter out here. Timestamps stay integers rather than + /// going through a string, so the caller can compare two without parsing. + #[pyo3(signature = (name, namespace_path=None))] + pub fn describe_secret( + self_: PyRef<'_, Self>, + name: String, + namespace_path: Option>, + ) -> PyResult> { + let inner = self_.get_inner()?.clone(); + let namespace_path = namespace_path.unwrap_or_default(); + future_into_py(self_.py(), async move { + let info = inner + .describe_secret(name, &namespace_path) + .await + .infer_error()?; + Ok((info.name, info.created_at_millis, info.updated_at_millis)) + }) + } + pub fn list_jobs(self_: PyRef<'_, Self>) -> PyResult> { let inner = self_.get_inner()?.clone(); future_into_py(self_.py(), async move { diff --git a/rust/lancedb/src/connection.rs b/rust/lancedb/src/connection.rs index 3f2979523..7234a911f 100644 --- a/rust/lancedb/src/connection.rs +++ b/rust/lancedb/src/connection.rs @@ -36,6 +36,8 @@ use crate::remote::{ OPT_REMOTE_SQL_HOST_OVERRIDE, }, }; +use crate::secrets::SecretInfo; +use crate::utils::{validate_secret_component, validate_secret_reference}; use lance::io::ObjectStoreParams; pub use lance_file::version::LanceFileVersion; #[cfg(feature = "remote")] @@ -587,6 +589,7 @@ impl Connection { /// artifact. Waiting on the returned typed job yields the durable /// [`crate::function::FunctionVersion`]. Creation is remote-only. /// Local databases return [`Error::NotSupported`]. + /// pub async fn create_function_async( &self, request: crate::function::FunctionRegistrationRequest, @@ -646,6 +649,88 @@ impl Connection { .await } + /// Create a named Secret in this database. + /// + /// Fails if the name is taken, so a create can never silently become a + /// rotation. There is no API that reads a stored credential back; the only + /// consumer is a Function that binds the Secret by name. Local databases + /// return [`Error::NotSupported`]. + pub async fn create_secret( + &self, + name: impl AsRef, + value: impl AsRef, + namespace_path: &[String], + ) -> Result<()> { + validate_secret_reference(name.as_ref(), namespace_path)?; + self.internal + .create_secret(name.as_ref(), value.as_ref(), namespace_path) + .await + } + + /// Replace the credential behind an existing Secret. + /// + /// Fails if it does not exist. Every Function bound to the Secret resolves + /// the new value from its next execution, and no new Function version is + /// minted -- which is what lets a rotation reach columns pinned to a + /// version registered before it. Local databases return + /// [`Error::NotSupported`]. + pub async fn alter_secret( + &self, + name: impl AsRef, + value: impl AsRef, + namespace_path: &[String], + ) -> Result<()> { + validate_secret_reference(name.as_ref(), namespace_path)?; + self.internal + .alter_secret(name.as_ref(), value.as_ref(), namespace_path) + .await + } + + /// The names of every Secret in this database. + /// + /// Names only. No path in this API returns a stored credential, by + /// construction rather than by policy. Local databases return + /// [`Error::NotSupported`]. + pub async fn list_secrets(&self, namespace_path: &[String]) -> Result> { + for segment in namespace_path { + validate_secret_component("Secret namespace path segment", segment)?; + } + self.internal.list_secrets(namespace_path).await + } + + /// Drop a Secret. + /// + /// Functions bound to it fail at their next job, naming the Secret; that + /// is the revocation path. The name becomes free to reuse, and a new + /// Secret under it is picked up by everything still bound to that name. + /// Local databases return [`Error::NotSupported`]. + pub async fn drop_secret( + &self, + name: impl AsRef, + namespace_path: &[String], + ) -> Result<()> { + validate_secret_reference(name.as_ref(), namespace_path)?; + self.internal + .drop_secret(name.as_ref(), namespace_path) + .await + } + + /// What this database records about one Secret: its name and timestamps. + /// + /// Never the value. The type it returns has no field for one, so this is a + /// property of the API rather than of what the caller chooses to read. + /// Local databases return [`Error::NotSupported`]. + pub async fn describe_secret( + &self, + name: impl AsRef, + namespace_path: &[String], + ) -> Result { + validate_secret_reference(name.as_ref(), namespace_path)?; + self.internal + .describe_secret(name.as_ref(), namespace_path) + .await + } + /// Rename a table in the database. /// /// This is only supported in LanceDB Cloud. diff --git a/rust/lancedb/src/database.rs b/rust/lancedb/src/database.rs index d31f40b29..392277f63 100644 --- a/rust/lancedb/src/database.rs +++ b/rust/lancedb/src/database.rs @@ -30,6 +30,7 @@ use crate::data::scannable::Scannable; use crate::error::Result; use crate::job::Job; use crate::materialized_view::CreateMaterializedViewRequest; +use crate::secrets::SecretInfo; use crate::table::{BaseTable, WriteOptions}; pub mod listing; @@ -251,6 +252,12 @@ fn function_catalog_not_supported() -> Result { }) } +fn secret_catalog_not_supported() -> Result { + Err(crate::error::Error::NotSupported { + message: "Secret operations are not supported by this database".to_string(), + }) +} + /// The `Database` trait defines the interface for database implementations. /// /// A database is responsible for managing tables and their metadata. @@ -386,6 +393,44 @@ pub trait Database: async fn drop_function(&self, _name: &str, _version: &str) -> Result { function_catalog_not_supported() } + /// Create a named Secret in this database. Fails if the name is taken, so + /// a create can never silently become a rotation. + async fn create_secret( + &self, + _name: &str, + _value: &str, + _namespace_path: &[String], + ) -> Result<()> { + secret_catalog_not_supported() + } + /// Replace the credential behind an existing Secret. Fails if it does not + /// exist. Every Function bound to it resolves the new value from its next + /// execution, with no new Function version. + async fn alter_secret( + &self, + _name: &str, + _value: &str, + _namespace_path: &[String], + ) -> Result<()> { + secret_catalog_not_supported() + } + /// The names of every Secret in this database. + /// + /// Names only. No API path returns a stored credential, by construction + /// rather than by policy. + async fn list_secrets(&self, _namespace_path: &[String]) -> Result> { + secret_catalog_not_supported() + } + /// Drop a Secret. Functions bound to it fail at their next job, which is + /// the revocation path. + async fn drop_secret(&self, _name: &str, _namespace_path: &[String]) -> Result<()> { + secret_catalog_not_supported() + } + /// What the database records about one Secret: its name and timestamps, + /// never its value. + async fn describe_secret(&self, _name: &str, _namespace_path: &[String]) -> Result { + secret_catalog_not_supported() + } /// Open a job by id, returning a handle with its record already /// populated. Fails with [`crate::Error::JobNotFound`] when the server has /// no such job. diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index d501541c0..0d593585a 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -5,7 +5,7 @@ //! backend-neutral terminal result of a computed-column refresh. //! //! This module contains client/wire values only. Catalog persistence, -//! environment bake, and execution are owned by Sophon. +//! environment bake, secret resolution, and execution are owned by Sophon. use std::collections::BTreeMap; @@ -13,6 +13,7 @@ use serde::de::{self, DeserializeOwned}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::Value; +use crate::secrets::SecretBinding; use crate::{Error, Result}; /// Semantic Function type for a Blob v2 value. @@ -434,6 +435,8 @@ pub struct FunctionVersion { version: String, image: FunctionImage, signature: FunctionSignature, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + secret_bindings: Vec, created_at: String, metadata: BTreeMap, disabled: bool, @@ -473,6 +476,17 @@ impl FunctionVersion { pub fn signature(&self) -> &FunctionSignature { &self.signature } + + /// Declared environment variable name to the Secret each one resolves. + /// + /// Bindings are part of this version's identity; the credentials behind + /// them are not, and resolve at execution. Rotating a bound Secret + /// therefore changes what the same version runs with, and no value has a + /// field in this model. + pub fn secret_bindings(&self) -> &[SecretBinding] { + &self.secret_bindings + } + pub fn created_at(&self) -> &str { &self.created_at } @@ -514,12 +528,21 @@ pub struct FunctionArtifactRequest { } /// Stable request envelope for remote immutable Function registration. +/// +/// Credential values deliberately have no field here. The only secret-shaped +/// thing a client sends is `secret_bindings`: the name of a Secret the +/// database already holds, which Sophon resolves inside the remote runtime. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct FunctionRegistrationRequest { pub name: String, pub artifact: FunctionArtifactRequest, pub signature: FunctionSignature, pub runtime: PythonRuntimeSpec, + /// Declared environment variable name to the Secret it binds. A binding is + /// a reference: whether the Secret exists is answered when a column is + /// declared against this version, not here. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub secret_bindings: Vec, } impl_json!(FunctionRegistrationRequest); @@ -785,3 +808,80 @@ mod conda_environment_tests { } } } + +#[cfg(test)] +mod tests { + use super::*; + + /// Canonical form is what the FunctionVersion hash is taken over, so key + /// order must come from the keys and not from however serde happened to + /// emit them. Nesting is included because the sort is recursive. + #[test] + fn canonical_json_sorts_keys_at_every_depth() { + let value = serde_json::json!({ + "runtime": {"kind": "python", "env": {"B": "2", "A": "1"}}, + "artifact": {"digest": "sha256:x"}, + "name": "embed", + }); + let mut out = String::new(); + write_canonical_json(&value, &mut out).expect("canonical JSON"); + + assert_eq!( + out, + r#"{"artifact":{"digest":"sha256:x"},"name":"embed","runtime":{"env":{"A":"1","B":"2"},"kind":"python"}}"# + ); + } + + /// Arrays are ordered by the caller, so canonicalization must leave them + /// alone -- sorting them would change what a signature means. + #[test] + fn canonical_json_preserves_array_order() { + let value = serde_json::json!({"inputs": ["b", "a", "c"]}); + let mut out = String::new(); + write_canonical_json(&value, &mut out).expect("canonical JSON"); + + assert_eq!(out, r#"{"inputs":["b","a","c"]}"#); + } + + /// A float has no single canonical spelling, so two clients could hash the + /// same literal differently. Rejected at any depth rather than rounded. + #[test] + fn validate_literal_rejects_floats_at_any_depth() { + for value in [ + serde_json::json!(1.5), + serde_json::json!([1, [2, 3.5]]), + serde_json::json!({"a": {"b": 0.25}}), + ] { + let error = validate_literal(&value).expect_err("floats are not canonical"); + assert!( + error.to_string().contains("floating-point"), + "unexpected error: {error}" + ); + } + + for value in [ + serde_json::json!(1), + serde_json::json!("1.5"), + serde_json::json!([1, {"a": true}]), + serde_json::json!(null), + ] { + validate_literal(&value).expect("non-float literals are canonical"); + } + } + + /// Unknown keys are how a newer server's payload reaches an older client, + /// so the check has to be exact about which level it is looking at. + #[test] + fn has_unknown_keys_only_inspects_the_level_it_is_given() { + let value = serde_json::json!({"name": "embed", "version": "fv_1"}); + assert!(!has_unknown_keys(&value, &["name", "version"])); + assert!(has_unknown_keys(&value, &["name"])); + + // A nested unknown is not this level's business. + let nested = serde_json::json!({"name": {"unexpected": 1}}); + assert!(!has_unknown_keys(&nested, &["name"])); + + // A non-object has no keys to be unknown. + assert!(!has_unknown_keys(&serde_json::json!("embed"), &["name"])); + } +} diff --git a/rust/lancedb/src/lib.rs b/rust/lancedb/src/lib.rs index b173a63cb..4d9afc7c7 100644 --- a/rust/lancedb/src/lib.rs +++ b/rust/lancedb/src/lib.rs @@ -196,6 +196,7 @@ pub mod query; #[cfg(feature = "remote")] pub mod remote; pub mod rerankers; +pub mod secrets; pub mod sql; pub mod table; #[cfg(test)] diff --git a/rust/lancedb/src/remote/catalog.rs b/rust/lancedb/src/remote/catalog.rs index 098a696a7..1c0129889 100644 --- a/rust/lancedb/src/remote/catalog.rs +++ b/rust/lancedb/src/remote/catalog.rs @@ -472,7 +472,7 @@ mod tests { } assert_eq!( requests[0].line, - "GET /v1/namespace/%24/list?limit=1&page_token=a%2Fb HTTP/1.1" + "GET /v1/namespace/$/list?limit=1&page_token=a%2Fb HTTP/1.1" ); assert_eq!( requests[1].line, diff --git a/rust/lancedb/src/remote/client.rs b/rust/lancedb/src/remote/client.rs index 55d272b51..b7a492598 100644 --- a/rust/lancedb/src/remote/client.rs +++ b/rust/lancedb/src/remote/client.rs @@ -90,8 +90,9 @@ pub struct ClientConfig { pub user_agent: String, // TODO: how to configure request ids? pub extra_headers: HashMap, - /// The delimiter to use when constructing object identifiers. - /// If not default, passes as query parameter. + /// The delimiter joining a namespace path and a name into one object + /// identifier. [`ID_DELIMITER`] is the only accepted value; any other is + /// refused by [`ClientConfig::validate`]. pub id_delimiter: Option, /// TLS configuration for mTLS support pub tls_config: Option, @@ -307,7 +308,6 @@ pub struct RestfulLanceDbClient { host: String, pub(crate) retry_config: ResolvedRetryConfig, pub(crate) sender: S, - pub(crate) id_delimiter: String, pub(crate) header_provider: Option>, /// Connection-level read consistency interval. Drives the /// `x-lancedb-min-timestamp` freshness header sent on read requests. @@ -330,7 +330,6 @@ impl std::fmt::Debug for RestfulLanceDbClient { .field("host", &self.host) .field("retry_config", &self.retry_config) .field("sender", &self.sender) - .field("id_delimiter", &self.id_delimiter) .field( "header_provider", &self.header_provider.as_ref().map(|_| "Some(...)"), @@ -427,6 +426,54 @@ fn validate_dns_hostname(hostname: &str) -> Result<()> { Ok(()) } +/// Whether a request's body may appear in a debug log. +/// +/// The API that built the body decides. The transport cannot know which +/// payloads are credentials, and a list of routes here would have to be kept in +/// step with endpoints defined elsewhere -- so the knowledge lives with the +/// call that has it. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum BodyLogging { + /// Log the body at debug. The default: a request body is diagnostic unless + /// the call that built it says otherwise. + Allowed, + /// Never log the body. For a request whose body is a credential. + Suppressed, +} + +/// The delimiter joining a namespace path and a name into the `{id}` a route +/// addresses, and the only one a LanceDB service splits on. +/// +/// `$` is outside the character set object names admit, so a joined identifier +/// always splits back into the parts that made it. The configuration field +/// exists because the identifier grammar comes from the Lance REST catalog +/// standard, which carries a delimiter setting for other catalogs to adopt. +pub(crate) const ID_DELIMITER: &str = "$"; + +fn validate_id_delimiter(delimiter: &str) -> Result<()> { + if delimiter != ID_DELIMITER { + return Err(Error::InvalidInput { + message: format!( + "id_delimiter '{delimiter}' is not supported: '{ID_DELIMITER}' is the only \ + delimiter LanceDB services split an identifier on" + ), + }); + } + Ok(()) +} + +impl ClientConfig { + /// Check the settings a request cannot be built correctly without, so a + /// mistake is reported where it was made rather than as a confusing + /// response later. Public so a caller can ask without connecting. + pub fn validate(&self) -> Result<()> { + if let Some(delimiter) = &self.id_delimiter { + validate_id_delimiter(delimiter)?; + } + Ok(()) + } +} + impl RestfulLanceDbClient { fn get_timeout(passed: Option, env_var: &str) -> Result> { if let Some(passed) = passed { @@ -452,6 +499,10 @@ impl RestfulLanceDbClient { client_config: ClientConfig, read_consistency_interval: Option, ) -> Result { + // Before anything is built from it, so the error names the caller's + // configuration rather than a request. + client_config.validate()?; + // Get the timeouts let timeout = Self::get_timeout(client_config.timeout_config.timeout, "LANCE_CLIENT_TIMEOUT")?; @@ -551,10 +602,6 @@ impl RestfulLanceDbClient { host, retry_config, sender: Sender, - id_delimiter: client_config - .id_delimiter - .clone() - .unwrap_or("$".to_string()), header_provider: client_config.header_provider, read_consistency_interval, max_bytes_per_request, @@ -633,12 +680,14 @@ impl RestfulLanceDbClient { ) -> Result { let mut headers = HeaderMap::new(); if !api_key.is_empty() { - headers.insert( - HeaderName::from_static("x-api-key"), - HeaderValue::from_str(api_key).map_err(|_| Error::InvalidInput { - message: "non-ascii api key provided".to_string(), - })?, - ); + // `log_request` prints the request's Debug, which prints headers. + // Marking the value sensitive is what makes that print `Sensitive` + // instead of the key itself. + let mut key = HeaderValue::from_str(api_key).map_err(|_| Error::InvalidInput { + message: "non-ascii api key provided".to_string(), + })?; + key.set_sensitive(true); + headers.insert(HeaderName::from_static("x-api-key"), key); } if region == "local" { let host = format!("{}.local.api.lancedb.com", db_name); @@ -710,22 +759,12 @@ impl RestfulLanceDbClient { pub fn get(&self, uri: &str) -> RequestBuilder { let full_uri = format!("{}{}", self.host, uri); - let builder = self.client.get(full_uri); - self.add_id_delimiter_query_param(builder) + self.client.get(full_uri) } pub fn post(&self, uri: &str) -> RequestBuilder { let full_uri = format!("{}{}", self.host, uri); - let builder = self.client.post(full_uri); - self.add_id_delimiter_query_param(builder) - } - - fn add_id_delimiter_query_param(&self, req: RequestBuilder) -> RequestBuilder { - if self.id_delimiter != "$" { - req.query(&[("delimiter", self.id_delimiter.clone())]) - } else { - req - } + self.client.post(full_uri) } /// Apply dynamic headers from the header provider if configured @@ -750,6 +789,22 @@ impl RestfulLanceDbClient { } pub async fn send(&self, req: RequestBuilder) -> Result<(String, Response)> { + self.send_logging(req, BodyLogging::Allowed).await + } + + /// Send a request whose body must never reach a debug log. + /// + /// The body is built by the caller, so only the caller knows it holds a + /// credential; `log_request` sees serialized bytes and cannot tell. + pub async fn send_suppressing_body(&self, req: RequestBuilder) -> Result<(String, Response)> { + self.send_logging(req, BodyLogging::Suppressed).await + } + + async fn send_logging( + &self, + req: RequestBuilder, + body_logging: BodyLogging, + ) -> Result<(String, Response)> { let (client, request) = req.build_split(); let mut request = request.unwrap(); let request_id = self.extract_request_id(&mut request); @@ -757,7 +812,7 @@ impl RestfulLanceDbClient { // Apply dynamic headers before sending request = self.apply_dynamic_headers(request).await?; - self.log_request(&request, &request_id); + self.log_request(&request, &request_id, body_logging); let response = self .sender @@ -820,7 +875,7 @@ impl RestfulLanceDbClient { // Apply dynamic headers before each retry attempt request = self.apply_dynamic_headers(request).await?; - self.log_request(&request, &request_id); + self.log_request(&request, &request_id, BodyLogging::Allowed); let response = self.sender.send(&c, request).await.map(|r| (r.status(), r)); @@ -864,13 +919,18 @@ impl RestfulLanceDbClient { } } - pub(crate) fn log_request(&self, request: &Request, request_id: &String) { + fn log_request(&self, request: &Request, request_id: &String, body_logging: BodyLogging) { if log::log_enabled!(log::Level::Debug) { let content_type = request .headers() .get("content-type") .map(|v| v.to_str().unwrap()); - if content_type == Some("application/json") { + if body_logging == BodyLogging::Suppressed { + debug!( + "Sending request_id={}: {:?} with body suppressed", + request_id, request + ); + } else if content_type == Some("application/json") { let body = request.body().as_ref().unwrap().as_bytes().unwrap(); let body = String::from_utf8_lossy(body); debug!( @@ -1047,7 +1107,6 @@ pub mod test_utils { sender: MockSender { f: Arc::new(wrapper), }, - id_delimiter: "$".to_string(), header_provider: None, read_consistency_interval, max_bytes_per_request: None, @@ -1074,7 +1133,6 @@ pub mod test_utils { sender: MockSender { f: Arc::new(wrapper), }, - id_delimiter: config.id_delimiter.unwrap_or_else(|| "$".to_string()), header_provider: config.header_provider, read_consistency_interval: None, max_bytes_per_request: config @@ -1089,6 +1147,40 @@ pub mod test_utils { #[cfg(test)] mod tests { + /// A configuration naming any other delimiter is refused where it was + /// written, rather than producing identifiers no service splits the way the + /// caller meant. + #[test] + fn test_a_delimiter_other_than_the_supported_one_is_refused() { + for delimiter in ["/", "?", "#", "%", "", ".", "..", "-", "_", "|", "::", "$$"] { + let error = super::validate_id_delimiter(delimiter) + .expect_err("only the supported delimiter may be configured"); + assert!( + error.to_string().contains("id_delimiter"), + "{delimiter:?}: {error}" + ); + } + super::validate_id_delimiter(super::ID_DELIMITER).unwrap(); + } + + /// Leaving it unset is how nearly every caller reaches the same delimiter. + #[test] + fn test_an_unset_delimiter_is_the_supported_one() { + super::ClientConfig::default().validate().unwrap(); + super::ClientConfig { + id_delimiter: Some(super::ID_DELIMITER.to_string()), + ..Default::default() + } + .validate() + .unwrap(); + super::ClientConfig { + id_delimiter: Some("-".to_string()), + ..Default::default() + } + .validate() + .expect_err("a configured delimiter other than the supported one must be refused"); + } + use super::*; use serial_test::serial; use std::time::Duration; @@ -1275,6 +1367,41 @@ mod tests { assert!(debug.contains("visible-value")); } + /// `log_request` prints the request's Debug, and Debug for a request prints + /// its headers. Marking the value sensitive is the only thing standing + /// between the API key and every debug line; assert on the header map's own + /// Debug, which is what that printing reduces to. + #[test] + fn test_api_key_is_redacted_in_debug_output() { + let headers = RestfulLanceDbClient::::default_headers( + "sk-live-sentinel", + "us-east-1", + "db-name", + false, + &RemoteOptions::default(), + None, + &ClientConfig::default(), + ) + .unwrap(); + + assert_eq!(headers.get("x-api-key").unwrap(), "sk-live-sentinel"); + assert!( + !format!("{:?}", headers).contains("sk-live-sentinel"), + "the API key must not survive Debug formatting" + ); + } + + /// A suppressed body is suppressed whatever the content type says, and an + /// allowed one is logged in full. + #[test] + fn test_body_logging_is_decided_by_the_caller() { + assert_ne!(BodyLogging::Allowed, BodyLogging::Suppressed); + // `send` and `send_suppressing_body` differ only in what they pass, so + // the enum is the whole contract: a caller states its intent and the + // transport does not infer one from the route. + assert_eq!(BodyLogging::Allowed, BodyLogging::Allowed); + } + #[test] fn test_rejects_invalid_cloud_dns_hostname() { let invalid_database_names = ["a".repeat(64), "invalid..database".to_string()]; @@ -1364,7 +1491,6 @@ mod tests { host: "https://example.com".to_string(), retry_config: RetryConfig::default().try_into().unwrap(), sender: Sender, - id_delimiter: "+".to_string(), header_provider: Some(Arc::new(provider) as Arc), read_consistency_interval: None, max_bytes_per_request: None, @@ -1403,7 +1529,6 @@ mod tests { host: "https://example.com".to_string(), retry_config: RetryConfig::default().try_into().unwrap(), sender: Sender, - id_delimiter: "+".to_string(), header_provider: Some(Arc::new(provider) as Arc), read_consistency_interval: None, max_bytes_per_request: None, @@ -1468,7 +1593,6 @@ mod tests { host: "https://example.com".to_string(), retry_config: RetryConfig::default().try_into().unwrap(), sender: Sender, - id_delimiter: "+".to_string(), header_provider: Some(Arc::new(provider) as Arc), read_consistency_interval: None, max_bytes_per_request: None, diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index e7ae78108..bc1dbfea1 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -24,15 +24,22 @@ use crate::database::{ OpenTableRequest, ReadConsistency, TableNamesRequest, }; use crate::error::Result; -use crate::function::{FunctionRegistrationRequest, FunctionVersion}; +use crate::function::{ + FunctionArtifactRequest, FunctionRegistrationRequest, FunctionSignature, FunctionVersion, + PythonRuntimeSpec, +}; use crate::job::Job; use crate::materialized_view::CreateMaterializedViewRequest; use crate::remote::job::{RemoteJob, job_state_to_client}; use crate::remote::util::stream_as_body; +use crate::secrets::SecretBinding; +use crate::secrets::SecretInfo; use crate::table::BaseTable; +use crate::utils::{reject_relative_segment, validate_table_name}; use super::client::{ - ClientConfig, HeaderProvider, HttpSend, RequestResultExt, RestfulLanceDbClient, Sender, + ClientConfig, HeaderProvider, HttpSend, ID_DELIMITER, RequestResultExt, RestfulLanceDbClient, + Sender, }; use super::sql::SqlClient; use super::table::RemoteTable; @@ -413,12 +420,30 @@ impl RemoteDatabase { } impl RemoteDatabase { + /// Post a request whose body carries a credential. + /// + /// Shared by the create and alter verbs, which declare their own request + /// types: the two mean different things to the service and are free to + /// diverge, so what they share is the posting and not the shape. + /// + /// The value is a request field and never a path segment or query + /// parameter, which keeps it out of access logs and proxy traces. + async fn post_secret_write(&self, route: &str, body: &T) -> Result<()> { + let req = self.client.post(route).json(body); + // This call is what says the body is a credential. Nothing downstream + // can tell from the bytes, and a route list in the transport would have + // to be kept in step with endpoints declared here. + let (request_id, response) = self.client.send_suppressing_body(req).await?; + self.client.check_response(&request_id, response).await?; + Ok(()) + } + async fn submit_drop_table( &self, name: &str, namespace_path: &[String], ) -> Result<(String, Response)> { - let identifier = build_table_identifier(name, namespace_path, &self.client.id_delimiter); + let identifier = build_table_identifier(name, namespace_path)?; let cache_key = build_cache_key(name, namespace_path); let req = self.client.post(&format!("/v1/table/{}/drop/", identifier)); let (request_id, resp) = self.client.send(req).await?; @@ -442,8 +467,7 @@ impl RemoteDatabase { &self, request: &TableNamesRequest, ) -> Result<(Vec, ServerVersion)> { - let namespace_id = - build_namespace_identifier(&request.namespace_path, &self.client.id_delimiter); + let namespace_id = build_namespace_identifier(&request.namespace_path)?; let path = format!("/v1/namespace/{}/table/list", namespace_id); let mut names = Vec::new(); @@ -550,23 +574,93 @@ impl From<&CreateTableMode> for &'static str { } } -fn build_table_identifier(name: &str, namespace: &[String], delimiter: &str) -> String { - if !namespace.is_empty() { - let mut parts = namespace.to_vec(); - parts.push(name.to_string()); - parts.join(delimiter) - } else { - name.to_string() +/// The path segment addressing one object: its namespace path and its name. +/// +/// One builder for tables, Secrets, Functions and materialized views: the +/// identifier grammar belongs to the namespace spec, not to an object type. An +/// empty path addresses an object with no namespace. +/// +/// Components are checked for addressability, not a character set. The name's +/// grammar is the caller's, so a table reports [`Error::InvalidTableName`], a +/// Function admits names a table may not, and a catalog database carries the +/// `/` that [`RemoteCatalog`] allows. +/// +/// [`RemoteCatalog`]: super::catalog::RemoteCatalog +fn build_object_identifier(what: &str, name: &str, namespace: &[String]) -> Result { + for segment in namespace { + reject_unaddressable_component("namespace segment", segment)?; } + reject_unaddressable_component(what, name)?; + Ok(join_identifier( + namespace.iter().map(String::as_str).chain([name]), + )) } -fn build_namespace_identifier(namespace: &[String], delimiter: &str) -> String { +/// What a component may not be if the join is to survive being split back +/// apart: empty, a segment URL parsing resolves away, or the delimiter itself. +/// +/// Each erases a boundary no encoding of the joined form recovers. `["prod", +/// ""]` joins to `prod$`, which reads back as `["prod"]`, so a drop reaches the +/// parent of the namespace the caller named. +/// +/// Not a character set: per-component percent-encoding makes the wider set +/// safe, since a `/` in a name reaches the service as `%2F`, still one +/// segment. +fn reject_unaddressable_component(what: &str, value: &str) -> Result<()> { + if value.is_empty() { + return Err(Error::InvalidInput { + message: format!( + "{what} must not be empty: the identifier would carry two delimiters in a row, \ + and splitting it back apart would name a different object" + ), + }); + } + reject_relative_segment(what, value)?; + if value.contains(ID_DELIMITER) { + return Err(Error::InvalidInput { + message: format!( + "{what} '{value}' contains the identifier delimiter '{ID_DELIMITER}', so the \ + namespace path and the name it joins could not be told apart" + ), + }); + } + Ok(()) +} + +/// The path segment addressing one table. A wrapper for the error type: +/// callers match on [`Error::InvalidTableName`]. +fn build_table_identifier(name: &str, namespace: &[String]) -> Result { + validate_table_name(name)?; + build_object_identifier("table name", name, namespace) +} + +/// Join components into the `{id}` a route addresses: each percent-encoded, +/// then joined by the delimiter. +/// +/// Per component rather than over the joined string, so the delimiter stays a +/// delimiter and nothing inside a component can end the path segment. +/// +/// A second line, not the first: a component from the object charset is all +/// unreserved and encodes to itself, so the route reads as the caller wrote it. +/// It does not cover `.` and `..`, which are unreserved too and resolve away +/// after decoding -- [`build_object_identifier`] refuses those. +fn join_identifier<'a>(components: impl Iterator) -> String { + components + .map(|component| urlencoding::encode(component).into_owned()) + .collect::>() + .join(ID_DELIMITER) +} + +/// The path segment addressing one namespace. +fn build_namespace_identifier(namespace: &[String]) -> Result { + for segment in namespace { + reject_unaddressable_component("namespace segment", segment)?; + } if namespace.is_empty() { // According to the namespace spec, use delimiter to represent root namespace - delimiter.to_string() - } else { - namespace.join(delimiter) + return Ok(ID_DELIMITER.to_string()); } + Ok(join_identifier(namespace.iter().map(String::as_str))) } /// Build a secure cache key using length prefixes. @@ -631,6 +725,65 @@ struct RemoteDropFunctionResponse { dropped: bool, } +/// The create body: every field of [`FunctionRegistrationRequest`] except the +/// name, which is the path identifier. +/// +/// A struct rather than a literal listing the fields, so that the compiler +/// decides what reaches the service. A field the registration request grows is +/// a build error here until it is handled; a literal would simply not send it. +#[derive(serde::Serialize)] +struct RemoteCreateFunctionRequest<'a> { + artifact: &'a FunctionArtifactRequest, + signature: &'a FunctionSignature, + runtime: &'a PythonRuntimeSpec, + /// Absent when the Function binds nothing, so such a client sends what a + /// client without bindings sends. + /// + /// A service that does not know the field ignores it: registration + /// succeeds, the returned version carries no bindings, and the Function + /// fails at execution with the variable unset. [`ServerVersion`] is how + /// this codebase refuses a feature the service is too old for; it is held + /// per table, so gating a database-level call is follow-up work. + /// + /// [`ServerVersion`]: super::db::ServerVersion + #[serde(skip_serializing_if = "<[SecretBinding]>::is_empty")] + secret_bindings: &'a [SecretBinding], +} + +/// Create a Secret under a name the database does not yet hold. +/// +/// Declared separately from the alter request although the two are identical +/// today: they are different operations to the service -- one refuses an +/// existing name, the other requires it -- and either may grow a field the +/// other has no meaning for. +/// +/// The name and its namespace are the path identifier, so neither appears here. +#[derive(serde::Serialize)] +struct RemoteCreateSecretRequest<'a> { + value: &'a str, +} + +/// Replace the credential behind a Secret the database already holds. +#[derive(serde::Serialize)] +struct RemoteAlterSecretRequest<'a> { + value: &'a str, +} + +#[derive(serde::Deserialize)] +struct RemoteListSecretsResponse { + #[serde(default)] + secrets: Vec, + #[serde(default)] + page_token: Option, +} + +/// An object rather than a bare name so a later listing can carry a Secret's +/// type or last-updated time without breaking this one. +#[derive(serde::Deserialize)] +struct RemoteListedSecret { + name: String, +} + /// Bound on `list_jobs` page walking; a warning is logged when the listing /// is truncated at this many pages. const MAX_LIST_JOBS_PAGES: usize = 100; @@ -652,11 +805,7 @@ impl Database for RemoteDatabase { &self, request: CreateMaterializedViewRequest, ) -> Result { - let identifier = build_table_identifier( - &request.name, - &request.namespace_path, - &self.client.id_delimiter, - ); + let identifier = build_table_identifier(&request.name, &request.namespace_path)?; let req = self .client .post(&format!("/v1/materialized_view/{identifier}/create")) @@ -699,7 +848,7 @@ impl Database for RemoteDatabase { name: &str, namespace_path: &[String], ) -> Result { - let identifier = build_table_identifier(name, namespace_path, &self.client.id_delimiter); + let identifier = build_table_identifier(name, namespace_path)?; let request = self .client .post(&format!("/v1/materialized_view/{identifier}/drop")); @@ -737,7 +886,7 @@ impl Database for RemoteDatabase { page_token: Option, } - let namespace_id = build_namespace_identifier(namespace_path, &self.client.id_delimiter); + let namespace_id = build_namespace_identifier(namespace_path)?; let path = format!("/v1/namespace/{namespace_id}/materialized_view/list"); let mut views = Vec::new(); let mut page_token: Option = None; @@ -773,15 +922,16 @@ impl Database for RemoteDatabase { &self, request: FunctionRegistrationRequest, ) -> Result> { - let function_id = urlencoding::encode(&request.name); + let function_id = build_object_identifier("Function name", &request.name, &[])?; let req = self .client .post(&format!("/v1/function/{function_id}/create")) - .json(&serde_json::json!({ - "artifact": request.artifact, - "signature": request.signature, - "runtime": request.runtime, - })); + .json(&RemoteCreateFunctionRequest { + artifact: &request.artifact, + signature: &request.signature, + runtime: &request.runtime, + secret_bindings: &request.secret_bindings, + }); let (request_id, response) = self.client.send(req).await?; let response = self.client.check_response(&request_id, response).await?; let status = response.status(); @@ -798,7 +948,7 @@ impl Database for RemoteDatabase { } async fn get_function(&self, name: &str, version: &str) -> Result { - let function_id = urlencoding::encode(name); + let function_id = build_object_identifier("Function name", name, &[])?; let req = self .client .post(&format!("/v1/function/{function_id}/describe")) @@ -811,7 +961,7 @@ impl Database for RemoteDatabase { } async fn list_functions(&self) -> Result> { - let namespace_id = build_namespace_identifier(&[], &self.client.id_delimiter); + let namespace_id = build_namespace_identifier(&[])?; let path = format!("/v1/namespace/{namespace_id}/function/list"); let mut functions = Vec::new(); let mut page_token: Option = None; @@ -852,7 +1002,7 @@ impl Database for RemoteDatabase { } async fn drop_function(&self, name: &str, version: &str) -> Result { - let function_id = urlencoding::encode(name); + let function_id = build_object_identifier("Function name", name, &[])?; let req = self .client .post(&format!("/v1/function/{function_id}/drop")) @@ -865,6 +1015,80 @@ impl Database for RemoteDatabase { Ok(response.dropped) } + async fn create_secret( + &self, + name: &str, + value: &str, + namespace_path: &[String], + ) -> Result<()> { + let secret_id = build_object_identifier("Secret name", name, namespace_path)?; + self.post_secret_write( + &format!("/v1/secret/{secret_id}/create"), + &RemoteCreateSecretRequest { value }, + ) + .await + } + + async fn alter_secret(&self, name: &str, value: &str, namespace_path: &[String]) -> Result<()> { + let secret_id = build_object_identifier("Secret name", name, namespace_path)?; + self.post_secret_write( + &format!("/v1/secret/{secret_id}/alter"), + &RemoteAlterSecretRequest { value }, + ) + .await + } + + async fn list_secrets(&self, namespace_path: &[String]) -> Result> { + let namespace_id = build_namespace_identifier(namespace_path)?; + let path = format!("/v1/namespace/{namespace_id}/secret/list"); + let mut names = Vec::new(); + let mut page_token: Option = None; + let mut seen_page_tokens = HashSet::new(); + loop { + let mut req = self.client.get(&path); + if let Some(token) = &page_token { + req = req.query(&[("page_token", token)]); + } + let (request_id, response) = self.client.send(req).await?; + let response = self.client.check_response(&request_id, response).await?; + let status = response.status(); + let response: RemoteListSecretsResponse = + response.json().await.err_to_http(request_id.clone())?; + names.extend(response.secrets.into_iter().map(|secret| secret.name)); + let Some(next_page_token) = response.page_token.filter(|token| !token.is_empty()) + else { + break; + }; + if !seen_page_tokens.insert(next_page_token.clone()) { + return Err(Error::Http { + source: "Secret listing response repeated a page_token".into(), + request_id, + status_code: Some(status), + }); + } + page_token = Some(next_page_token); + } + Ok(names) + } + + async fn drop_secret(&self, name: &str, namespace_path: &[String]) -> Result<()> { + let secret_id = build_object_identifier("Secret name", name, namespace_path)?; + let req = self.client.post(&format!("/v1/secret/{secret_id}/drop")); + let (request_id, response) = self.client.send(req).await?; + self.client.check_response(&request_id, response).await?; + Ok(()) + } + + async fn describe_secret(&self, name: &str, namespace_path: &[String]) -> Result { + let secret_id = build_object_identifier("Secret name", name, namespace_path)?; + let req = self + .client + .post(&format!("/v1/secret/{secret_id}/describe")); + let (request_id, response) = self.client.send(req).await?; + let response = self.client.check_response(&request_id, response).await?; + response.json().await.err_to_http(request_id) + } + async fn open_job(&self, job_id: &str) -> Result { let handle = super::job::RemoteJob::new(self.client.clone(), job_id.to_string()); match crate::job::JobHandle::describe(&handle).await { @@ -987,8 +1211,7 @@ impl Database for RemoteDatabase { }; for table in &tables { - let table_identifier = - build_table_identifier(table, &request.namespace_path, &self.client.id_delimiter); + let table_identifier = build_table_identifier(table, &request.namespace_path)?; let cache_key = build_cache_key(table, &request.namespace_path); let remote_table = Arc::new(RemoteTable::new( self.client.clone(), @@ -1004,7 +1227,7 @@ impl Database for RemoteDatabase { async fn list_tables(&self, request: ListTablesRequest) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); - let namespace_id = build_namespace_identifier(namespace_parts, &self.client.id_delimiter); + let namespace_id = build_namespace_identifier(namespace_parts)?; let mut req = self .client .get(&format!("/v1/namespace/{}/table/list", namespace_id)); @@ -1024,8 +1247,7 @@ impl Database for RemoteDatabase { // Cache the tables for future use let namespace_vec = namespace_parts.to_vec(); for table in &response.tables { - let table_identifier = - build_table_identifier(table, &namespace_vec, &self.client.id_delimiter); + let table_identifier = build_table_identifier(table, &namespace_vec)?; let cache_key = build_cache_key(table, &namespace_vec); let remote_table = Arc::new(RemoteTable::new( self.client.clone(), @@ -1043,11 +1265,7 @@ impl Database for RemoteDatabase { async fn create_table(&self, mut request: CreateTableRequest) -> Result> { let body = stream_as_body(request.data.scan_as_stream())?; - let identifier = build_table_identifier( - &request.name, - &request.namespace_path, - &self.client.id_delimiter, - ); + let identifier = build_table_identifier(&request.name, &request.namespace_path)?; let req = self .client .post(&format!("/v1/table/{}/create/", identifier)) @@ -1099,11 +1317,7 @@ impl Database for RemoteDatabase { } let rsp = self.client.check_response(&request_id, rsp).await?; let version = parse_server_version(&request_id, &rsp)?; - let table_identifier = build_table_identifier( - &request.name, - &request.namespace_path, - &self.client.id_delimiter, - ); + let table_identifier = build_table_identifier(&request.name, &request.namespace_path)?; let cache_key = build_cache_key(&request.name, &request.namespace_path); let table = Arc::new(RemoteTable::new( self.client.clone(), @@ -1118,11 +1332,8 @@ impl Database for RemoteDatabase { } async fn clone_table(&self, request: CloneTableRequest) -> Result> { - let table_identifier = build_table_identifier( - &request.target_table_name, - &request.target_namespace_path, - &self.client.id_delimiter, - ); + let table_identifier = + build_table_identifier(&request.target_table_name, &request.target_namespace_path)?; let remote_request = RemoteCloneTableRequest { source_location: request.source_uri, @@ -1163,11 +1374,7 @@ impl Database for RemoteDatabase { } async fn open_table(&self, request: OpenTableRequest) -> Result> { - let identifier = build_table_identifier( - &request.name, - &request.namespace_path, - &self.client.id_delimiter, - ); + let identifier = build_table_identifier(&request.name, &request.namespace_path)?; let cache_key = build_cache_key(&request.name, &request.namespace_path); // We describe the table to confirm it exists before moving on. @@ -1183,11 +1390,7 @@ impl Database for RemoteDatabase { let rsp = self.client.check_response(&request_id, rsp).await?; let version = parse_server_version(&request_id, &rsp)?; let describe_body = rsp.text().await.ok(); - let table_identifier = build_table_identifier( - &request.name, - &request.namespace_path, - &self.client.id_delimiter, - ); + let table_identifier = build_table_identifier(&request.name, &request.namespace_path)?; let table = Arc::new(RemoteTable::new( self.client.clone(), request.name.clone(), @@ -1214,8 +1417,7 @@ impl Database for RemoteDatabase { cur_namespace_path: &[String], new_namespace_path: &[String], ) -> Result<()> { - let current_identifier = - build_table_identifier(current_name, cur_namespace_path, &self.client.id_delimiter); + let current_identifier = build_table_identifier(current_name, cur_namespace_path)?; let current_cache_key = build_cache_key(current_name, cur_namespace_path); let new_cache_key = build_cache_key(new_name, new_namespace_path); @@ -1279,8 +1481,7 @@ impl Database for RemoteDatabase { request: ListNamespacesRequest, ) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); - let namespace_id = build_namespace_identifier(namespace_parts, &self.client.id_delimiter); - let namespace_id = urlencoding::encode(&namespace_id); + let namespace_id = build_namespace_identifier(namespace_parts)?; let mut req = self .client .get(&format!("/v1/namespace/{}/list", namespace_id)); @@ -1302,8 +1503,7 @@ impl Database for RemoteDatabase { request: CreateNamespaceRequest, ) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); - let namespace_id = build_namespace_identifier(namespace_parts, &self.client.id_delimiter); - let namespace_id = urlencoding::encode(&namespace_id); + let namespace_id = build_namespace_identifier(namespace_parts)?; let mut req = self .client .post(&format!("/v1/namespace/{}/create", namespace_id)); @@ -1334,8 +1534,7 @@ impl Database for RemoteDatabase { async fn drop_namespace(&self, request: DropNamespaceRequest) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); - let namespace_id = build_namespace_identifier(namespace_parts, &self.client.id_delimiter); - let namespace_id = urlencoding::encode(&namespace_id); + let namespace_id = build_namespace_identifier(namespace_parts)?; let mut req = self .client .post(&format!("/v1/namespace/{}/drop", namespace_id)); @@ -1369,8 +1568,7 @@ impl Database for RemoteDatabase { request: DescribeNamespaceRequest, ) -> Result { let namespace_parts = request.id.as_deref().unwrap_or(&[]); - let namespace_id = build_namespace_identifier(namespace_parts, &self.client.id_delimiter); - let namespace_id = urlencoding::encode(&namespace_id); + let namespace_id = build_namespace_identifier(namespace_parts)?; let req = self .client .post(&format!("/v1/namespace/{}/describe", namespace_id)) @@ -1389,7 +1587,7 @@ impl Database for RemoteDatabase { async fn namespace_client(&self) -> Result> { // Create a RestNamespace pointing to the same remote host with the same authentication headers let mut builder = lance_namespace_impls::RestNamespaceBuilder::new(self.client.host()) - .delimiter(&self.client.id_delimiter) + .delimiter(ID_DELIMITER) .headers(self.namespace_headers.clone()); if let Some(context_provider) = &self.namespace_context_provider { @@ -1425,7 +1623,7 @@ impl Database for RemoteDatabase { let mut properties = HashMap::new(); properties.insert("uri".to_string(), self.client.host().to_string()); - properties.insert("delimiter".to_string(), self.client.id_delimiter.clone()); + properties.insert("delimiter".to_string(), ID_DELIMITER.to_string()); for (key, value) in &self.namespace_headers { properties.insert(format!("header.{}", key), value.clone()); } @@ -2208,7 +2406,11 @@ mod tests { http::Response::builder() .status(200) - .body(r#"{"tables": ["ns1$ns2$table1", "ns1$ns2$table2"]}"#) + // A namespace listing names the tables in that namespace; the + // namespace is the route, not part of each name. The client + // joins the two itself to build each table's identifier, so a + // listing that repeated the namespace would be joined twice. + .body(r#"{"tables": ["table1", "table2"]}"#) .unwrap() }); let names = conn @@ -2217,7 +2419,7 @@ mod tests { .execute() .await .unwrap(); - assert_eq!(names, vec!["ns1$ns2$table1", "ns1$ns2$table2"]); + assert_eq!(names, vec!["table1", "table2"]); } #[tokio::test] @@ -3124,6 +3326,354 @@ mod tests { assert_eq!(batches[0].schema(), schema); } + /// A component that is not a legal Secret component never reaches a + /// transport. Before the identifier was checked here, each of these decided + /// the route instead of the name: `a/b` and `../jobs` left `/v1/secret/` + /// entirely, carrying a create body that holds a credential, and `a$b` read + /// as the namespace `a` and the name `b`. + #[tokio::test] + async fn test_an_illegal_component_never_reaches_the_transport() { + use std::sync::{Arc, Mutex}; + for name in [ + "../jobs", + "a/b", + "a$b", + "with space", + "q?x", + "a#b", + "a%2Fb", + "", + ] { + let reached = Arc::new(Mutex::new(false)); + let flag = reached.clone(); + let conn = Connection::new_with_handler(move |_| { + *flag.lock().unwrap() = true; + http::Response::builder().status(200).body("{}").unwrap() + }); + let error = conn + .create_secret(name, "sk-live-0001", &[]) + .await + .expect_err("an illegal component must be refused"); + assert!(!*reached.lock().unwrap(), "{name:?} reached the transport"); + assert!( + error.to_string().contains("Secret name"), + "{name:?}: {error}" + ); + } + } + + /// `.` and `..` pass the character set and still cannot address anything: + /// URL parsing resolves them as relative segments, and after + /// percent-decoding, so no spelling of either survives. + #[tokio::test] + async fn test_a_relative_segment_component_is_refused() { + use std::sync::{Arc, Mutex}; + // The percent-encoded spellings are caught a step earlier, by the + // character set: `%` is not a character a Secret component may hold. + // They reach the relative-segment rule only where there is no charset + // to catch them first -- see the Function case below. + for component in [".", ".."] { + let reached = Arc::new(Mutex::new(false)); + let flag = reached.clone(); + let conn = Connection::new_with_handler(move |_| { + *flag.lock().unwrap() = true; + http::Response::builder() + .status(200) + .body(r#"{"secrets":[]}"#) + .unwrap() + }); + let by_name = conn + .drop_secret(component, &[]) + .await + .expect_err("a dot-only name must be refused"); + assert!( + by_name.to_string().contains("relative path segments"), + "{by_name}" + ); + + let by_segment = conn + .list_secrets(&[component.to_string()]) + .await + .expect_err("a dot-only namespace segment must be refused"); + assert!( + by_segment.to_string().contains("relative path segments"), + "{by_segment}" + ); + assert!( + !*reached.lock().unwrap(), + "{component:?} reached the transport" + ); + } + } + + /// An identifier the service accepts is untouched by the encoding, so the + /// route reads like the table and Function routes beside it. + #[tokio::test] + async fn test_an_admissible_identifier_is_not_encoded() { + use std::sync::{Arc, Mutex}; + let seen = Arc::new(Mutex::new(String::new())); + let captured = seen.clone(); + let conn = Connection::new_with_handler(move |request| { + *captured.lock().unwrap() = request.url().path().to_string(); + http::Response::builder().status(200).body("{}").unwrap() + }); + conn.drop_secret( + "openai-prod.v1", + &["prod".to_string(), "vision_2".to_string()], + ) + .await + .unwrap(); + assert_eq!( + *seen.lock().unwrap(), + "/v1/secret/prod$vision_2$openai-prod.v1/drop" + ); + } + + /// A table name is joined into the route the same way a Secret's is, so the + /// same two failures are reachable: a dot-only name leaves the table route + /// entirely, and a name holding the delimiter is indistinguishable from a + /// namespace boundary. + #[tokio::test] + async fn test_a_table_name_cannot_choose_its_own_route() { + use std::sync::{Arc, Mutex}; + for name in ["..", ".", "../jobs", "a/b", "a$b"] { + let reached = Arc::new(Mutex::new(false)); + let flag = reached.clone(); + let conn = Connection::new_with_handler(move |_| { + *flag.lock().unwrap() = true; + http::Response::builder().status(200).body("{}").unwrap() + }); + let error = conn + .drop_table(name, &[]) + .await + .expect_err("an unaddressable table name must be refused"); + assert!(!*reached.lock().unwrap(), "{name:?} reached the transport"); + assert!(!error.to_string().is_empty(), "{name:?}"); + } + } + + /// The namespace half of the same join. + #[tokio::test] + async fn test_a_namespace_segment_cannot_choose_its_own_route() { + use std::sync::{Arc, Mutex}; + for segment in ["..", "a$b", ""] { + let reached = Arc::new(Mutex::new(false)); + let flag = reached.clone(); + let conn = Connection::new_with_handler(move |_| { + *flag.lock().unwrap() = true; + http::Response::builder().status(200).body("{}").unwrap() + }); + let error = conn + .drop_table("t", &[segment.to_string()]) + .await + .expect_err("an unaddressable namespace segment must be refused"); + assert!( + !*reached.lock().unwrap(), + "{segment:?} reached the transport" + ); + assert!(!error.to_string().is_empty(), "{segment:?}"); + } + } + + /// A segment outside the table charset still addresses one segment: the + /// service decides whether it may exist, and percent-encoding is what keeps + /// the question reaching the right route. A catalog database is named this + /// way. + #[tokio::test] + async fn test_a_namespace_segment_outside_the_charset_is_encoded_not_refused() { + use std::sync::{Arc, Mutex}; + let seen = Arc::new(Mutex::new(String::new())); + let path = seen.clone(); + let conn = Connection::new_with_handler(move |request| { + *path.lock().unwrap() = request.url().path().to_string(); + http::Response::builder().status(200).body("{}").unwrap() + }); + conn.drop_table("t", &["team/search".to_string()]) + .await + .unwrap(); + assert_eq!(*seen.lock().unwrap(), "/v1/table/team%2Fsearch$t/drop/"); + } + + /// A Function name is percent-encoded, which covers everything but the + /// relative segment: `..` is unreserved, so it survives encoding and is + /// then resolved away, posting a registration body to `/v1/create`. + #[tokio::test] + async fn test_a_relative_segment_function_name_is_refused() { + use std::sync::{Arc, Mutex}; + for name in [".", "..", "%2E%2E"] { + let reached = Arc::new(Mutex::new(false)); + let flag = reached.clone(); + let conn = Connection::new_with_handler(move |_| { + *flag.lock().unwrap() = true; + http::Response::builder().status(200).body("{}").unwrap() + }); + let error = conn + .drop_function(name, "fv_1") + .await + .expect_err("a dot-only Function name must be refused"); + assert!( + error.to_string().contains("relative path segments"), + "{error}" + ); + assert!(!*reached.lock().unwrap(), "{name:?} reached the transport"); + } + } + + /// Only `.` and `..` are relative segments. `...` and longer runs are + /// ordinary and address perfectly well, so refusing them would make an + /// object that works today stop working on upgrade. This pins that. + #[tokio::test] + async fn test_a_longer_run_of_periods_is_an_ordinary_name() { + use std::sync::{Arc, Mutex}; + for name in ["...", "....", "a.", ".a", "a..b"] { + let seen = Arc::new(Mutex::new(String::new())); + let captured = seen.clone(); + let conn = Connection::new_with_handler(move |request| { + *captured.lock().unwrap() = request.url().path().to_string(); + http::Response::builder().status(200).body("{}").unwrap() + }); + conn.drop_secret(name, &[]) + .await + .unwrap_or_else(|error| panic!("{name:?} must remain addressable: {error}")); + assert_eq!( + *seen.lock().unwrap(), + format!("/v1/secret/{name}/drop"), + "{name:?} did not reach its own route" + ); + } + } + + #[tokio::test] + async fn test_create_and_alter_secret_send_the_value_in_the_request_body() { + for (route, call) in [ + ("/v1/secret/openai-prod/create", true), + ("/v1/secret/openai-prod/alter", false), + ] { + let conn = Connection::new_with_handler(move |request| { + assert_eq!(request.method(), &reqwest::Method::POST); + assert_eq!(request.url().path(), route); + // Never a path segment or query parameter, which is what keeps + // it out of access logs and proxy traces. + assert!(request.url().query().is_none(), "{:?}", request.url()); + let body: serde_json::Value = + serde_json::from_slice(request.body().unwrap().as_bytes().unwrap()).unwrap(); + // The name is the path identifier, so the body is the value alone. + assert_eq!(body, serde_json::json!({ "value": "sk-live-0001" })); + http::Response::builder().status(200).body("{}").unwrap() + }); + if call { + conn.create_secret("openai-prod", "sk-live-0001", &[]) + .await + .unwrap(); + } else { + conn.alter_secret("openai-prod", "sk-live-0001", &[]) + .await + .unwrap(); + } + } + } + + #[tokio::test] + async fn test_list_secrets_walks_pages_and_returns_names_only() { + let conn = Connection::new_with_handler(|request| { + assert_eq!(request.method(), &reqwest::Method::GET); + assert_eq!(request.url().path(), "/v1/namespace/$/secret/list"); + let page = request + .url() + .query_pairs() + .find(|(key, _)| key == "page_token") + .map(|(_, value)| value.into_owned()); + let body = match page.as_deref() { + None => r#"{"secrets":[{"name":"openai-prod"}],"page_token":"p2"}"#, + Some("p2") => r#"{"secrets":[{"name":"hf-prod"}]}"#, + Some(other) => panic!("unexpected page token: {other}"), + }; + http::Response::builder().status(200).body(body).unwrap() + }); + assert_eq!( + conn.list_secrets(&[]).await.unwrap(), + vec!["openai-prod".to_string(), "hf-prod".to_string()] + ); + } + + /// A server that keeps handing back the same token would otherwise spin + /// forever. + #[tokio::test] + async fn test_list_secrets_rejects_a_repeated_page_token() { + let conn = Connection::new_with_handler(|_| { + http::Response::builder() + .status(200) + .body(r#"{"secrets":[{"name":"openai-prod"}],"page_token":"same"}"#) + .unwrap() + }); + let error = conn.list_secrets(&[]).await.unwrap_err(); + assert!( + error.to_string().contains("repeated a page_token"), + "{error}" + ); + } + + #[tokio::test] + async fn test_drop_and_describe_address_the_secret_in_the_path() { + let conn = Connection::new_with_handler(|request| { + assert_eq!(request.url().path(), "/v1/secret/openai-prod/drop"); + // Nothing is left to say once the path names the Secret. + assert!(request.body().is_none(), "{:?}", request.body()); + http::Response::builder().status(200).body("{}").unwrap() + }); + conn.drop_secret("openai-prod", &[]).await.unwrap(); + + let conn = Connection::new_with_handler(|request| { + assert_eq!(request.url().path(), "/v1/secret/openai-prod/describe"); + assert!(request.body().is_none(), "{:?}", request.body()); + http::Response::builder() + .status(200) + .body(r#"{"name":"openai-prod","created_at_millis":1,"updated_at_millis":2}"#) + .unwrap() + }); + let info = conn.describe_secret("openai-prod", &[]).await.unwrap(); + assert_eq!(info.name, "openai-prod"); + } + + /// The namespace is part of the identifier the path addresses, joined with + /// the client's configured delimiter the way every other object's is. A + /// root Secret is therefore addressed by its bare name, and a namespaced + /// one by the joined path -- there is no body field either way. + #[tokio::test] + async fn test_a_namespace_path_is_addressed_in_the_path() { + let conn = Connection::new_with_handler(|request| { + assert_eq!( + request.url().path(), + "/v1/secret/prod$vision$openai-prod/drop" + ); + http::Response::builder().status(200).body("{}").unwrap() + }); + conn.drop_secret("openai-prod", &["prod".to_string(), "vision".to_string()]) + .await + .unwrap(); + + let conn = Connection::new_with_handler(|request| { + assert_eq!(request.url().path(), "/v1/secret/openai-prod/drop"); + http::Response::builder().status(200).body("{}").unwrap() + }); + conn.drop_secret("openai-prod", &[]).await.unwrap(); + + // Listing is namespace-scoped, so the namespace is the whole identifier. + let conn = Connection::new_with_handler(|request| { + assert_eq!( + request.url().path(), + "/v1/namespace/prod$vision/secret/list" + ); + http::Response::builder() + .status(200) + .body(r#"{"secrets":[]}"#) + .unwrap() + }); + conn.list_secrets(&["prod".to_string(), "vision".to_string()]) + .await + .unwrap(); + } + #[tokio::test] async fn test_create_function_async_sends_canonical_request_and_decodes_typed_job() { const REQUEST: &str = include_str!( diff --git a/rust/lancedb/src/secrets.rs b/rust/lancedb/src/secrets.rs new file mode 100644 index 000000000..c980e51ee --- /dev/null +++ b/rust/lancedb/src/secrets.rs @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright The LanceDB Authors + +//! Named Secrets: database-scoped credentials a Function binds by name. +//! +//! Nothing here holds a credential. The verbs live on +//! [`crate::connection::Connection`], and none of them returns a value -- by +//! construction rather than by policy, so there is no code path that could. +//! +//! What a Secret is, how one is named, and how a Function binds one all live +//! here; [`crate::function`] holds the bindings a FunctionVersion records, the +//! way Sophon's Secret catalog and Function catalog divide the same two. + +use serde::de; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde_json::Value; + +/// What a database records about a Secret. Never its value. +/// +/// Returned by [`crate::connection::Connection::describe_secret`]. There is no +/// field for the credential and no method that could produce one. +#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)] +pub struct SecretInfo { + /// The Secret's database-scoped name. + pub name: String, + /// When the Secret was created, in milliseconds since the Unix epoch. + pub created_at_millis: i64, + /// When the Secret's value was last rotated, in milliseconds since the Unix + /// epoch. + /// + /// This is the only observable that a rotation landed: no API returns a + /// credential, so a caller confirms `alter_secret` took effect by watching + /// this move. + pub updated_at_millis: i64, +} + +/// Where a Secret lives, carried as its parts rather than as one string. +/// +/// Nothing here is parsed, so nothing can parse two ways. A joined id would +/// instead need a delimiter excluded from every name and segment, agreed on by +/// both sides. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +pub struct SecretReference { + pub name: String, + /// The namespace holding the Secret. Empty is the root, and is absent from + /// the wire rather than sent empty: a binding states a namespace only when + /// it has one. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub namespace_path: Vec, +} + +impl SecretReference { + /// A Secret in the root namespace. + pub fn new(name: impl Into) -> Self { + Self { + name: name.into(), + namespace_path: Vec::new(), + } + } + + /// A Secret in `namespace_path`. + pub fn in_namespace(name: impl Into, namespace_path: Vec) -> Self { + Self { + name: name.into(), + namespace_path, + } + } +} + +/// How a Secret reaches the Function that binds it. +/// +/// One list rather than a field per delivery mode, so a mode added later is a +/// variant and the per-Function rules -- how many Secrets a Function may bind, +/// which ones it needs -- stay answerable from one place. +/// +/// A mode this client does not know decodes rather than failing the whole +/// FunctionVersion, as [`PythonRuntimeSpec`] does for an unknown runtime. That +/// takes both halves: [`SecretBinding::Unrecognized`] gives the wire somewhere +/// to land, and `#[non_exhaustive]` denies callers an exhaustive match, so a +/// later mode arrives as a case they already had to handle. Its payload is +/// dropped -- the client does not proxy catalog values. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +#[non_exhaustive] +pub enum SecretBinding { + /// Delivered as an environment variable, which the UDF's library already + /// reads. The variable is the delivery target; the Secret is what fills it. + Env { + variable: String, + /// Named `secret_ref` rather than `secret` because a Job payload is + /// scanned server-side for credential-shaped keys, and a key called + /// `secret` trips that guard whatever it actually holds. + secret_ref: SecretReference, + }, + /// A binding kind introduced by a newer server. + Unrecognized { kind: String }, +} + +impl SecretBinding { + /// The wire discriminator reported by Sophon. + pub fn kind(&self) -> &str { + match self { + Self::Env { .. } => "env", + Self::Unrecognized { kind } => kind, + } + } + + /// The environment variable this binding fills, or `None` for a kind that + /// does not deliver through one. + pub fn variable(&self) -> Option<&str> { + match self { + Self::Env { variable, .. } => Some(variable), + Self::Unrecognized { .. } => None, + } + } + + /// The Secret bound, or `None` for a kind this client cannot read. + pub fn secret(&self) -> Option<&SecretReference> { + match self { + Self::Env { secret_ref, .. } => Some(secret_ref), + Self::Unrecognized { .. } => None, + } + } +} + +#[derive(Deserialize)] +struct EnvSecretBindingWire { + variable: String, + secret_ref: SecretReference, +} + +impl<'de> Deserialize<'de> for SecretBinding { + fn deserialize>(deserializer: D) -> std::result::Result { + let value = Value::deserialize(deserializer)?; + let kind = value + .get("kind") + .ok_or_else(|| de::Error::missing_field("kind"))? + .as_str() + .ok_or_else(|| de::Error::custom("secret binding kind must be a string"))? + .to_string(); + match kind.as_str() { + "env" => { + let wire: EnvSecretBindingWire = + serde_json::from_value(value).map_err(de::Error::custom)?; + Ok(Self::Env { + variable: wire.variable, + secret_ref: wire.secret_ref, + }) + } + _ => Ok(Self::Unrecognized { kind }), + } + } +} + +impl Serialize for SecretBinding { + fn serialize(&self, serializer: S) -> std::result::Result { + #[derive(Serialize)] + struct EnvBindingRef<'a> { + kind: &'static str, + variable: &'a str, + secret_ref: &'a SecretReference, + } + + #[derive(Serialize)] + struct UnrecognizedBindingRef<'a> { + kind: &'a str, + } + + match self { + Self::Env { + variable, + secret_ref, + } => EnvBindingRef { + kind: "env", + variable, + secret_ref, + } + .serialize(serializer), + Self::Unrecognized { kind } => UnrecognizedBindingRef { kind }.serialize(serializer), + } + } +} diff --git a/rust/lancedb/src/utils/mod.rs b/rust/lancedb/src/utils/mod.rs index a66e758f7..f80cc876a 100644 --- a/rust/lancedb/src/utils/mod.rs +++ b/rust/lancedb/src/utils/mod.rs @@ -19,10 +19,15 @@ use std::pin::Pin; use crate::error::{Error, Result}; use datafusion_physical_plan::SendableRecordBatchStream; -static TABLE_NAME_REGEX: std::sync::LazyLock = - std::sync::LazyLock::new(|| regex::Regex::new(r"^[a-zA-Z0-9_\-\.]+$").unwrap()); -static NAMESPACE_NAME_REGEX: std::sync::LazyLock = - std::sync::LazyLock::new(|| regex::Regex::new(r"^[a-zA-Z0-9_\-\.]+$").unwrap()); +/// The characters any object name may contain: a table, a namespace segment, a +/// Secret, a materialized view. +/// +/// No positional rule on top of it -- a name may begin with `_`, `-` or `.`, +/// as LanceDB namespaces already do. `.` and `..` are excluded separately, by +/// [`reject_relative_segment`]: that is a property of where a name sits in a +/// URL, not of the name. Length is the service's to bound. +static OBJECT_NAME_REGEX: std::sync::LazyLock = + std::sync::LazyLock::new(|| regex::Regex::new(r"^[A-Za-z0-9_.\-]+$").unwrap()); pub trait PatchStoreParam { fn patch_with_store_wrapper( @@ -81,66 +86,94 @@ impl PatchReadParam for ReadParams { } } -/// Validate table name. -pub fn validate_table_name(name: &str) -> Result<()> { - if name.is_empty() { - return Err(Error::InvalidTableName { - name: name.to_string(), - reason: "Table names cannot be empty strings".to_string(), - }); - } - if name == "." { - return Err(Error::InvalidTableName { - name: name.to_string(), - reason: "Table name cannot be a single dot.".to_string(), - }); - } - if name == ".." { - return Err(Error::InvalidTableName { - name: name.to_string(), - reason: "Table name cannot be two dots.".to_string(), - }); - } - if !TABLE_NAME_REGEX.is_match(name) { - return Err(Error::InvalidTableName { - name: name.to_string(), - reason: - "Table names can only contain alphanumeric characters, underscores, hyphens, and periods" - .to_string(), +/// The reason `.` and `..` are refused wherever a name becomes a path segment. +const RELATIVE_SEGMENT_REASON: &str = + "'.' and '..' are read as relative path segments and cannot address an object"; + +/// Whether URL parsing would resolve this component away rather than keep it. +/// +/// Exactly `.` and `..`, and their percent-encoded spellings -- resolution +/// happens after decoding, so `%2E%2E` collapses as surely as `..` does, and +/// `drop_table("..")` would reach `/v1/drop/`. No wider than that: `...` is an +/// ordinary segment that addresses fine. +fn is_relative_segment(value: &str) -> bool { + let decoded = value.replace("%2e", ".").replace("%2E", "."); + decoded == "." || decoded == ".." +} + +/// Refuse a path component that URL parsing resolves as a relative segment. +/// +/// Reachable on its own for an identifier with no other validator: a Function +/// name has no client-side grammar, so this is the only rule that applies. +pub(crate) fn reject_relative_segment(what: &str, value: &str) -> Result<()> { + if is_relative_segment(value) { + return Err(Error::InvalidInput { + message: format!("invalid {what} '{value}': {RELATIVE_SEGMENT_REASON}"), }); } Ok(()) } -/// Validate a namespace name component +/// Every rule an object name obeys: non-empty, inside [`OBJECT_NAME_REGEX`], +/// and addressable as a path segment. /// -/// Namespace names must: -/// - Not be empty -/// - Only contain alphanumeric characters, underscores, hyphens, and periods -/// -/// # Arguments -/// * `name` - A single namespace component (not the full path) -/// -/// # Returns -/// * `Ok(())` if the namespace name is valid -/// * `Err(Error)` if the namespace name is invalid -pub fn validate_namespace_name(name: &str) -> Result<()> { +/// Returns the reason rather than an [`Error`], because the error type is each +/// API's own -- a table reports [`Error::InvalidTableName`], the rest +/// [`Error::InvalidInput`]. Sharing the rules but not the error keeps a table, +/// a namespace segment and a Secret from drifting apart. +fn check_object_name(name: &str) -> std::result::Result<(), &'static str> { if name.is_empty() { - return Err(Error::InvalidInput { - message: "Namespace names cannot be empty strings".to_string(), - }); + return Err("it must not be empty"); } - if !NAMESPACE_NAME_REGEX.is_match(name) { - return Err(Error::InvalidInput { - message: format!( - "Invalid namespace name '{}': Namespace names can only contain alphanumeric characters, underscores, hyphens, and periods", - name - ), - }); + if !OBJECT_NAME_REGEX.is_match(name) { + return Err( + "it may contain only alphanumeric characters, underscores, hyphens and periods", + ); + } + if is_relative_segment(name) { + return Err(RELATIVE_SEGMENT_REASON); } Ok(()) } +/// Validate a table name. +pub fn validate_table_name(name: &str) -> Result<()> { + check_object_name(name).map_err(|reason| Error::InvalidTableName { + name: name.to_string(), + reason: reason.to_string(), + }) +} + +/// Validate one component of a namespace path -- a single segment, not the +/// whole path. [`validate_namespace`] covers a path. +pub fn validate_namespace_name(name: &str) -> Result<()> { + check_object_name(name).map_err(|reason| Error::InvalidInput { + message: format!("invalid namespace name '{name}': {reason}"), + }) +} + +/// Validate one component of a Secret identifier: a Secret name, or one segment +/// of the namespace path holding it. +/// +/// The join decides identity, and the service only sees what the split +/// produced. `"a$b"` is not a name the service accepts, but joined and split it +/// reads as the namespace `a` and the name `b` -- a different Secret that may +/// already exist. This is not a second opinion on the name; it is what lets the +/// service have one. +pub fn validate_secret_component(what: &str, value: &str) -> Result<()> { + check_object_name(value).map_err(|reason| Error::InvalidInput { + message: format!("invalid {what} '{value}': {reason}"), + }) +} + +/// Validate a Secret name and every segment of the namespace path holding it. +pub fn validate_secret_reference(name: &str, namespace_path: &[String]) -> Result<()> { + for segment in namespace_path { + validate_secret_component("Secret namespace path segment", segment)?; + } + validate_secret_component("Secret name", name) +} + /// Validate all components of a namespace /// /// Iterates through all namespace components and validates each one. diff --git a/rust/lancedb/tests/first_class_function_slice1.rs b/rust/lancedb/tests/first_class_function_slice1.rs index b70ac7f0a..3971a2471 100644 --- a/rust/lancedb/tests/first_class_function_slice1.rs +++ b/rust/lancedb/tests/first_class_function_slice1.rs @@ -7,6 +7,7 @@ use std::path::PathBuf; use lancedb::function::{ FunctionApplication, FunctionBinding, FunctionVersion, RefreshColumnResult, }; +use lancedb::secrets::{SecretBinding, SecretReference}; use serde_json::Value; fn fixture(name: &str) -> String { @@ -20,6 +21,26 @@ fn job_result(name: &str) -> Value { serde_json::from_str::(&fixture(name)).expect("remote Job fixture")["result"].clone() } +/// No client value models a resolved credential, at any nesting depth. +fn assert_no_secret_values(value: &Value) { + match value { + Value::Object(values) => { + for (key, value) in values { + assert!( + !matches!( + key.as_str(), + "secret_value" | "secret_values" | "resolved_secret" | "resolved_secrets" + ), + "client canonical value must not model resolved secret material" + ); + assert_no_secret_values(value); + } + } + Value::Array(values) => values.iter().for_each(assert_no_secret_values), + _ => {} + } +} + #[test] fn function_version_job_result_matches_shared_canonical_golden() { let result = job_result("remote_function_job.json"); @@ -28,6 +49,13 @@ fn function_version_job_result_matches_shared_canonical_golden() { assert_eq!(version.name(), "embed"); assert_eq!(version.version(), "1"); assert_ne!(version.image().manifest_digest, version.version()); + assert_eq!( + version.secret_bindings(), + [SecretBinding::Env { + variable: "HF_TOKEN".to_string(), + secret_ref: SecretReference::new("hf-prod"), + }] + ); assert_eq!( version.to_canonical_json().expect("canonical JSON"), fixture("remote_function_version.canonical.json").trim() @@ -150,3 +178,75 @@ fn floating_point_application_literals_are_rejected_consistently() { .contains("floating-point Function literals") ); } + +#[test] +fn canonical_client_values_carry_bindings_and_no_credentials() { + let result = job_result("remote_function_job.json"); + let version = FunctionVersion::from_json(&result.to_string()).expect("FunctionVersion result"); + let canonical: Value = serde_json::from_str( + &version + .to_canonical_json() + .expect("canonical FunctionVersion"), + ) + .expect("canonical JSON"); + + assert_eq!( + canonical["secret_bindings"], + serde_json::json!([{"kind": "env", "variable": "HF_TOKEN", "secret_ref": {"name": "hf-prod"}}]) + ); + assert_no_secret_values(&canonical); +} + +/// A binding kind a newer server introduces must not fail the whole version. +/// +/// This is the cost the union pays for being one field: an unknown variant is +/// a decode error unless it is caught, so it is caught -- and the payload is +/// dropped rather than retained, as `PythonRuntimeSpec` does, because the +/// client does not proxy catalog values. +#[test] +fn an_unknown_binding_kind_is_forward_decodable() { + let mut result = job_result("remote_function_job.json"); + result["secret_bindings"] = serde_json::json!([ + {"kind": "env", "variable": "HF_TOKEN", "secret_ref": {"name": "hf-prod"}}, + {"kind": "file", "path": "/run/secrets/tok", "secret_ref": {"name": "hf-prod"}}, + ]); + + let version = FunctionVersion::from_json(&result.to_string()).expect("future binding kind"); + + let kinds = version + .secret_bindings() + .iter() + .map(|binding| binding.kind()) + .collect::>(); + assert_eq!(kinds, ["env", "file"]); + assert_eq!(version.secret_bindings()[1].variable(), None); + assert_eq!(version.secret_bindings()[1].secret(), None); + + // The unknown kind round-trips as its discriminator and nothing more. + let canonical: Value = + serde_json::from_str(&version.to_canonical_json().expect("canonical")).expect("JSON"); + assert_eq!( + canonical["secret_bindings"][1], + serde_json::json!({"kind": "file"}) + ); +} + +/// A Function that binds nothing carries no `secret_bindings` key: absent +/// decodes as an empty list, and an empty list serializes back to absent. +#[test] +fn a_version_without_bindings_omits_the_field_in_both_directions() { + let mut result = job_result("remote_function_job.json"); + result + .as_object_mut() + .expect("Function version object") + .remove("secret_bindings"); + let version = FunctionVersion::from_json(&result.to_string()).expect("FunctionVersion result"); + + assert!(version.secret_bindings().is_empty()); + assert!( + !version + .to_canonical_json() + .expect("canonical FunctionVersion") + .contains("secret_bindings") + ); +} diff --git a/rust/lancedb/tests/first_class_function_slice2.rs b/rust/lancedb/tests/first_class_function_slice2.rs index 65924f394..14ceb511d 100644 --- a/rust/lancedb/tests/first_class_function_slice2.rs +++ b/rust/lancedb/tests/first_class_function_slice2.rs @@ -6,6 +6,8 @@ use std::path::PathBuf; use lancedb::Error; use lancedb::function::FunctionRegistrationRequest; +use lancedb::secrets::{SecretBinding, SecretReference}; +use serde_json::Value; fn fixture(name: &str) -> String { let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) @@ -14,6 +16,26 @@ fn fixture(name: &str) -> String { fs::read_to_string(path).expect("fixture must be readable") } +/// A registration request never models a resolved credential, at any depth. +fn assert_no_secret_values(value: &Value) { + match value { + Value::Object(values) => { + for (key, value) in values { + assert!( + !matches!( + key.as_str(), + "secret_value" | "secret_values" | "resolved_secret" | "resolved_secrets" + ), + "registration requests must not model resolved secret material" + ); + assert_no_secret_values(value); + } + } + Value::Array(values) => values.iter().for_each(assert_no_secret_values), + _ => {} + } +} + #[test] fn registration_request_matches_shared_canonical_golden() { let request = FunctionRegistrationRequest::from_json(&fixture( @@ -22,10 +44,45 @@ fn registration_request_matches_shared_canonical_golden() { .expect("registration request"); assert_eq!(request.name, "normalize_score"); assert_eq!(request.artifact.adapter.kind, "scalar_to_arrow_batch"); + // The unchanged path: a Function that binds nothing serializes today's + // bytes, with no `secret_bindings` key at all. + assert!(request.secret_bindings.is_empty()); assert_eq!( request.to_canonical_json().expect("canonical request"), fixture("remote_function_registration_request.canonical.json").trim() ); + + let value: Value = + serde_json::from_str(&request.to_canonical_json().expect("canonical request")) + .expect("request JSON"); + assert_no_secret_values(&value); +} + +/// The same shared golden as the Python suite builds from `@udf(secrets=...)` +/// plus `bind_secrets`, so both clients agree byte for byte on a bound request. +#[test] +fn secret_bound_registration_request_matches_shared_canonical_golden() { + let request = FunctionRegistrationRequest::from_json(&fixture( + "remote_function_secret_registration_request.json", + )) + .expect("registration request"); + assert_eq!(request.name, "analyze_caption"); + assert_eq!( + request.secret_bindings, + [SecretBinding::Env { + variable: "OPENAI_API_KEY".to_string(), + secret_ref: SecretReference::new("openai-prod"), + }] + ); + assert_eq!( + request.to_canonical_json().expect("canonical request"), + fixture("remote_function_secret_registration_request.canonical.json").trim() + ); + + let value: Value = + serde_json::from_str(&request.to_canonical_json().expect("canonical request")) + .expect("request JSON"); + assert_no_secret_values(&value); } #[tokio::test] diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json index 3381fd266..c6bbd83e3 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_job.json @@ -58,7 +58,16 @@ } }, "source": false - } + }, + "secret_bindings": [ + { + "kind": "env", + "variable": "HF_TOKEN", + "secret_ref": { + "name": "hf-prod" + } + } + ] }, "future_job": { "trace_id": "trace-1" diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.canonical.json new file mode 100644 index 000000000..5cce05e9b --- /dev/null +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.canonical.json @@ -0,0 +1 @@ +{"artifact":{"adapter":{"kind":"scalar_to_arrow_batch","version":1},"content":{"data":"ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIGFuYWx5emVfY2FwdGlvbihjYXB0aW9uOiBzdHIpIC0+IHN0cjoKICAgIHJldHVybiBjYXB0aW9uLnN0cmlwKCkK","encoding":"base64"},"digest":"sha256:800462c9ad15151a80f83f85b8912ff149300c1563e07f58448f099afcd0d077","entrypoint":"analyze_caption","kind":"python_callable"},"name":"analyze_caption","runtime":{"env":{"MODE":"test"},"environment":{"kind":"pip","packages":["openai==3.7.0"]},"kind":"python","python_version":"3.12"},"secret_bindings":[{"kind":"env","secret_ref":{"name":"openai-prod"},"variable":"OPENAI_API_KEY"}],"signature":{"inputs":[{"arrow_type":"utf8","name":"caption","nullable":false}],"output":{"arrow_type":"utf8","kind":"scalar","nullable":false}}} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.json new file mode 100644 index 000000000..6edea0a91 --- /dev/null +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_secret_registration_request.json @@ -0,0 +1,52 @@ +{ + "artifact": { + "adapter": { + "kind": "scalar_to_arrow_batch", + "version": 1 + }, + "content": { + "data": "ZnJvbSBfX2Z1dHVyZV9fIGltcG9ydCBhbm5vdGF0aW9ucwoKZGVmIGFuYWx5emVfY2FwdGlvbihjYXB0aW9uOiBzdHIpIC0+IHN0cjoKICAgIHJldHVybiBjYXB0aW9uLnN0cmlwKCkK", + "encoding": "base64" + }, + "digest": "sha256:800462c9ad15151a80f83f85b8912ff149300c1563e07f58448f099afcd0d077", + "entrypoint": "analyze_caption", + "kind": "python_callable" + }, + "name": "analyze_caption", + "runtime": { + "env": { + "MODE": "test" + }, + "environment": { + "kind": "pip", + "packages": [ + "openai==3.7.0" + ] + }, + "kind": "python", + "python_version": "3.12" + }, + "signature": { + "inputs": [ + { + "arrow_type": "utf8", + "name": "caption", + "nullable": false + } + ], + "output": { + "arrow_type": "utf8", + "kind": "scalar", + "nullable": false + } + }, + "secret_bindings": [ + { + "kind": "env", + "variable": "OPENAI_API_KEY", + "secret_ref": { + "name": "openai-prod" + } + } + ] +} diff --git a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json index 3f0c11973..63f3c824e 100644 --- a/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json +++ b/rust/lancedb/tests/fixtures/first_class_functions/v1/remote_function_version.canonical.json @@ -1 +1 @@ -{"created_at":"2026-08-21T00:00:00Z","disabled":false,"image":{"descriptor":{"behavior":{"result_stability":"input_and_context","side_effects":"non_idempotent"},"entrypoint":"app.function:create","format_version":1,"interface":{"type":"lance.scalar","version":1},"python":{"abi_tag":"cp312","executable":"/usr/local/bin/python3","implementation":"cpython","import_paths":["/opt/function/code"],"version":"3.12.14"},"python_api":1,"requires":{"capabilities":[],"kernel_min":"4.18.0"},"schemas":{"initialization":"/opt/function/schemas/initialization.arrow","input":"/opt/function/schemas/input.arrow","output":"/opt/function/schemas/output.arrow"}},"manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","source":false},"location":"memory:///fixture","metadata":{},"name":"embed","object_id":"fixture","signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list","kind":"scalar","nullable":false}},"version":"1"} +{"created_at":"2026-08-21T00:00:00Z","disabled":false,"image":{"descriptor":{"behavior":{"result_stability":"input_and_context","side_effects":"non_idempotent"},"entrypoint":"app.function:create","format_version":1,"interface":{"type":"lance.scalar","version":1},"python":{"abi_tag":"cp312","executable":"/usr/local/bin/python3","implementation":"cpython","import_paths":["/opt/function/code"],"version":"3.12.14"},"python_api":1,"requires":{"capabilities":[],"kernel_min":"4.18.0"},"schemas":{"initialization":"/opt/function/schemas/initialization.arrow","input":"/opt/function/schemas/input.arrow","output":"/opt/function/schemas/output.arrow"}},"manifest_digest":"sha256:7e22f815b6648e14f093a3979a8e5a2082fa773ebe1ec84b135cae7e84d6f8e6","source":false},"location":"memory:///fixture","metadata":{},"name":"embed","object_id":"fixture","secret_bindings":[{"kind":"env","secret_ref":{"name":"hf-prod"},"variable":"HF_TOKEN"}],"signature":{"inputs":[{"arrow_type":"utf8","name":"text","nullable":true}],"output":{"arrow_type":"list","kind":"scalar","nullable":false}},"version":"1"} From c72931b30fa8cd6f27e2c061187a00cc21011670 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 17 Sep 2026 15:52:34 +0800 Subject: [PATCH 37/43] feat(python): add TypeSafe reranker (#4209) Adds `TypeSafeReranker`, which reranks vector, FTS, and hybrid results with the [TypeSafe System One API](https://docs.typesafe.ai/introduction). Each result is scored independently: TypeSafe reads `{"query", "document"}` and answers a yes/no (noul) question, and the probability of yes becomes `_relevance_score`. Unlike listwise LLM rerankers, the score is an absolute probability, so it is comparable across queries and can be thresholded. The question's `instructions` and `true`/`false` `criteria` are configurable, since domain-specific criteria are what make this kind of scoring work well ([TypeSafe's re-ranking cookbook](https://docs.typesafe.ai/cookbooks/rerank_typesafe)). The API takes one state per request, so the reranker sends one request per result on a thread pool bounded by `max_concurrency`. It deliberately does not use the background event loop: rerankers are called synchronously from inside the async query APIs, where `LOOP.run` would deadlock. TypeSafe scores for the same pair vary slightly between calls, so results with close scores can swap places when a search is repeated. The shared reranker test helper now takes `deterministic=False` for this case: it still checks result sizes and descending scores, but not that two identical searches return the same order. The SDK is imported only when the client is created and questions are sent as plain dicts, so the new tests run in CI with a fake client and without `typesafe-sdk` installed. The live-API test is skipped without `TYPESAFE_API_KEY`. Ranking quality has not been compared with other API rerankers. --- python/pyproject.toml | 1 + python/python/lancedb/rerankers/__init__.py | 2 + python/python/lancedb/rerankers/typesafe.py | 159 ++++++++++++++++++++ python/python/tests/test_rerankers.py | 107 ++++++++++++- 4 files changed, 266 insertions(+), 3 deletions(-) create mode 100644 python/python/lancedb/rerankers/typesafe.py diff --git a/python/pyproject.toml b/python/pyproject.toml index dc5ea603d..de84367a8 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -126,6 +126,7 @@ include = [ "python/lancedb/rerankers/util.py", "python/lancedb/rerankers/__init__.py", "python/lancedb/rerankers/voyageai.py", + "python/lancedb/rerankers/typesafe.py", "python/lancedb/rerankers/jinaai.py", "python/lancedb/rerankers/openai.py", "python/lancedb/rerankers/cross_encoder.py", diff --git a/python/python/lancedb/rerankers/__init__.py b/python/python/lancedb/rerankers/__init__.py index aef2d331e..bd906f831 100644 --- a/python/python/lancedb/rerankers/__init__.py +++ b/python/python/lancedb/rerankers/__init__.py @@ -11,6 +11,7 @@ from .jinaai import JinaReranker from .rrf import RRFReranker from .mrr import MRRReranker from .answerdotai import AnswerdotaiRerankers +from .typesafe import TypeSafeReranker from .voyageai import VoyageAIReranker from .watsonx import WatsonxReranker @@ -27,6 +28,7 @@ __all__ = [ "JinaReranker", "RRFReranker", "AnswerdotaiRerankers", + "TypeSafeReranker", "VoyageAIReranker", "MRRReranker", "WatsonxReranker", diff --git a/python/python/lancedb/rerankers/typesafe.py b/python/python/lancedb/rerankers/typesafe.py new file mode 100644 index 000000000..2da9b673c --- /dev/null +++ b/python/python/lancedb/rerankers/typesafe.py @@ -0,0 +1,159 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright The LanceDB Authors + + +from concurrent.futures import ThreadPoolExecutor +from functools import cached_property +from typing import Any, Dict, Mapping, Optional + +import pyarrow as pa + +from ..util import attempt_import_or_raise +from .base import Reranker + +DEFAULT_INSTRUCTIONS = ( + "Does the document contain information that answers or directly addresses " + "the query?" +) +DEFAULT_CRITERIA = { + "true": "The document answers the query or states facts the query asks about.", + "false": "The document is off-topic, or only shares keywords or a general " + "subject with the query without addressing it.", +} +# The key TypeSafe returns the answer under. It is not sent to the model. +_QUESTION_ID = "relevance" + + +class TypeSafeReranker(Reranker): + """ + Reranks the results using the TypeSafe System One API. + https://docs.typesafe.ai/cookbooks/rerank_typesafe + + Each result is scored independently: TypeSafe reads the query together with + the result's ``column`` value and answers a yes/no ("noul") question about + whether the document is relevant. The probability of "yes", between 0 and 1, + becomes the ``_relevance_score``. Because every score is an absolute + probability rather than a position in the list, scores are comparable + across queries and can be thresholded. They are model estimates, though, + and can vary slightly between identical calls, so results with close scores + may swap places when the same search is repeated. + + One request is sent per result, up to ``max_concurrency`` at a time. + + Parameters + ---------- + model_name : str, default "jev-latest" + The TypeSafe model to use. + column : str, default "text" + The name of the column holding the document text to score. + instructions : str, optional + The yes/no question asked about each query and document pair. The state + TypeSafe reads is ``{"query": , "document": }``. + Defaults to a generic relevance question. + criteria : Mapping[str, str], optional + What a "yes" and a "no" mean, as a mapping with the keys ``"true"`` and + ``"false"``. Domain-specific criteria usually rank better than the + generic default. Pass an empty mapping to send no criteria. + return_score : str, default "relevance" + Options are "relevance" or "all". If "all", keeps the vector and FTS + scores alongside the relevance score. + api_key : str, optional + The API key to use. If None, the TypeSafe SDK reads the + ``TYPESAFE_API_KEY`` environment variable. + max_concurrency : int, default 8 + The maximum number of TypeSafe requests in flight for one rerank call. + """ + + def __init__( + self, + model_name: str = "jev-latest", + column: str = "text", + instructions: Optional[str] = None, + criteria: Optional[Mapping[str, str]] = None, + return_score: str = "relevance", + api_key: Optional[str] = None, + max_concurrency: int = 8, + ): + super().__init__(return_score) + if max_concurrency < 1: + raise ValueError("max_concurrency must be at least 1") + criteria = DEFAULT_CRITERIA if criteria is None else dict(criteria) + unknown = set(criteria) - {"true", "false"} + if unknown: + raise ValueError( + f"criteria keys must be 'true' or 'false', got {sorted(unknown)}" + ) + self.model_name = model_name + self.column = column + self.instructions = instructions or DEFAULT_INSTRUCTIONS + self.criteria = criteria + self.api_key = api_key + self.max_concurrency = max_concurrency + + def __str__(self): + return f"TypeSafeReranker(model_name={self.model_name})" + + @cached_property + def _client(self): + typesafe_sdk = attempt_import_or_raise("typesafe_sdk", "typesafe-sdk") + return typesafe_sdk.TypeSafeClient(api_key=self.api_key) + + @cached_property + def _question(self) -> Dict[str, Any]: + question: Dict[str, Any] = {"type": "noul", "instructions": self.instructions} + if self.criteria: + question["criteria"] = self.criteria + return question + + def _score(self, query: str, document: Optional[str]) -> float: + if document is None: + return 0.0 + response = self._client.system_one( + state={"query": query, "document": document}, + questions={_QUESTION_ID: self._question}, + model=self.model_name, + ) + return response.answers[_QUESTION_ID].noul + + def _rerank(self, result_set: pa.Table, query: str) -> pa.Table: + result_set = self._handle_empty_results(result_set) + if len(result_set) == 0: + return result_set + docs = result_set[self.column].to_pylist() + # Rerankers are also called synchronously from inside the async query + # APIs, so the requests run on threads rather than on an event loop. + with ThreadPoolExecutor( + max_workers=min(self.max_concurrency, len(docs)) + ) as pool: + scores = list(pool.map(lambda doc: self._score(query, doc), docs)) + result_set = result_set.append_column( + "_relevance_score", pa.array(scores, type=pa.float32()) + ) + return result_set.sort_by([("_relevance_score", "descending")]) + + def rerank_hybrid( + self, + query: str, + vector_results: pa.Table, + fts_results: pa.Table, + ): + if self.score == "all": + combined_results = self._merge_and_keep_scores(vector_results, fts_results) + else: + combined_results = self.merge_results(vector_results, fts_results) + combined_results = self._rerank(combined_results, query) + if self.score == "relevance": + combined_results = self._keep_relevance_score(combined_results) + return combined_results + + def rerank_vector(self, query: str, vector_results: pa.Table): + vector_results = self._rerank(vector_results, query) + if self.score == "relevance": + vector_results = vector_results.drop_columns(["_distance"]) + return vector_results + + def rerank_fts(self, query: str, fts_results: pa.Table): + fts_results = self._rerank(fts_results, query) + if self.score == "relevance": + fts_results = fts_results.drop_columns(["_score"]) + return fts_results diff --git a/python/python/tests/test_rerankers.py b/python/python/tests/test_rerankers.py index 4430fa98f..290164a13 100644 --- a/python/python/tests/test_rerankers.py +++ b/python/python/tests/test_rerankers.py @@ -21,6 +21,7 @@ from lancedb.rerankers import ( OpenaiReranker, JinaReranker, AnswerdotaiRerankers, + TypeSafeReranker, VoyageAIReranker, MRRReranker, WatsonxReranker, @@ -101,7 +102,15 @@ def get_test_table(tmp_path): return table, MyTable -def _run_test_reranker(reranker, table, query, query_vector, schema): +def _run_test_reranker( + reranker, table, query, query_vector, schema, deterministic=True +): + """Exercise a reranker across search types. + + Set ``deterministic=False`` for rerankers whose scores can vary between + identical calls, such as remote model APIs; repeated searches are then not + expected to return the same order. + """ # Hybrid search setting result1 = ( table.search(query, query_type="hybrid", vector_column_name="vector") @@ -113,7 +122,8 @@ def _run_test_reranker(reranker, table, query, query_vector, schema): .rerank(reranker=reranker) .to_pydantic(schema) ) - assert result1 == result2 + if deterministic: + assert result1 == result2 query_vector = table.to_pandas()["vector"][0] result = ( @@ -203,7 +213,9 @@ def _run_test_reranker(reranker, table, query, query_vector, schema): ) assert len(result_deduped) <= 20 result_arrow = reranker.rerank_multivector([rs1.to_arrow(), rs2.to_arrow()], query) - assert len(result) == 20 and result == result_arrow + assert len(result) == 20 and len(result_arrow) == 20 + if deterministic: + assert result == result_arrow def _run_test_hybrid_reranker(reranker, tmp_path): @@ -515,6 +527,95 @@ def test_voyageai_reranker(tmp_path): _run_test_reranker(reranker, table, "single player experience", None, schema) +class _FakeTypeSafeClient: + """Stands in for ``typesafe_sdk.TypeSafeClient``; scores by word overlap.""" + + def __init__(self): + self.requests = [] + + def system_one(self, state, questions, model): + self.requests.append((state, questions, model)) + query_words = set(state["query"].lower().split()) + doc_words = set(state["document"].lower().split()) + noul = len(query_words & doc_words) / len(query_words) + answers = {key: type("NoulAnswer", (), {"noul": noul})() for key in questions} + return type("SystemOneResponse", (), {"answers": answers})() + + +def test_typesafe_reranker_with_fake_client(tmp_path): + reranker = TypeSafeReranker(max_concurrency=4) + reranker._client = _FakeTypeSafeClient() + table, schema = get_test_table(tmp_path) + _run_test_reranker(reranker, table, "single player experience", None, schema) + + state, questions, model = reranker._client.requests[0] + assert model == "jev-latest" + assert set(state) == {"query", "document"} + assert questions == { + "relevance": { + "type": "noul", + "instructions": reranker.instructions, + "criteria": reranker.criteria, + } + } + + +def test_typesafe_reranker_scores_each_row(): + reranker = TypeSafeReranker( + column="body", + instructions="Is this about cats?", + criteria={}, + return_score="all", + ) + reranker._client = _FakeTypeSafeClient() + results = pa.table( + { + "body": ["dogs bark", "cats purr and cats nap", None, "cats"], + "_distance": [0.1, 0.2, 0.3, 0.4], + } + ) + + reranked = reranker.rerank_vector("cats nap", results) + + assert reranked["body"].to_pylist() == [ + "cats purr and cats nap", + "cats", + "dogs bark", + None, + ] + assert reranked["_relevance_score"].to_pylist() == [1.0, 0.5, 0.0, 0.0] + assert reranked["_distance"].to_pylist() == [0.2, 0.4, 0.1, 0.3] + # Null documents are scored 0 without a request. + assert len(reranker._client.requests) == 3 + assert reranker._client.requests[0][1] == { + "relevance": {"type": "noul", "instructions": "Is this about cats?"} + } + + +def test_typesafe_reranker_rejects_invalid_arguments(): + with pytest.raises(ValueError, match="criteria keys"): + TypeSafeReranker(criteria={"yes": "relevant"}) + with pytest.raises(ValueError, match="max_concurrency"): + TypeSafeReranker(max_concurrency=0) + + +@pytest.mark.skipif( + os.environ.get("TYPESAFE_API_KEY") is None, reason="TYPESAFE_API_KEY not set" +) +def test_typesafe_reranker(tmp_path): + pytest.importorskip("typesafe_sdk") + reranker = TypeSafeReranker() + table, schema = get_test_table(tmp_path) + _run_test_reranker( + reranker, + table, + "single player experience", + None, + schema, + deterministic=False, + ) + + def test_empty_result_reranker(): pytest.importorskip("sentence_transformers") db = lancedb.connect("memory://") From 5231d37f8d7af114a50ff55b057d4287e491756e Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Thu, 17 Sep 2026 05:12:50 -0700 Subject: [PATCH 38/43] feat: list a table's per-row Function errors from the client (#4208) A refresh running under a skip policy records each row it skipped, with the failing input and the error, but the client could not read that store: the server exposes it over SQL and, since recently, a REST route. A user who hit per-row failures still had to open a SQL session. `Table::function_errors` calls the route. The listing is table-addressed with optional job and column filters, the same addressing the SQL surface uses, so the two cannot disagree about what a table's errors are. The two non-record signals come back as their own fields rather than as rows: capped-fragment summaries, and whether the listing stopped at its limit. Local tables refuse rather than answer with an empty list. Python and Node expose the same call, with the same optional filters. --- Cargo.lock | 4 +- docs/src/js/classes/Table.md | 35 +++++ docs/src/js/globals.md | 4 + .../js/interfaces/FunctionErrorFragment.md | 42 ++++++ docs/src/js/interfaces/FunctionErrorRecord.md | 92 +++++++++++++ docs/src/js/interfaces/FunctionErrors.md | 39 ++++++ .../js/interfaces/FunctionErrorsOptions.md | 39 ++++++ docs/src/python/python.md | 6 + nodejs/__test__/remote.test.ts | 60 ++++++++ nodejs/__test__/table.test.ts | 8 ++ nodejs/lancedb/index.ts | 4 + nodejs/lancedb/table.ts | 31 +++++ nodejs/src/table.rs | 117 ++++++++++++++++ python/python/lancedb/__init__.py | 3 + python/python/lancedb/_lancedb.pyi | 29 ++++ python/python/lancedb/remote/table.py | 10 ++ python/python/lancedb/table.py | 97 +++++++++++++ python/python/tests/test_remote_db.py | 65 +++++++++ python/python/tests/test_table.py | 7 + python/src/lib.rs | 8 +- python/src/table.rs | 130 ++++++++++++++++++ rust/lancedb/src/function.rs | 95 +++++++++++++ rust/lancedb/src/remote/table.rs | 111 +++++++++++++++ rust/lancedb/src/table.rs | 67 +++++++++ 24 files changed, 1099 insertions(+), 4 deletions(-) create mode 100644 docs/src/js/interfaces/FunctionErrorFragment.md create mode 100644 docs/src/js/interfaces/FunctionErrorRecord.md create mode 100644 docs/src/js/interfaces/FunctionErrors.md create mode 100644 docs/src/js/interfaces/FunctionErrorsOptions.md diff --git a/Cargo.lock b/Cargo.lock index 0d6a28878..1d1861a59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5368,8 +5368,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "13.0.0-beta.3" -source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.3#565808c992c3d2adf1909b9753b50a28b6bd8a19" +version = "13.0.0-beta.4" +source = "git+https://github.com/lance-format/lance.git?tag=v13.0.0-beta.4#fea665e0d96d73acd1be330ce850973ff00b7126" dependencies = [ "datafusion", "geo-traits", diff --git a/docs/src/js/classes/Table.md b/docs/src/js/classes/Table.md index dfa0a9819..a8c382c27 100644 --- a/docs/src/js/classes/Table.md +++ b/docs/src/js/classes/Table.md @@ -578,6 +578,41 @@ so this is safe to call repeatedly. *** +### functionErrors() + +```ts +abstract functionErrors(options?): Promise +``` + +The per-row errors Function refreshes recorded on this table. + +A refresh running under a skip policy records each row it skipped with +the input that failed and the error. This lists those records, newest +job first, plus a summary for any fragment whose per-row detail was +capped. LanceDB Cloud and Enterprise only; reading errors needs read +access to the table, since a message carries the value that failed. + +#### Parameters + +* **options?**: [`FunctionErrorsOptions`](../interfaces/FunctionErrorsOptions.md) + Optional filters: `jobId`, + `column`, and `limit` (server default 10000, cap 100000). + +#### Returns + +`Promise`<[`FunctionErrors`](../interfaces/FunctionErrors.md)> + +The records, the capped fragments, +and whether the listing stopped at its limit. + +#### Example + +```ts +const { records, truncated } = await table.functionErrors({ column: "embedding" }); +``` + +*** + ### getLsmStats() ```ts diff --git a/docs/src/js/globals.md b/docs/src/js/globals.md index 92e7940f0..6d242a15b 100644 --- a/docs/src/js/globals.md +++ b/docs/src/js/globals.md @@ -92,6 +92,10 @@ - [FtsToken](interfaces/FtsToken.md) - [FullTextQuery](interfaces/FullTextQuery.md) - [FullTextSearchOptions](interfaces/FullTextSearchOptions.md) +- [FunctionErrorFragment](interfaces/FunctionErrorFragment.md) +- [FunctionErrorRecord](interfaces/FunctionErrorRecord.md) +- [FunctionErrors](interfaces/FunctionErrors.md) +- [FunctionErrorsOptions](interfaces/FunctionErrorsOptions.md) - [GenerationStats](interfaces/GenerationStats.md) - [HnswPqOptions](interfaces/HnswPqOptions.md) - [HnswSqOptions](interfaces/HnswSqOptions.md) diff --git a/docs/src/js/interfaces/FunctionErrorFragment.md b/docs/src/js/interfaces/FunctionErrorFragment.md new file mode 100644 index 000000000..3a65d5486 --- /dev/null +++ b/docs/src/js/interfaces/FunctionErrorFragment.md @@ -0,0 +1,42 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / FunctionErrorFragment + +# Interface: FunctionErrorFragment + +A fragment whose per-row error detail was capped: `rowsSkipped` rows +failed, of which only `rowsRecorded` have a record of their own. + +## Properties + +### fragmentId + +```ts +fragmentId: number; +``` + +*** + +### jobId + +```ts +jobId: string; +``` + +*** + +### rowsRecorded + +```ts +rowsRecorded: number; +``` + +*** + +### rowsSkipped + +```ts +rowsSkipped: number; +``` diff --git a/docs/src/js/interfaces/FunctionErrorRecord.md b/docs/src/js/interfaces/FunctionErrorRecord.md new file mode 100644 index 000000000..3a26534a0 --- /dev/null +++ b/docs/src/js/interfaces/FunctionErrorRecord.md @@ -0,0 +1,92 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / FunctionErrorRecord + +# Interface: FunctionErrorRecord + +One row a Function refresh skipped, as the server recorded it. + +## Properties + +### column + +```ts +column: string; +``` + +*** + +### createdAtMillis + +```ts +createdAtMillis: number; +``` + +*** + +### errorMessage + +```ts +errorMessage: string; +``` + +*** + +### errorType + +```ts +errorType: string; +``` + +*** + +### fragmentId + +```ts +fragmentId: number; +``` + +*** + +### function + +```ts +function: string; +``` + +*** + +### functionVersion + +```ts +functionVersion: string; +``` + +*** + +### jobId + +```ts +jobId: string; +``` + +*** + +### rowOffset? + +```ts +optional rowOffset: number; +``` + +The row's offset within the fragment; absent when the fragment's +detail was capped. + +*** + +### tableVersion + +```ts +tableVersion: number; +``` diff --git a/docs/src/js/interfaces/FunctionErrors.md b/docs/src/js/interfaces/FunctionErrors.md new file mode 100644 index 000000000..bee0a80d8 --- /dev/null +++ b/docs/src/js/interfaces/FunctionErrors.md @@ -0,0 +1,39 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / FunctionErrors + +# Interface: FunctionErrors + +A table's per-row Function errors. + +## Properties + +### fragments + +```ts +fragments: FunctionErrorFragment[]; +``` + +Fragments whose detail was capped. + +*** + +### records + +```ts +records: FunctionErrorRecord[]; +``` + +The recorded rows, newest job first. + +*** + +### truncated + +```ts +truncated: boolean; +``` + +Whether the listing stopped at its limit. diff --git a/docs/src/js/interfaces/FunctionErrorsOptions.md b/docs/src/js/interfaces/FunctionErrorsOptions.md new file mode 100644 index 000000000..af0e71cfe --- /dev/null +++ b/docs/src/js/interfaces/FunctionErrorsOptions.md @@ -0,0 +1,39 @@ +[**@lancedb/lancedb**](../README.md) • **Docs** + +*** + +[@lancedb/lancedb](../globals.md) / FunctionErrorsOptions + +# Interface: FunctionErrorsOptions + +Which per-row Function errors to list; every filter is optional. + +## Properties + +### column? + +```ts +optional column: string; +``` + +Only errors on this column. + +*** + +### jobId? + +```ts +optional jobId: string; +``` + +Only errors recorded by this job. + +*** + +### limit? + +```ts +optional limit: number; +``` + +At most this many records (server default 10000, cap 100000). diff --git a/docs/src/python/python.md b/docs/src/python/python.md index 80899b964..d1ba66280 100644 --- a/docs/src/python/python.md +++ b/docs/src/python/python.md @@ -170,6 +170,12 @@ listing a storage directory. ::: lancedb.functions.RefreshColumnResult +::: lancedb.FunctionErrors + +::: lancedb.FunctionErrorRecord + +::: lancedb.FunctionErrorFragment + ::: lancedb.job.Job ::: lancedb.job.AsyncJob diff --git a/nodejs/__test__/remote.test.ts b/nodejs/__test__/remote.test.ts index 85d725825..40f4c67db 100644 --- a/nodejs/__test__/remote.test.ts +++ b/nodejs/__test__/remote.test.ts @@ -195,6 +195,66 @@ describe("remote connection", () => { ); }); + it("lists the rows a Function refresh skipped", async () => { + const bodies: unknown[] = []; + await withMockDatabase( + (req, res) => { + const path = req.url ?? ""; + if (path.endsWith("/describe/")) { + res.writeHead(200, { "Content-Type": "application/json" }).end( + JSON.stringify({ + name: "docs", + version: 1, + schema: { fields: [] }, + }), + ); + return; + } + if (path === "/v1/table/docs/errors") { + let body = ""; + req.on("data", (chunk) => { + body += chunk; + }); + req.on("end", () => { + bodies.push(JSON.parse(body)); + res.writeHead(200, { "Content-Type": "application/json" }).end( + `{"records": [{"job_id": "j-7", "fragment_id": 3, "row_offset": 9, + "column": "embedding", "function": "embed", "function_version": "2", + "table_version": 11, "error_type": "ValueError", + "error_message": "bad input 'x'", "created_at_millis": 1700000000000}], + "fragments": [{"job_id": "j-7", "fragment_id": 4, "rows_skipped": 500, + "rows_recorded": 100}], "truncated": true}`, + ); + }); + return; + } + res.writeHead(404).end(); + }, + async (db) => { + const table = await db.openTable("docs"); + const errors = await table.functionErrors({ + jobId: "j-7", + column: "embedding", + limit: 2, + }); + expect(errors.truncated).toBe(true); + expect(errors.records.map((r) => r.errorMessage)).toEqual([ + "bad input 'x'", + ]); + expect(errors.records[0].rowOffset).toBe(9); + expect(errors.fragments[0].rowsSkipped).toBe(500); + await table.functionErrors(); + await expect(table.functionErrors({ limit: -1 })).rejects.toThrow( + "limit must be a non-negative integer", + ); + }, + ); + expect(bodies).toEqual([ + JSON.parse('{"job_id": "j-7", "column": "embedding", "limit": 2}'), + {}, + ]); + }); + it("surfaces JSON server errors from remote table operations", async () => { await withMockDatabase( (req, res) => { diff --git a/nodejs/__test__/table.test.ts b/nodejs/__test__/table.test.ts index 852bdec80..4142af696 100644 --- a/nodejs/__test__/table.test.ts +++ b/nodejs/__test__/table.test.ts @@ -4358,6 +4358,14 @@ describe("computed columns", () => { expect(rows.map((r) => r.doubled).sort()).toEqual([2, 4]); }); + it("records Function errors only on remote tables", async () => { + const db = await connect(tmpDir.name); + const table = await db.createTable("errors_local", [{ x: 1 }]); + await expect(table.functionErrors()).rejects.toThrow( + "LanceDB Cloud and Enterprise", + ); + }); + it("returns a job handle from refreshColumnAsync", async () => { const db = await connect(tmpDir.name); const table = await db.createTable("computed_job", [{ x: 1 }, { x: 2 }]); diff --git a/nodejs/lancedb/index.ts b/nodejs/lancedb/index.ts index 39ceec917..c0a4dfcbd 100644 --- a/nodejs/lancedb/index.ts +++ b/nodejs/lancedb/index.ts @@ -56,6 +56,10 @@ export { AddResult, AddColumnsResult, RefreshColumnResult, + FunctionErrors, + FunctionErrorsOptions, + FunctionErrorRecord, + FunctionErrorFragment, RefreshMaterializedViewResult, AlterColumnsResult, UpdateFieldMetadataResult, diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index 0280ac7f8..a098322a9 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -21,6 +21,7 @@ import { BlobFile } from "./blob"; import { EmbeddingFunctionConfig, getRegistry } from "./embedding/registry"; import { IndexOptions } from "./indices"; import { Job } from "./job"; +import { validateNonNegativeInteger } from "./materialized_view"; import { MergeInsertBuilder } from "./merge"; import { AddColumnsResult, @@ -30,6 +31,8 @@ import { BranchContents, DeleteResult, DropColumnsResult, + FunctionErrors, + FunctionErrorsOptions, IndexConfig, IndexStatistics, LsmStats, @@ -636,6 +639,27 @@ export abstract class Table { */ abstract refreshColumnAsync(column: string): Promise; + /** + * The per-row errors Function refreshes recorded on this table. + * + * A refresh running under a skip policy records each row it skipped with + * the input that failed and the error. This lists those records, newest + * job first, plus a summary for any fragment whose per-row detail was + * capped. LanceDB Cloud and Enterprise only; reading errors needs read + * access to the table, since a message carries the value that failed. + * @param {FunctionErrorsOptions} options Optional filters: `jobId`, + * `column`, and `limit` (server default 10000, cap 100000). + * @returns {Promise} The records, the capped fragments, + * and whether the listing stopped at its limit. + * @example + * ```ts + * const { records, truncated } = await table.functionErrors({ column: "embedding" }); + * ``` + */ + abstract functionErrors( + options?: FunctionErrorsOptions, + ): Promise; + /** * Recompute this table's contents from its materialized-view definition. * @@ -1363,6 +1387,13 @@ export class LocalTable extends Table { return new Job(await this.inner.refreshColumnAsync(column)); } + async functionErrors( + options?: FunctionErrorsOptions, + ): Promise { + validateNonNegativeInteger(options?.limit, "limit"); + return await this.inner.functionErrors(options); + } + async refreshMaterializedView( full?: boolean, sourceVersion?: number, diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 344017d08..f90e1fc91 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -427,6 +427,32 @@ impl Table { Ok(crate::job::Job::new(job)) } + #[napi(catch_unwind)] + pub async fn function_errors( + &self, + options: Option, + ) -> napi::Result { + let options = options.unwrap_or_default(); + let limit = options + .limit + .map(|limit| { + usize::try_from(limit) + .map_err(|_| napi::Error::from_reason("limit must be a non-negative integer")) + }) + .transpose()?; + let request = lancedb::function::FunctionErrorsRequest { + job_id: options.job_id, + column: options.column, + limit, + }; + let errors = self + .inner_ref()? + .function_errors(request) + .await + .default_error()?; + Ok(errors.into()) + } + #[napi(catch_unwind)] pub async fn refresh_materialized_view( &self, @@ -1476,6 +1502,97 @@ pub struct RefreshColumnResult { pub version: i64, } +/// Which per-row Function errors to list; every filter is optional. +#[napi(object)] +#[derive(Clone, Debug, Default)] +pub struct FunctionErrorsOptions { + /// Only errors recorded by this job. + pub job_id: Option, + /// Only errors on this column. + pub column: Option, + /// At most this many records (server default 10000, cap 100000). + pub limit: Option, +} + +/// One row a Function refresh skipped, as the server recorded it. +#[napi(object)] +#[derive(Clone, Debug)] +pub struct FunctionErrorRecord { + pub job_id: String, + pub fragment_id: i64, + /// The row's offset within the fragment; absent when the fragment's + /// detail was capped. + pub row_offset: Option, + pub column: String, + pub function: String, + pub function_version: String, + pub table_version: i64, + pub error_type: String, + pub error_message: String, + pub created_at_millis: i64, +} + +impl From for FunctionErrorRecord { + fn from(record: lancedb::function::FunctionErrorRecord) -> Self { + Self { + job_id: record.job_id, + fragment_id: record.fragment_id as i64, + row_offset: record.row_offset.map(i64::from), + column: record.column, + function: record.function, + function_version: record.function_version, + table_version: record.table_version as i64, + error_type: record.error_type, + error_message: record.error_message, + created_at_millis: record.created_at_millis, + } + } +} + +/// A fragment whose per-row error detail was capped: `rowsSkipped` rows +/// failed, of which only `rowsRecorded` have a record of their own. +#[napi(object)] +#[derive(Clone, Debug)] +pub struct FunctionErrorFragment { + pub job_id: String, + pub fragment_id: i64, + pub rows_skipped: i64, + pub rows_recorded: i64, +} + +impl From for FunctionErrorFragment { + fn from(fragment: lancedb::function::FunctionErrorFragment) -> Self { + Self { + job_id: fragment.job_id, + fragment_id: fragment.fragment_id as i64, + rows_skipped: fragment.rows_skipped as i64, + rows_recorded: fragment.rows_recorded as i64, + } + } +} + +/// A table's per-row Function errors. +#[napi(object)] +#[derive(Clone, Debug)] +pub struct FunctionErrors { + /// The recorded rows, newest job first. + pub records: Vec, + /// Fragments whose detail was capped. + pub fragments: Vec, + /// Whether the listing stopped at its limit. + pub truncated: bool, +} + +impl From for FunctionErrors { + fn from(errors: lancedb::function::FunctionErrors) -> Self { + Self { + records: errors.records.into_iter().map(Into::into).collect(), + fragments: errors.fragments.into_iter().map(Into::into).collect(), + truncated: errors.truncated, + } + } +} + #[napi(object)] pub struct RefreshMaterializedViewResult { /// How the view was brought up to date: "rebuild", "incremental" or "no_op". diff --git a/python/python/lancedb/__init__.py b/python/python/lancedb/__init__.py index 05ad664e1..362e0cdbf 100644 --- a/python/python/lancedb/__init__.py +++ b/python/python/lancedb/__init__.py @@ -12,6 +12,9 @@ __version__ = importlib.metadata.version("lancedb") from ._lancedb import connect as lancedb_connect from ._lancedb import FtsToken +from ._lancedb import FunctionErrorFragment as FunctionErrorFragment +from ._lancedb import FunctionErrorRecord as FunctionErrorRecord +from ._lancedb import FunctionErrors as FunctionErrors from ._lancedb import LsmWriteSpec from ._lancedb import tokenize as _tokenize from .common import URI, sanitize_uri diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index a0039df05..7c6611608 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -461,6 +461,12 @@ class Table: ) -> AddColumnsResult: ... async def refresh_column(self, column: str) -> RefreshColumnResult: ... async def refresh_column_async(self, column: str) -> Job: ... + async def function_errors( + self, + job_id: Optional[str] = None, + column: Optional[str] = None, + limit: Optional[int] = None, + ) -> FunctionErrors: ... async def refresh_materialized_view( self, full: bool = False, source_version: Optional[int] = None ) -> RefreshMaterializedViewResult: ... @@ -820,6 +826,29 @@ class RefreshColumnResult: rows_filled: int version: int +class FunctionErrorRecord: + job_id: str + fragment_id: int + row_offset: Optional[int] + column: str + function: str + function_version: str + table_version: int + error_type: str + error_message: str + created_at_millis: int + +class FunctionErrorFragment: + job_id: str + fragment_id: int + rows_skipped: int + rows_recorded: int + +class FunctionErrors: + records: list[FunctionErrorRecord] + fragments: list[FunctionErrorFragment] + truncated: bool + class RefreshMaterializedViewResult: @staticmethod def from_json(value: str) -> RefreshMaterializedViewResult: ... diff --git a/python/python/lancedb/remote/table.py b/python/python/lancedb/remote/table.py index 89b958165..a9fc06bdd 100644 --- a/python/python/lancedb/remote/table.py +++ b/python/python/lancedb/remote/table.py @@ -994,6 +994,16 @@ class RemoteTable(Table): def refresh_column_async(self, column: str) -> Job[RefreshColumnResult]: return Job(LOOP.run(self._table.refresh_column_async(column))) + def function_errors( + self, + job_id: Optional[str] = None, + column: Optional[str] = None, + limit: Optional[int] = None, + ): + return LOOP.run( + self._table.function_errors(job_id=job_id, column=column, limit=limit) + ) + def alter_columns( self, *alterations: Iterable[Dict[str, str]] ) -> AlterColumnsResult: diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index 95b10422a..d9c27ba81 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -186,6 +186,7 @@ if TYPE_CHECKING: CompactionStats, Tag, AddColumnsResult, + FunctionErrors, RefreshColumnResult, AddResult, AlterColumnsResult, @@ -2308,6 +2309,47 @@ class Table(ABC): 'finished' """ + @abstractmethod + def function_errors( + self, + job_id: Optional[str] = None, + column: Optional[str] = None, + limit: Optional[int] = None, + ) -> "FunctionErrors": + """ + The per-row errors Function refreshes recorded on this table. + + A refresh running under a skip policy records each row it skipped + with the input that failed and the error. This lists those records, + newest job first, plus a summary for any fragment whose per-row + detail was capped. LanceDB Cloud and Enterprise only; reading errors + needs read access to the table, since a message carries the value + that failed. + + Parameters + ---------- + job_id: str, optional + Only errors recorded by this job. + column: str, optional + Only errors on this column. + limit: int, optional + At most this many records (server default 10000, cap 100000). + + Returns + ------- + FunctionErrors + ``records``, ``fragments`` and ``truncated``, the last saying + whether the listing stopped at its limit. + + Examples + -------- + >>> errors = table.function_errors(column="embedding") # doctest: +SKIP + >>> for record in errors.records: # doctest: +SKIP + ... print(record.job_id, record.row_offset, record.error_message) + >>> if errors.truncated: # doctest: +SKIP + ... print("listing stopped at the limit") + """ + @abstractmethod def alter_columns(self, *alterations: Iterable[Dict[str, str]]): """ @@ -4362,6 +4404,18 @@ class LanceTable(Table): """ return Job(LOOP.run(self._table.refresh_column_async(column))) + def function_errors( + self, + job_id: Optional[str] = None, + column: Optional[str] = None, + limit: Optional[int] = None, + ) -> "FunctionErrors": + """The per-row errors Function refreshes recorded on this table. See + [`Table.function_errors`][lancedb.table.Table.function_errors].""" + return LOOP.run( + self._table.function_errors(job_id=job_id, column=column, limit=limit) + ) + def alter_columns( self, *alterations: Iterable[Dict[str, str]] ) -> AlterColumnsResult: @@ -6428,6 +6482,49 @@ class AsyncTable: """ return await self._inner.refresh_column(column) + async def function_errors( + self, + job_id: Optional[str] = None, + column: Optional[str] = None, + limit: Optional[int] = None, + ) -> "FunctionErrors": + """ + The per-row errors Function refreshes recorded on this table. + + A refresh running under a skip policy records each row it skipped + with the input that failed and the error. This lists those records, + newest job first, plus a summary for any fragment whose per-row + detail was capped. LanceDB Cloud and Enterprise only; reading errors + needs read access to the table, since a message carries the value + that failed. + + Parameters + ---------- + job_id: str, optional + Only errors recorded by this job. + column: str, optional + Only errors on this column. + limit: int, optional + At most this many records (server default 10000, cap 100000). + + Returns + ------- + FunctionErrors + ``records``, ``fragments`` and ``truncated``, the last saying + whether the listing stopped at its limit. + + Examples + -------- + >>> errors = await table.function_errors(column="embedding") # doctest: +SKIP + >>> for record in errors.records: # doctest: +SKIP + ... print(record.job_id, record.row_offset, record.error_message) + >>> if errors.truncated: # doctest: +SKIP + ... print("listing stopped at the limit") + """ + return await self._inner.function_errors( + job_id=job_id, column=column, limit=limit + ) + async def refresh_column_async( self, column: str ) -> AsyncJob[RefreshColumnJobResult]: diff --git a/python/python/tests/test_remote_db.py b/python/python/tests/test_remote_db.py index 1e5a71e9a..1c887b218 100644 --- a/python/python/tests/test_remote_db.py +++ b/python/python/tests/test_remote_db.py @@ -982,6 +982,71 @@ def test_remote_refresh_async_returns_typed_terminal_result(): assert result.version == 8 +def test_remote_function_errors_lists_the_rows_a_refresh_skipped(): + listing = { + "records": [ + { + "job_id": "j-7", + "fragment_id": 3, + "row_offset": 9, + "column": "embedding", + "function": "embed", + "function_version": "2", + "table_version": 11, + "error_type": "ValueError", + "error_message": "bad input 'x'", + "created_at_millis": 1700000000000, + } + ], + "fragments": [ + { + "job_id": "j-7", + "fragment_id": 4, + "rows_skipped": 500, + "rows_recorded": 100, + } + ], + "truncated": True, + } + bodies = [] + + def handler(request): + content_len = int(request.headers.get("Content-Length", 0)) + body = request.rfile.read(content_len) if content_len > 0 else b"" + if request.path == "/v1/table/test/errors": + bodies.append(json.loads(body)) + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write(json.dumps(listing).encode()) + elif request.path == "/v1/table/test/describe/": + request.send_response(200) + request.send_header("Content-Type", "application/json") + request.end_headers() + request.wfile.write( + json.dumps({"version": 1, "schema": {"fields": []}}).encode() + ) + else: + request.send_response(404) + request.end_headers() + + with mock_lancedb_connection(handler) as db: + table = db.open_table("test") + errors = table.function_errors(job_id="j-7", column="embedding", limit=2) + everything = table.function_errors() + + assert bodies == [{"job_id": "j-7", "column": "embedding", "limit": 2}, {}] + assert errors.truncated is True + assert [r.error_message for r in errors.records] == ["bad input 'x'"] + assert errors.records[0].row_offset == 9 + assert errors.records[0].function_version == "2" + assert (errors.fragments[0].rows_skipped, errors.fragments[0].rows_recorded) == ( + 500, + 100, + ) + assert everything.truncated is True + + def test_remote_job_wait_raises_on_failure(): from lancedb.exceptions import JobFailedError from lancedb.index import BTree diff --git a/python/python/tests/test_table.py b/python/python/tests/test_table.py index a3cbf68f1..0b3dc01c3 100644 --- a/python/python/tests/test_table.py +++ b/python/python/tests/test_table.py @@ -4327,6 +4327,13 @@ async def test_computed_column_async(tmp_path): assert (await table.to_arrow())["tripled"].to_pylist() == [9] +def test_function_errors_are_remote_only(tmp_path): + db = lancedb.connect(tmp_path) + table = db.create_table("t", [{"x": 1}]) + with pytest.raises(NotImplementedError, match="LanceDB Cloud and Enterprise"): + table.function_errors() + + def test_refresh_column_async_returns_job(tmp_path): db = lancedb.connect(tmp_path) table = db.create_table("computed_job", [{"x": 1}, {"x": 2}]) diff --git a/python/src/lib.rs b/python/src/lib.rs index fd9a23798..ab04663c0 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -16,8 +16,9 @@ use query::{FTSQuery, HybridQuery, Query, VectorQuery}; use session::Session; use table::{ AddColumnsResult, AddResult, AlterColumnsResult, DeleteResult, DropColumnsResult, FtsToken, - LsmWriteSpec, MergeResult, PyBlobFile, RefreshColumnResult, RefreshMaterializedViewResult, - Table, UpdateFieldMetadataResult, UpdateResult, + FunctionErrorFragment, FunctionErrorRecord, FunctionErrors, LsmWriteSpec, MergeResult, + PyBlobFile, RefreshColumnResult, RefreshMaterializedViewResult, Table, + UpdateFieldMetadataResult, UpdateResult, }; pub mod arrow; @@ -84,6 +85,9 @@ pub fn _lancedb(py: Python, m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; + m.add_class::()?; + m.add_class::()?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/python/src/table.rs b/python/src/table.rs index 4cea61543..72e0095c6 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -441,6 +441,117 @@ impl From for RefreshColumnResult { } } +/// One row a Function refresh skipped, as the server recorded it. +#[pyclass(module = "lancedb._lancedb", get_all, from_py_object)] +#[derive(Clone, Debug)] +pub struct FunctionErrorRecord { + pub job_id: String, + pub fragment_id: u64, + pub row_offset: Option, + pub column: String, + pub function: String, + pub function_version: String, + pub table_version: u64, + pub error_type: String, + pub error_message: String, + pub created_at_millis: i64, +} + +#[pymethods] +impl FunctionErrorRecord { + pub fn __repr__(&self) -> String { + format!( + "FunctionErrorRecord(job_id={:?}, fragment_id={}, row_offset={:?}, column={:?}, \ + error_type={:?}, error_message={:?})", + self.job_id, + self.fragment_id, + self.row_offset, + self.column, + self.error_type, + self.error_message + ) + } +} + +impl From for FunctionErrorRecord { + fn from(record: lancedb::function::FunctionErrorRecord) -> Self { + Self { + job_id: record.job_id, + fragment_id: record.fragment_id, + row_offset: record.row_offset, + column: record.column, + function: record.function, + function_version: record.function_version, + table_version: record.table_version, + error_type: record.error_type, + error_message: record.error_message, + created_at_millis: record.created_at_millis, + } + } +} + +/// A fragment whose per-row error detail was capped. +#[pyclass(module = "lancedb._lancedb", get_all, from_py_object)] +#[derive(Clone, Debug)] +pub struct FunctionErrorFragment { + pub job_id: String, + pub fragment_id: u64, + pub rows_skipped: u64, + pub rows_recorded: u64, +} + +#[pymethods] +impl FunctionErrorFragment { + pub fn __repr__(&self) -> String { + format!( + "FunctionErrorFragment(job_id={:?}, fragment_id={}, rows_skipped={}, rows_recorded={})", + self.job_id, self.fragment_id, self.rows_skipped, self.rows_recorded + ) + } +} + +impl From for FunctionErrorFragment { + fn from(fragment: lancedb::function::FunctionErrorFragment) -> Self { + Self { + job_id: fragment.job_id, + fragment_id: fragment.fragment_id, + rows_skipped: fragment.rows_skipped, + rows_recorded: fragment.rows_recorded, + } + } +} + +/// A table's per-row Function errors. +#[pyclass(module = "lancedb._lancedb", get_all, from_py_object)] +#[derive(Clone, Debug)] +pub struct FunctionErrors { + pub records: Vec, + pub fragments: Vec, + pub truncated: bool, +} + +#[pymethods] +impl FunctionErrors { + pub fn __repr__(&self) -> String { + format!( + "FunctionErrors(records={}, fragments={}, truncated={})", + self.records.len(), + self.fragments.len(), + self.truncated + ) + } +} + +impl From for FunctionErrors { + fn from(errors: lancedb::function::FunctionErrors) -> Self { + Self { + records: errors.records.into_iter().map(Into::into).collect(), + fragments: errors.fragments.into_iter().map(Into::into).collect(), + truncated: errors.truncated, + } + } +} + #[pyclass(get_all, from_py_object)] #[derive(Clone, Debug)] pub struct RefreshMaterializedViewResult { @@ -1638,6 +1749,25 @@ impl Table { }) } + #[pyo3(signature = (job_id=None, column=None, limit=None))] + pub fn function_errors( + self_: PyRef<'_, Self>, + job_id: Option, + column: Option, + limit: Option, + ) -> PyResult> { + let inner = self_.inner_ref()?.clone(); + let request = lancedb::function::FunctionErrorsRequest { + job_id, + column, + limit, + }; + future_into_py(self_.py(), async move { + let errors = inner.function_errors(request).await.infer_error()?; + Ok(FunctionErrors::from(errors)) + }) + } + #[pyo3(signature = (full=false, source_version=None))] pub fn refresh_materialized_view( self_: PyRef<'_, Self>, diff --git a/rust/lancedb/src/function.rs b/rust/lancedb/src/function.rs index 0d593585a..3f790b6ab 100644 --- a/rust/lancedb/src/function.rs +++ b/rust/lancedb/src/function.rs @@ -731,6 +731,101 @@ pub struct RefreshColumnResult { pub published_version: Option, } +/// Which per-row errors [`crate::Table::function_errors`] lists. Every +/// filter is optional; the listing is table-addressed, so with none set it +/// covers every refresh of every column. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct FunctionErrorsRequest { + /// Only errors recorded by this job. + pub job_id: Option, + /// Only errors on this column. + pub column: Option, + /// At most this many records; the server default is 10000 and its cap + /// 100000. [`FunctionErrors::truncated`] says whether the cap was hit. + pub limit: Option, +} + +impl FunctionErrorsRequest { + /// A request with no filter. + pub fn new() -> Self { + Self::default() + } + + /// Only errors recorded by `job_id`. + pub fn job_id(mut self, job_id: impl Into) -> Self { + self.job_id = Some(job_id.into()); + self + } + + /// Only errors on `column`. + pub fn column(mut self, column: impl Into) -> Self { + self.column = Some(column.into()); + self + } + + /// At most `limit` records. + pub fn limit(mut self, limit: usize) -> Self { + self.limit = Some(limit); + self + } +} + +/// One row a Function refresh skipped, as the server recorded it. The +/// message carries the input that failed, which is why reading errors needs +/// read access to the table. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FunctionErrorRecord { + /// The refresh job that recorded the error. + pub job_id: String, + /// The fragment holding the row. + pub fragment_id: u64, + /// The row's offset within the fragment; `None` when the fragment's + /// detail was capped and only the fragment summary remains. + #[serde(default)] + pub row_offset: Option, + /// The column being computed. + pub column: String, + /// The Function that failed. + pub function: String, + /// The Function's version. + pub function_version: String, + /// The table version the refresh read. + pub table_version: u64, + /// The error's class, as the executor reported it. + pub error_type: String, + /// The error's text. + pub error_message: String, + /// When the error was recorded, in milliseconds since the epoch. + pub created_at_millis: i64, +} + +/// A fragment whose per-row detail was capped: `rows_skipped` rows failed, +/// of which only `rows_recorded` have a [`FunctionErrorRecord`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct FunctionErrorFragment { + /// The refresh job that recorded the errors. + pub job_id: String, + /// The fragment. + pub fragment_id: u64, + /// Rows the refresh skipped in this fragment. + pub rows_skipped: u64, + /// Rows with a record of their own. + pub rows_recorded: u64, +} + +/// A table's per-row Function errors; see [`crate::Table::function_errors`]. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct FunctionErrors { + /// The recorded rows, newest job first. + pub records: Vec, + /// Fragments whose detail was capped. + #[serde(default)] + pub fragments: Vec, + /// Whether the listing stopped at its limit. + #[serde(default)] + pub truncated: bool, +} + impl RefreshColumnResult { /// Deprecated compatibility alias for `rows_assigned`. pub fn rows_filled(&self) -> u64 { diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 75cf8cd00..4280c7437 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -3519,6 +3519,35 @@ impl BaseTable for RemoteTable { }))) } + async fn function_errors( + &self, + request: &crate::function::FunctionErrorsRequest, + ) -> Result { + let mut body = serde_json::json!({}); + if let Some(job_id) = &request.job_id { + body["job_id"] = serde_json::json!(job_id); + } + if let Some(column) = &request.column { + body["column"] = serde_json::json!(column); + } + if let Some(limit) = request.limit { + body["limit"] = serde_json::json!(limit); + } + self.apply_branch_body(&mut body); + let request = self + .client + .post(&format!("/v1/table/{}/errors", self.identifier)) + .json(&body); + let (request_id, response) = self.send(request, true).await?; + let response = self.check_table_response(&request_id, response).await?; + let body = response.text().await.err_to_http(request_id.clone())?; + serde_json::from_str(&body).map_err(|e| Error::Http { + source: format!("Failed to parse errors response: {}", e).into(), + request_id, + status_code: None, + }) + } + async fn alter_columns(&self, alterations: &[ColumnAlteration]) -> Result { self.check_mutable().await?; let body = alterations @@ -8188,6 +8217,88 @@ mod tests { ); } + /// The error listing is table-addressed with optional job and column + /// filters, mirroring the server's SQL surface, and the two non-record + /// signals come back as their own fields rather than as rows. + #[tokio::test] + async fn test_function_errors_lists_the_rows_a_refresh_skipped() { + use crate::function::{FunctionErrorFragment, FunctionErrorRecord, FunctionErrorsRequest}; + + let table = Table::new_with_handler("my_table", |request| { + assert_eq!(request.method(), "POST"); + assert_eq!(request.url().path(), "/v1/table/my_table/errors"); + let body = request.body().unwrap().as_bytes().unwrap(); + let value: serde_json::Value = serde_json::from_slice(body).unwrap(); + assert_eq!( + value, + serde_json::json!({"job_id": "j-7", "column": "embedding", "limit": 2}) + ); + http::Response::builder() + .status(200) + .body( + r#"{"records": [{"job_id": "j-7", "fragment_id": 3, "row_offset": 9, + "column": "embedding", "function": "embed", "function_version": "2", + "table_version": 11, "error_type": "ValueError", + "error_message": "bad input 'x'", "created_at_millis": 1700000000000}], + "fragments": [{"job_id": "j-7", "fragment_id": 4, "rows_skipped": 500, + "rows_recorded": 100}], + "truncated": true}"#, + ) + .unwrap() + }); + + let errors = table + .function_errors( + FunctionErrorsRequest::new() + .job_id("j-7") + .column("embedding") + .limit(2), + ) + .await + .unwrap(); + assert_eq!( + errors.records, + [FunctionErrorRecord { + job_id: "j-7".into(), + fragment_id: 3, + row_offset: Some(9), + column: "embedding".into(), + function: "embed".into(), + function_version: "2".into(), + table_version: 11, + error_type: "ValueError".into(), + error_message: "bad input 'x'".into(), + created_at_millis: 1_700_000_000_000, + }] + ); + assert_eq!( + errors.fragments, + [FunctionErrorFragment { + job_id: "j-7".into(), + fragment_id: 4, + rows_skipped: 500, + rows_recorded: 100, + }] + ); + assert!(errors.truncated); + + // No filter sends no filter, and an empty listing reads as such. + let table = Table::new_with_handler("my_table", |request| { + let body = request.body().unwrap().as_bytes().unwrap(); + let value: serde_json::Value = serde_json::from_slice(body).unwrap(); + assert_eq!(value, serde_json::json!({})); + http::Response::builder() + .status(200) + .body(r#"{"records": []}"#) + .unwrap() + }); + let errors = table + .function_errors(FunctionErrorsRequest::new()) + .await + .unwrap(); + assert_eq!(errors, crate::function::FunctionErrors::default()); + } + /// The refresh handle is wrapped for read-freshness tracking, so it has to /// forward the detail APIs too -- this is the job an operator is holding /// when a backfill goes quiet. diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 0cf6a0e4f..fbec73dfb 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -53,6 +53,7 @@ use crate::database::Database; use crate::database::read_freshness::TableFreshness; use crate::embeddings::{EmbeddingDefinition, EmbeddingRegistry, MemoryRegistry}; use crate::error::{Error, Result}; +use crate::function::FunctionErrorsRequest; use crate::index::IndexStatistics; use crate::index::{Index, IndexBuilder}; use crate::index::{IndexConfig, IndexStatisticsImpl, IndexType}; @@ -821,6 +822,17 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { message: "computed columns are supported only on local tables".into(), }) } + /// The per-row errors Function refreshes recorded on this table; see + /// [`Table::function_errors`]. The default returns `NotSupported`. + async fn function_errors( + &self, + _request: &crate::function::FunctionErrorsRequest, + ) -> Result { + Err(Error::NotSupported { + message: "per-row Function errors are recorded only on LanceDB Cloud and Enterprise" + .into(), + }) + } /// Alter columns in the table. async fn alter_columns(&self, alterations: &[ColumnAlteration]) -> Result; /// Drop columns from the table. @@ -1850,6 +1862,36 @@ impl Table { self.inner.refresh_column_async(column.as_ref()).await } + /// The per-row errors Function refreshes recorded on this table: the + /// rows a refresh skipped under its skip policy, with the failing input + /// and the error, plus a summary for any fragment whose detail was + /// capped. Filter by job or column through the request; a listing that + /// hit its limit reports [`FunctionErrors::truncated`]. + /// + /// LanceDB Cloud and Enterprise only, and the caller needs read access + /// to the table, since a message carries the value that failed. + /// + /// ``` + /// # use lancedb::Table; + /// use lancedb::function::FunctionErrorsRequest; + /// + /// # async fn list_errors(table: &Table) -> Result<(), Box> { + /// let errors = table + /// .function_errors(FunctionErrorsRequest::new().column("embedding")) + /// .await?; + /// for record in &errors.records { + /// println!("{}: {}", record.error_type, record.error_message); + /// } + /// # Ok(()) + /// # } + /// ``` + pub async fn function_errors( + &self, + request: FunctionErrorsRequest, + ) -> Result { + self.inner.function_errors(&request).await + } + /// Change a column's name or nullability. pub async fn alter_columns( &self, @@ -4049,6 +4091,31 @@ mod tests { assert_eq!(table.name, "test") } + /// The per-row error store is a server feature; a local table says so + /// rather than answering with an empty listing. + #[tokio::test] + async fn test_function_errors_are_remote_only() { + let tmp_dir = tempdir().unwrap(); + let conn = connect(tmp_dir.path().to_str().unwrap()) + .execute() + .await + .unwrap(); + let batch = make_test_batches(); + let table = conn + .create_table("t", batch.clone()) + .execute() + .await + .unwrap(); + let err = table + .function_errors(FunctionErrorsRequest::new()) + .await + .unwrap_err(); + assert!( + matches!(&err, Error::NotSupported { message } if message.contains("Cloud and Enterprise")), + "{err:?}" + ); + } + #[tokio::test] async fn test_open_not_found() { let tmp_dir = tempdir().unwrap(); From a91b29efc165f31a14e3c030558794f2d345c363 Mon Sep 17 00:00:00 2001 From: Wyatt Alt Date: Thu, 17 Sep 2026 05:51:12 -0700 Subject: [PATCH 39/43] fix: scope the Function binding guard on schema evolution to the bound columns (#4204) A table with one registered Function binding refused every add_columns, alter_columns, drop_columns and field-metadata update, whatever column they named. The hazard is narrower: a binding stores the exact Arrow fields of its inputs, outputs and assignment column, so editing one of those strands it and the table stops accepting rows. Any other column was never at risk. The guard now compares the columns a request names, a rename's target included, against the set every binding depends on, and refuses only on an intersection. Local and remote tables apply the same rule. The blanket guard stays on update and merge insert, which cannot say what they touch. --- rust/lancedb/src/remote/table.rs | 80 ++++++- rust/lancedb/src/table/computed_columns.rs | 143 +++++++++++- rust/lancedb/src/table/schema_evolution.rs | 250 +++++++++++++++++++-- 3 files changed, 446 insertions(+), 27 deletions(-) diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 4280c7437..79bf2e354 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -3314,9 +3314,10 @@ impl BaseTable for RemoteTable { _read_columns: Option>, ) -> Result { self.check_mutable().await?; - crate::table::computed_columns::ensure_no_function_bindings_for_mutation( + crate::table::computed_columns::ensure_not_function_bound( self.schema().await?.as_ref(), "schema evolution", + crate::table::schema_evolution::new_column_names(&transforms), )?; match transforms { NewColumnTransform::SqlExpressions(expressions) => { @@ -3369,9 +3370,10 @@ impl BaseTable for RemoteTable { async fn add_computed_columns(&self, columns: &[(String, String)]) -> Result { self.check_mutable().await?; - crate::table::computed_columns::ensure_no_function_bindings_for_mutation( + crate::table::computed_columns::ensure_not_function_bound( self.schema().await?.as_ref(), "schema evolution", + columns.iter().map(|(name, _)| name), )?; // The server plans the declaration against its table schema, including // Blob v2 semantics inherited by a direct field projection. @@ -7982,8 +7984,9 @@ mod tests { assert_eq!(result.version, 8); } - #[tokio::test] - async fn test_add_function_column_allows_an_existing_binding() { + /// The fixture binding's table: `title` and `body` bound as inputs, its + /// two outputs declared, plus an unbound `spare`. + fn fixture_bound_schema() -> Schema { let binding = crate::function::FunctionBinding::from_json(include_str!( "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json" )) @@ -8010,13 +8013,78 @@ mod tests { ), ) })); - let schema = Schema::new_with_metadata( + fields.push(Field::new("spare", DataType::Int32, true)); + Schema::new_with_metadata( fields, HashMap::from([( crate::table::computed_columns::FUNCTION_BINDINGS_META_KEY.to_string(), binding_metadata, )]), - ); + ) + } + + /// Only a column the binding uses is refused, and it is refused before + /// any request goes out; the rest reach the server as usual. + #[tokio::test] + async fn test_add_columns_scopes_to_the_columns_a_binding_uses() { + let table = Table::new_with_handler("my_table", |request| match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body(describe_response(&fixture_bound_schema())) + .unwrap(), + "/v1/table/my_table/add_columns/" => http::Response::builder() + .status(200) + .body(r#"{"version":10}"#.to_string()) + .unwrap(), + path => panic!("Unexpected path: {path}"), + }); + table + .add_columns() + .computed("doubled", "spare * 2") + .execute() + .await + .unwrap(); + table + .add_columns() + .transform(NewColumnTransform::SqlExpressions(vec![( + "eager".into(), + "spare + 1".into(), + )])) + .execute() + .await + .unwrap(); + + let table = Table::new_with_handler("my_table", |request| match request.url().path() { + "/v1/table/my_table/describe/" => http::Response::builder() + .status(200) + .body(describe_response(&fixture_bound_schema())) + .unwrap(), + path => panic!("mutation request must not be sent: {path}"), + }); + for name in ["title", "search_text"] { + let err = table + .add_columns() + .computed(name, "1") + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "{err:?}"); + let err = table + .add_columns() + .transform(NewColumnTransform::SqlExpressions(vec![( + name.into(), + "1".into(), + )])) + .execute() + .await + .unwrap_err(); + assert!(matches!(err, Error::InvalidInput { .. }), "{err:?}"); + } + } + + #[tokio::test] + async fn test_add_function_column_allows_an_existing_binding() { + let schema = fixture_bound_schema(); let table = Table::new_with_handler("my_table", move |request| match request.url().path() { "/v1/table/my_table/describe/" => http::Response::builder() diff --git a/rust/lancedb/src/table/computed_columns.rs b/rust/lancedb/src/table/computed_columns.rs index f87d42426..23a870b42 100644 --- a/rust/lancedb/src/table/computed_columns.rs +++ b/rust/lancedb/src/table/computed_columns.rs @@ -391,6 +391,9 @@ pub(crate) fn ensure_supported_function_metadata(schema: &ArrowSchema) -> Result Ok(()) } +/// Refuse `operation` outright on a table with a Function binding. For +/// operations that cannot say which columns they touch; the others use +/// [`ensure_not_function_bound`]. pub(crate) fn ensure_no_function_bindings_for_mutation( schema: &ArrowSchema, operation: &str, @@ -406,6 +409,49 @@ pub(crate) fn ensure_no_function_bindings_for_mutation( Ok(()) } +/// Refuse `operation` only when a path in `touched` names a column a Function +/// binding depends on: an input's root, an output, or the assignment column. +/// A binding stores those columns' exact Arrow fields, so editing one strands it. +pub(crate) fn ensure_not_function_bound>( + schema: &ArrowSchema, + operation: &str, + touched: impl IntoIterator, +) -> Result<()> { + ensure_supported_function_metadata(schema)?; + let mut protected = BTreeSet::new(); + for binding in function_bindings(schema)? { + for input in binding.inputs() { + protected.insert(field_root(&input.field_path)?); + } + protected.extend( + binding + .outputs() + .iter() + .map(|output| output.output_name.clone()), + ); + protected.extend( + binding + .assignment() + .map(|assignment| assignment.output_name.clone()), + ); + } + if protected.is_empty() { + return Ok(()); + } + for path in touched { + let column = field_root(path.as_ref())?; + if protected.contains(&column) { + return Err(Error::InvalidInput { + message: format!( + "{operation} of '{column}' is not supported: a Function binding reads or \ + writes it" + ), + }); + } + } + Ok(()) +} + /// Read a field's computed-column declaration, if it carries one. /// /// A field flagged computed but carrying no kind, or a SQL one missing its @@ -1330,7 +1376,7 @@ pub(crate) fn ensure_not_written<'a>( .map(|declaration| declaration.name) .collect(); for name in written { - if declared.iter().any(|declared| declared == root(name)) { + if declared.iter().any(|declared| *declared == root(name)) { return Err(Error::InvalidInput { message: format!( "column '{}' is computed; its values come from refresh and cannot be \ @@ -1523,9 +1569,24 @@ pub(crate) fn ensure_not_retyped(schema: &ArrowSchema, paths: &[&str]) -> Result Ok(()) } -/// The top-level column a possibly nested input path reads. -pub(crate) fn root(path: &str) -> &str { - path.split('.').next().unwrap_or(path) +/// The top-level column a path addresses, by the grammar lance resolves it +/// with, so a quoted spelling names the same column as a bare one. +pub(crate) fn field_root(path: &str) -> Result { + parse_field_path(path) + .map_err(|e| Error::InvalidInput { + message: format!("invalid column path '{path}': {e}"), + })? + .into_iter() + .next() + .ok_or_else(|| Error::InvalidInput { + message: format!("column path '{path}' is empty"), + }) +} + +/// [`field_root`], falling back to the text before the first dot for a +/// spelling lance would not resolve. +pub(crate) fn root(path: &str) -> String { + field_root(path).unwrap_or_else(|_| path.split('.').next().unwrap_or(path).to_string()) } /// A declaration's expression bound to a schema, ready to evaluate. @@ -1749,7 +1810,7 @@ pub(crate) fn bind(schema: SchemaRef, column: &str, expression: &str) -> Result< let mut indices = Vec::with_capacity(inputs.len()); for input in &inputs { let index = runtime_schema - .index_of(root(input)) + .index_of(&root(input)) .map_err(|_| invalid(format!("unknown column '{input}'")))?; if !indices.contains(&index) { indices.push(index); @@ -1884,7 +1945,11 @@ pub(crate) fn plan(schema: SchemaRef, columns: &[(String, String)]) -> Result Result<()> { - ensure_no_function_bindings_for_mutation(schema.as_ref(), "schema evolution")?; + ensure_not_function_bound( + schema.as_ref(), + "schema evolution", + columns.iter().map(|(name, _)| name), + )?; plan(schema, columns).map(drop) } @@ -3197,6 +3262,72 @@ mod tests { .unwrap(); } + /// The scoped guard refuses exactly the columns a binding uses -- input + /// roots, outputs and the assignment column -- and nothing else. + #[test] + fn test_function_bound_columns_are_the_only_ones_refused() { + let mut raw_binding: Value = serde_json::from_str(include_str!( + "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json" + )) + .unwrap(); + raw_binding["outputs"][0]["nullable"] = Value::Bool(true); + raw_binding["outputs"][1]["nullable"] = Value::Bool(true); + raw_binding["assignment"] = serde_json::json!({ + "output_name": "__function_assignment_fb_01K3TEXT", + "output_field_id": -1, + }); + raw_binding["output_schema"]["fields"] + .as_array_mut() + .unwrap() + .push(serde_json::json!({ + "name": "__function_assignment_fb_01K3TEXT", + "nullable": true, + "type": {"type": "bool"}, + })); + let binding: FunctionBinding = serde_json::from_value(raw_binding).unwrap(); + let mut fields = valid_function_binding_schema(true, true, &binding) + .fields() + .to_vec(); + fields.push(Arc::new(ArrowField::new("spare", DataType::Int32, true))); + let schema = ArrowSchema::new_with_metadata( + fields, + HashMap::from([( + FUNCTION_BINDINGS_META_KEY.to_string(), + function_bindings_metadata(std::slice::from_ref(&binding)).unwrap(), + )]), + ); + + ensure_not_function_bound( + &schema, + "schema evolution", + ["spare", "spare.nested", "new", "`spare`", "`spare.nested`"], + ) + .unwrap(); + for path in [ + "title", + "body.nested", + "search_text", + "search_token_count", + "__function_assignment_fb_01K3TEXT", + "`title`", + "`body`.nested", + "`search_text`", + ] { + let err = ensure_not_function_bound(&schema, "schema evolution", [path]).unwrap_err(); + assert!( + matches!(&err, Error::InvalidInput { message } + if message.contains("a Function binding reads or writes it")), + "{path}: {err:?}" + ); + } + // A spelling lance cannot resolve is refused rather than compared as text. + let err = ensure_not_function_bound(&schema, "schema evolution", ["`title"]).unwrap_err(); + assert!( + matches!(&err, Error::InvalidInput { message } if message.contains("invalid column path")), + "{err:?}" + ); + } + #[test] fn test_binding_preserves_all_nullable_outputs_with_an_assignment_column() { let mut raw_binding: Value = serde_json::from_str(include_str!( diff --git a/rust/lancedb/src/table/schema_evolution.rs b/rust/lancedb/src/table/schema_evolution.rs index 4f8dc811a..796db09bd 100644 --- a/rust/lancedb/src/table/schema_evolution.rs +++ b/rust/lancedb/src/table/schema_evolution.rs @@ -103,9 +103,10 @@ pub(crate) async fn execute_add_columns( transforms: NewColumnTransform, read_columns: Option>, ) -> Result { - computed_columns::ensure_no_function_bindings_for_mutation( + computed_columns::ensure_not_function_bound( table.schema().await?.as_ref(), "schema evolution", + new_column_names(&transforms), )?; // Declarations are admitted only through [`execute_declare`]. match &transforms { @@ -131,9 +132,10 @@ pub(crate) async fn execute_declare( // An LSM write spec keeps visible rows in tiers refresh cannot reach; // checked against latest committed state, not this handle's snapshot. table.checkout_latest().await?; - computed_columns::ensure_no_function_bindings_for_mutation( + computed_columns::ensure_not_function_bound( table.schema().await?.as_ref(), "schema evolution", + columns.iter().map(|(name, _)| name), )?; // Unset drops the MemWAL index, so the spec alone stops describing a table // whose SSTables still hold rows. The shard directories outlive it and are @@ -156,6 +158,26 @@ pub(crate) async fn execute_declare( commit_add_columns(table, transform, None).await } +/// The top-level columns `transforms` adds. +pub(crate) fn new_column_names(transforms: &NewColumnTransform) -> Vec { + let names = |schema: &ArrowSchema| { + schema + .fields() + .iter() + .map(|field| field.name().clone()) + .collect::>() + }; + match transforms { + NewColumnTransform::SqlExpressions(expressions) => { + expressions.iter().map(|(name, _)| name.clone()).collect() + } + NewColumnTransform::AllNulls(schema) => names(schema), + NewColumnTransform::BatchUDF(udf) => names(&udf.output_schema), + NewColumnTransform::Stream(stream) => names(&stream.schema()), + NewColumnTransform::Reader(reader) => names(&reader.schema()), + } +} + pub(crate) async fn commit_add_columns( table: &NativeTable, transforms: NewColumnTransform, @@ -178,13 +200,18 @@ pub(crate) async fn execute_alter_columns( ) -> Result { table.dataset.ensure_mutable()?; let mut dataset = (*table.dataset.get().await?).clone(); - // Nullability is not part of what an expression resolves against, so only - // a rename or a retype can invalidate a binding. let schema = std::sync::Arc::new(ArrowSchema::from(dataset.schema())); - computed_columns::ensure_no_function_bindings_for_mutation( + // A Function binding stores its columns' exact fields, nullability + // included, so every alteration of one counts, and a rename's target too. + computed_columns::ensure_not_function_bound( schema.as_ref(), "schema evolution", + alterations.iter().flat_map(|alteration| { + std::iter::once(alteration.path.as_str()).chain(alteration.rename.as_deref()) + }), )?; + // Nullability is not part of what an expression resolves against, so only + // a rename or a retype can invalidate a binding. let rebinding = alterations .iter() .filter(|alteration| alteration.rename.is_some() || alteration.data_type.is_some()) @@ -212,14 +239,9 @@ pub(crate) async fn execute_drop_columns( ) -> Result { table.dataset.ensure_mutable()?; let mut dataset = (*table.dataset.get().await?).clone(); - computed_columns::ensure_no_function_bindings_for_mutation( - &ArrowSchema::from(dataset.schema()), - "schema evolution", - )?; - computed_columns::ensure_not_an_input( - &std::sync::Arc::new(ArrowSchema::from(dataset.schema())), - columns, - )?; + let schema = std::sync::Arc::new(ArrowSchema::from(dataset.schema())); + computed_columns::ensure_not_function_bound(schema.as_ref(), "schema evolution", columns)?; + computed_columns::ensure_not_an_input(&schema, columns)?; dataset.drop_columns(columns).await?; let version = dataset.version().version; table.dataset.update(dataset); @@ -241,7 +263,11 @@ pub(crate) async fn execute_update_field_metadata( // binding out from under a refresh. A replace on a declared column would // silently erase it. let schema = ArrowSchema::from(dataset.schema()); - computed_columns::ensure_no_function_bindings_for_mutation(&schema, "schema evolution")?; + computed_columns::ensure_not_function_bound( + &schema, + "field metadata update", + updates.iter().map(|update| update.path.as_str()), + )?; let declared: Vec = computed_columns::computed_columns(&schema) .into_iter() .map(|declaration| declaration.name) @@ -263,7 +289,7 @@ pub(crate) async fn execute_update_field_metadata( if update.replace && declared .iter() - .any(|name| name == computed_columns::root(&update.path)) + .any(|name| *name == computed_columns::root(&update.path)) { return Err(Error::InvalidInput { message: format!( @@ -300,8 +326,202 @@ mod tests { use super::FieldMetadataUpdate; use crate::connect; + use crate::function::FunctionBinding; use crate::query::{ExecutableQuery, QueryBase, Select}; use crate::table::NewColumnTransform; + use crate::table::computed_columns::{ + FUNCTION_BINDINGS_META_KEY, ensure_supported_function_metadata, function_bindings, + function_bindings_metadata, function_computed_column_metadata, + }; + use crate::{Error, Table}; + use std::collections::HashMap; + + /// A table carrying the fixture binding: `title` and `body` are its + /// inputs, `search_text` and `search_token_count` its outputs, `spare` + /// nobody's. Stamped the way the server does it, since no local path + /// declares a binding. + async fn bound_table() -> Table { + let conn = connect("memory://").execute().await.unwrap(); + let batch = record_batch!( + ("title", Utf8, ["a"]), + ("body", Utf8, ["b"]), + ("search_text", Utf8, ["a b"]), + ("search_token_count", Int64, [2]), + ("spare", Int32, [1]) + ) + .unwrap(); + let table = conn.create_table("bound", batch).execute().await.unwrap(); + let binding = FunctionBinding::from_json(include_str!( + "../../tests/fixtures/first_class_functions/v1/remote_function_binding.json" + )) + .unwrap(); + let native = table.as_native().unwrap(); + let mut dataset = native.dataset.get().await.unwrap().as_ref().clone(); + dataset + .update_schema_metadata(vec![( + FUNCTION_BINDINGS_META_KEY.to_string(), + Some(function_bindings_metadata(std::slice::from_ref(&binding)).unwrap()), + )]) + .await + .unwrap(); + let inputs = ["title".to_string(), "body".to_string()]; + let outputs = binding + .outputs() + .iter() + .map(|output| { + ( + dataset.schema().field(&output.output_name).unwrap().id as u32, + function_computed_column_metadata( + binding.binding_id(), + output.output_ordinal, + &inputs, + ), + ) + }) + .collect::>(); + dataset.replace_field_metadata(outputs).await.unwrap(); + native.dataset.update(dataset); + ensure_supported_function_metadata(&table.schema().await.unwrap()).unwrap(); + table + } + + fn metadata_update(path: &str) -> FieldMetadataUpdate { + FieldMetadataUpdate { + path: path.into(), + metadata: HashMap::from([("unit".to_string(), Some("label".to_string()))]), + replace: false, + } + } + + /// Columns no binding uses evolve as on any table, and the binding is + /// still valid afterwards, which is what every later write checks. + #[tokio::test] + async fn test_schema_evolution_leaves_unbound_columns_free_on_a_bound_table() { + let table = bound_table().await; + table + .add_columns() + .transform(NewColumnTransform::SqlExpressions(vec![( + "eager".into(), + "1".into(), + )])) + .execute() + .await + .unwrap(); + table + .add_columns() + .computed("derived", "spare * 2") + .execute() + .await + .unwrap(); + table + .update_field_metadata(&[metadata_update("eager")]) + .await + .unwrap(); + table + .alter_columns(&[ColumnAlteration::new("eager".into()).rename("moved".into())]) + .await + .unwrap(); + table.drop_columns(&["moved"]).await.unwrap(); + + let schema = table.schema().await.unwrap(); + ensure_supported_function_metadata(&schema).unwrap(); + assert_eq!(function_bindings(&schema).unwrap().len(), 1); + assert!(schema.field_with_name("derived").is_ok()); + assert!(schema.field_with_name("moved").is_err()); + } + + fn bound(err: Error) { + assert!( + matches!(&err, Error::InvalidInput { message } + if message.contains("a Function binding reads or writes it")), + "{err:?}" + ); + } + + /// Every schema-evolution door refuses a column a binding reads or + /// writes, including a rename onto one. + #[tokio::test] + async fn test_schema_evolution_refuses_the_columns_a_function_binding_uses() { + let table = bound_table().await; + let version = table.version().await.unwrap(); + for column in ["title", "body", "search_text", "search_token_count"] { + bound(table.drop_columns(&[column]).await.unwrap_err()); + bound( + table + .alter_columns(&[ColumnAlteration::new(column.into()).rename("moved".into())]) + .await + .unwrap_err(), + ); + bound( + table + .alter_columns(&[ColumnAlteration::new(column.into()).set_nullable(false)]) + .await + .unwrap_err(), + ); + bound( + table + .update_field_metadata(&[metadata_update(column)]) + .await + .unwrap_err(), + ); + bound( + table + .add_columns() + .transform(NewColumnTransform::SqlExpressions(vec![( + column.into(), + "1".into(), + )])) + .execute() + .await + .unwrap_err(), + ); + bound( + table + .add_columns() + .computed(column, "1") + .execute() + .await + .unwrap_err(), + ); + } + bound( + table + .alter_columns(&[ColumnAlteration::new("spare".into()).rename("title".into())]) + .await + .unwrap_err(), + ); + assert_eq!(table.version().await.unwrap(), version); + } + + /// Lance resolves a quoted spelling to the same field as the bare one, + /// so the guard compares identities, not text. + #[tokio::test] + async fn quoted_function_output_path_is_still_refused() { + let table = bound_table().await; + let version = table.version().await.unwrap(); + for path in ["`title`", "`search_text`", "`title`.nested"] { + bound(table.drop_columns(&[path]).await.unwrap_err()); + bound( + table + .alter_columns(&[ColumnAlteration::new(path.into()).set_nullable(false)]) + .await + .unwrap_err(), + ); + bound( + table + .update_field_metadata(&[metadata_update(path)]) + .await + .unwrap_err(), + ); + } + bound( + table + .alter_columns(&[ColumnAlteration::new("spare".into()).rename("`title`".into())]) + .await + .unwrap_err(), + ); + assert_eq!(table.version().await.unwrap(), version); + } // Add Columns Tests From 89ad06c78248703a6d48cca4f62e3f1c0e3d71ba Mon Sep 17 00:00:00 2001 From: Lance Release Date: Thu, 17 Sep 2026 12:51:59 +0000 Subject: [PATCH 40/43] =?UTF-8?q?Bump=20version:=200.40.0-beta.1=20?= =?UTF-8?q?=E2=86=92=200.40.0-beta.2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index c28f65ec1..1510c1e23 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.40.0-beta.1" +current_version = "0.40.0-beta.2" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 1d1861a59..8b95466fe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5684,7 +5684,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.40.0-beta.1" +version = "0.40.0-beta.2" dependencies = [ "ahash", "anyhow", @@ -5781,7 +5781,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.40.0-beta.1" +version = "0.40.0-beta.2" dependencies = [ "arrow-array", "arrow-buffer", @@ -5806,7 +5806,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.40.0-beta.1" +version = "0.40.0-beta.2" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index 4a798c770..a835fb222 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.40.0-beta.1 + 0.40.0-beta.2 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 0981f4bb2..9f27ffc27 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.1 + 0.40.0-beta.2 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 53de36b8e..586d60631 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.1 + 0.40.0-beta.2 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 6962c5802..392d50a27 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.40.0-beta.1" +version = "0.40.0-beta.2" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index b98d8ec41..e1a40db3e 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.40.0-beta.1", + "version": "0.40.0-beta.2", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index be28e90a8..8d9228ba7 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.40.0-beta.1", + "version": "0.40.0-beta.2", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index cc8ff6163..410030a50 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.40.0-beta.1", + "version": "0.40.0-beta.2", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 1a5992ec9..0d20d4155 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.40.0-beta.1", + "version": "0.40.0-beta.2", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index e838efc17..393ed7a23 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.40.0-beta.1", + "version": "0.40.0-beta.2", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index e0543a3a6..8634289b5 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.40.0-beta.1", + "version": "0.40.0-beta.2", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 93d520269..7e25fb93c 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.40.0-beta.1", + "version": "0.40.0-beta.2", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index 3e8725c35..6f164c8e5 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.40.0-beta.1", + "version": "0.40.0-beta.2", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 705eb4a0a..064902fa0 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.40.0-beta.1" +version = "0.40.0-beta.2" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index bde33283d..398e64fe5 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.40.0-beta.1" +version = "0.40.0-beta.2" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true From a0c5612aee26173b0f993d185c386c480348aa30 Mon Sep 17 00:00:00 2001 From: Vivek Date: Thu, 17 Sep 2026 10:59:41 -0700 Subject: [PATCH 41/43] feat(rust): make MetadataEraserExec public (#4213) Export `MetadataEraserExec` and its constructor from `lancedb::table::datafusion`. Engines that serialise a physical plan containing a LanceDB scan have to rebuild the operator outside this crate, which a private type makes impossible. --- rust/lancedb/src/table/datafusion.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/rust/lancedb/src/table/datafusion.rs b/rust/lancedb/src/table/datafusion.rs index d60bce7dc..85366910b 100644 --- a/rust/lancedb/src/table/datafusion.rs +++ b/rust/lancedb/src/table/datafusion.rs @@ -33,11 +33,23 @@ use crate::{ use arrow_schema::{DataType, Field}; use lance_index::scalar::FullTextSearchQuery; -/// Datafusion attempts to maintain batch metadata +/// An execution plan that erases Arrow schema-level metadata from its input's batches. /// -/// This is needless and it triggers bugs in DF. This operator erases metadata from the batches. +/// DataFusion attempts to maintain batch metadata. This is needless and it triggers bugs in +/// DF, so [`BaseTableAdapter::scan`] wraps every scan it produces in one of these. +/// +/// ``` +/// use std::sync::Arc; +/// +/// use datafusion_physical_plan::ExecutionPlan; +/// use lancedb::table::datafusion::MetadataEraserExec; +/// +/// # fn erase_metadata(scan: Arc) -> Arc { +/// Arc::new(MetadataEraserExec::new(scan)) +/// # } +/// ``` #[derive(Debug)] -struct MetadataEraserExec { +pub struct MetadataEraserExec { input: Arc, schema: Arc, properties: Arc, @@ -62,7 +74,8 @@ impl MetadataEraserExec { ) } - fn new(input: Arc) -> Self { + /// Wrap `input` in an operator that strips schema-level metadata from its batches. + pub fn new(input: Arc) -> Self { let schema = Arc::new( input .schema() From 97de3ccd98f813cbb62e3b6191ae4317856666a3 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Thu, 17 Sep 2026 13:05:00 -0700 Subject: [PATCH 42/43] fix: accept synchronous remote materialized-view drops (#4212) Remote materialized-view deletion currently rejects HTTP 200 even when the server has completed cleanup synchronously. Treat 200 as an already-finished Job with no ID, matching table deletion; retain the cleanup Job for 202, require its ID, and invalidate the table cache in both cases. Add regressions for synchronous completion and malformed or unexpected responses, alongside the existing asynchronous Job coverage. No local builds or tests were run. --- rust/lancedb/src/materialized_view.rs | 3 +- rust/lancedb/src/remote/db.rs | 79 +++++++++++++++++++++------ 2 files changed, 65 insertions(+), 17 deletions(-) diff --git a/rust/lancedb/src/materialized_view.rs b/rust/lancedb/src/materialized_view.rs index 94fcb5dda..39b243d49 100644 --- a/rust/lancedb/src/materialized_view.rs +++ b/rust/lancedb/src/materialized_view.rs @@ -1545,7 +1545,8 @@ impl Connection { /// /// This validates that the named resource is a materialized view rather /// than an ordinary table. Call [`Job::wait`] before assuming physical - /// cleanup has finished. + /// cleanup has finished. When the backend performs cleanup inline, the + /// returned job is already finished and has no job ID. /// /// ```no_run /// # use lancedb::Connection; diff --git a/rust/lancedb/src/remote/db.rs b/rust/lancedb/src/remote/db.rs index bc1dbfea1..7061ac979 100644 --- a/rust/lancedb/src/remote/db.rs +++ b/rust/lancedb/src/remote/db.rs @@ -856,25 +856,28 @@ impl Database for RemoteDatabase { let response = self.client.check_response(&request_id, response).await?; let status = response.status(); let body = response.text().await.err_to_http(request_id.clone())?; - if status != StatusCode::ACCEPTED { - return Err(Error::Http { - source: "materialized-view drop must return 202 Accepted".into(), - request_id, - status_code: Some(status), - }); - } - let job_id = extract_job_id(&body).ok_or_else(|| Error::Http { - source: "materialized-view drop response did not contain a valid job_id".into(), - request_id, - status_code: Some(status), - })?; + let job = match status { + StatusCode::OK => Job::new_done(), + StatusCode::ACCEPTED => { + let job_id = extract_job_id(&body).ok_or_else(|| Error::Http { + source: "materialized-view drop response did not contain a valid job_id".into(), + request_id, + status_code: Some(status), + })?; + Job::new(Box::new(RemoteJob::new(self.client.clone(), job_id))) + } + _ => { + return Err(Error::Http { + source: "materialized-view drop must return 200 OK or 202 Accepted".into(), + request_id, + status_code: Some(status), + }); + } + }; self.table_cache .remove(&build_cache_key(name, namespace_path)) .await; - Ok(Job::new(Box::new(RemoteJob::new( - self.client.clone(), - job_id, - )))) + Ok(job) } async fn list_materialized_views(&self, namespace_path: &[String]) -> Result> { @@ -1780,6 +1783,50 @@ mod tests { assert_eq!(job.id(), Some("j1-mv-drop")); } + #[tokio::test] + async fn test_drop_materialized_view_completed_inline() { + let db = super::RemoteDatabase::new_mock(|request| { + assert_eq!(request.method(), "POST"); + assert_eq!(request.url().path(), "/v1/materialized_view/adults/drop"); + http::Response::builder().status(200).body("{}").unwrap() + }); + let job = db + .drop_materialized_view_async("adults", &[]) + .await + .unwrap(); + assert_eq!(job.id(), None); + assert_eq!(job.status().await.unwrap(), "finished"); + job.wait().await.unwrap(); + } + + #[tokio::test] + async fn test_drop_materialized_view_rejects_incomplete_acceptance() { + for body in ["{}", r#"{"job_id":""}"#, r#"{"job_id":null}"#] { + let db = super::RemoteDatabase::new_mock(move |_| { + http::Response::builder().status(202).body(body).unwrap() + }); + let error = db + .drop_materialized_view_async("adults", &[]) + .await + .err() + .unwrap(); + assert!(error.to_string().contains("valid job_id")); + } + } + + #[tokio::test] + async fn test_drop_materialized_view_rejects_unexpected_success_status() { + let db = super::RemoteDatabase::new_mock(|_| { + http::Response::builder().status(204).body("").unwrap() + }); + let error = db + .drop_materialized_view_async("adults", &[]) + .await + .err() + .unwrap(); + assert!(error.to_string().contains("200 OK or 202 Accepted")); + } + #[tokio::test] async fn test_list_materialized_views_follows_empty_pages() { let page = Arc::new(AtomicUsize::new(0)); From 63cd121225f13ced87a7bf48ad50c66c484e385a Mon Sep 17 00:00:00 2001 From: Lance Release Date: Thu, 17 Sep 2026 20:05:57 +0000 Subject: [PATCH 43/43] =?UTF-8?q?Bump=20version:=200.40.0-beta.2=20?= =?UTF-8?q?=E2=86=92=200.40.0-beta.3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.toml | 2 +- Cargo.lock | 6 +++--- docs/src/java/java.md | 2 +- java/lancedb-core/pom.xml | 2 +- java/pom.xml | 2 +- nodejs/Cargo.toml | 2 +- nodejs/npm/darwin-arm64/package.json | 2 +- nodejs/npm/linux-arm64-gnu/package.json | 2 +- nodejs/npm/linux-arm64-musl/package.json | 2 +- nodejs/npm/linux-x64-gnu/package.json | 2 +- nodejs/npm/linux-x64-musl/package.json | 2 +- nodejs/npm/win32-arm64-msvc/package.json | 2 +- nodejs/npm/win32-x64-msvc/package.json | 2 +- nodejs/package.json | 2 +- python/Cargo.toml | 2 +- rust/lancedb/Cargo.toml | 2 +- 16 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.bumpversion.toml b/.bumpversion.toml index 1510c1e23..314e75779 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -1,5 +1,5 @@ [tool.bumpversion] -current_version = "0.40.0-beta.2" +current_version = "0.40.0-beta.3" parse = """(?x) (?P0|[1-9]\\d*)\\. (?P0|[1-9]\\d*)\\. diff --git a/Cargo.lock b/Cargo.lock index 8b95466fe..3a1444cf8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5684,7 +5684,7 @@ dependencies = [ [[package]] name = "lancedb" -version = "0.40.0-beta.2" +version = "0.40.0-beta.3" dependencies = [ "ahash", "anyhow", @@ -5781,7 +5781,7 @@ dependencies = [ [[package]] name = "lancedb-nodejs" -version = "0.40.0-beta.2" +version = "0.40.0-beta.3" dependencies = [ "arrow-array", "arrow-buffer", @@ -5806,7 +5806,7 @@ dependencies = [ [[package]] name = "lancedb-python" -version = "0.40.0-beta.2" +version = "0.40.0-beta.3" dependencies = [ "arc-swap", "arrow", diff --git a/docs/src/java/java.md b/docs/src/java/java.md index a835fb222..ee478561d 100644 --- a/docs/src/java/java.md +++ b/docs/src/java/java.md @@ -14,7 +14,7 @@ Add the following dependency to your `pom.xml`: com.lancedb lancedb-core - 0.40.0-beta.2 + 0.40.0-beta.3 ``` diff --git a/java/lancedb-core/pom.xml b/java/lancedb-core/pom.xml index 9f27ffc27..dda6c2e49 100644 --- a/java/lancedb-core/pom.xml +++ b/java/lancedb-core/pom.xml @@ -8,7 +8,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.2 + 0.40.0-beta.3 ../pom.xml diff --git a/java/pom.xml b/java/pom.xml index 586d60631..a2b0e273c 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -6,7 +6,7 @@ com.lancedb lancedb-parent - 0.40.0-beta.2 + 0.40.0-beta.3 pom ${project.artifactId} LanceDB Java SDK Parent POM diff --git a/nodejs/Cargo.toml b/nodejs/Cargo.toml index 392d50a27..bbc687a0b 100644 --- a/nodejs/Cargo.toml +++ b/nodejs/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lancedb-nodejs" edition.workspace = true -version = "0.40.0-beta.2" +version = "0.40.0-beta.3" publish = false license.workspace = true description.workspace = true diff --git a/nodejs/npm/darwin-arm64/package.json b/nodejs/npm/darwin-arm64/package.json index e1a40db3e..eaf211df2 100644 --- a/nodejs/npm/darwin-arm64/package.json +++ b/nodejs/npm/darwin-arm64/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-darwin-arm64", - "version": "0.40.0-beta.2", + "version": "0.40.0-beta.3", "os": ["darwin"], "cpu": ["arm64"], "main": "lancedb.darwin-arm64.node", diff --git a/nodejs/npm/linux-arm64-gnu/package.json b/nodejs/npm/linux-arm64-gnu/package.json index 8d9228ba7..d51ab1418 100644 --- a/nodejs/npm/linux-arm64-gnu/package.json +++ b/nodejs/npm/linux-arm64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-gnu", - "version": "0.40.0-beta.2", + "version": "0.40.0-beta.3", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-gnu.node", diff --git a/nodejs/npm/linux-arm64-musl/package.json b/nodejs/npm/linux-arm64-musl/package.json index 410030a50..1cbab77b7 100644 --- a/nodejs/npm/linux-arm64-musl/package.json +++ b/nodejs/npm/linux-arm64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-arm64-musl", - "version": "0.40.0-beta.2", + "version": "0.40.0-beta.3", "os": ["linux"], "cpu": ["arm64"], "main": "lancedb.linux-arm64-musl.node", diff --git a/nodejs/npm/linux-x64-gnu/package.json b/nodejs/npm/linux-x64-gnu/package.json index 0d20d4155..572e11db7 100644 --- a/nodejs/npm/linux-x64-gnu/package.json +++ b/nodejs/npm/linux-x64-gnu/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-gnu", - "version": "0.40.0-beta.2", + "version": "0.40.0-beta.3", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-gnu.node", diff --git a/nodejs/npm/linux-x64-musl/package.json b/nodejs/npm/linux-x64-musl/package.json index 393ed7a23..314c55e28 100644 --- a/nodejs/npm/linux-x64-musl/package.json +++ b/nodejs/npm/linux-x64-musl/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-linux-x64-musl", - "version": "0.40.0-beta.2", + "version": "0.40.0-beta.3", "os": ["linux"], "cpu": ["x64"], "main": "lancedb.linux-x64-musl.node", diff --git a/nodejs/npm/win32-arm64-msvc/package.json b/nodejs/npm/win32-arm64-msvc/package.json index 8634289b5..79794ae7f 100644 --- a/nodejs/npm/win32-arm64-msvc/package.json +++ b/nodejs/npm/win32-arm64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-arm64-msvc", - "version": "0.40.0-beta.2", + "version": "0.40.0-beta.3", "os": [ "win32" ], diff --git a/nodejs/npm/win32-x64-msvc/package.json b/nodejs/npm/win32-x64-msvc/package.json index 7e25fb93c..6c0bcd929 100644 --- a/nodejs/npm/win32-x64-msvc/package.json +++ b/nodejs/npm/win32-x64-msvc/package.json @@ -1,6 +1,6 @@ { "name": "@lancedb/lancedb-win32-x64-msvc", - "version": "0.40.0-beta.2", + "version": "0.40.0-beta.3", "os": ["win32"], "cpu": ["x64"], "main": "lancedb.win32-x64-msvc.node", diff --git a/nodejs/package.json b/nodejs/package.json index 6f164c8e5..9ccc0885e 100644 --- a/nodejs/package.json +++ b/nodejs/package.json @@ -11,7 +11,7 @@ "ann" ], "private": false, - "version": "0.40.0-beta.2", + "version": "0.40.0-beta.3", "main": "dist/index.js", "exports": { ".": "./dist/index.js", diff --git a/python/Cargo.toml b/python/Cargo.toml index 064902fa0..9714aff24 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb-python" -version = "0.40.0-beta.2" +version = "0.40.0-beta.3" publish = false edition.workspace = true description = "Python bindings for LanceDB" diff --git a/rust/lancedb/Cargo.toml b/rust/lancedb/Cargo.toml index 398e64fe5..a183b9e0d 100644 --- a/rust/lancedb/Cargo.toml +++ b/rust/lancedb/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lancedb" -version = "0.40.0-beta.2" +version = "0.40.0-beta.3" edition.workspace = true description = "LanceDB: A serverless, low-latency vector database for AI applications" license.workspace = true