From 0859312b83cf3b2c9c3ae1da4651575c4ff4c1a9 Mon Sep 17 00:00:00 2001 From: Jack Ye Date: Wed, 4 Feb 2026 10:31:39 -0800 Subject: [PATCH] feat: add initial and latest storage options apis (#2966) Expose `initial_storage_options()` and `latest_storage_options()` in lance Dataset, in lancedb rust, python and typescript SDKs. --------- Co-authored-by: Claude Opus 4.5 --- nodejs/lancedb/table.ts | 41 ++++++++++++++++++++ nodejs/src/table.rs | 13 +++++++ python/python/lancedb/_lancedb.pyi | 2 + python/python/lancedb/table.py | 62 ++++++++++++++++++++++++++++++ python/src/table.rs | 14 +++++++ rust/lancedb/src/remote/table.rs | 8 ++++ rust/lancedb/src/table.rs | 41 ++++++++++++++++++++ 7 files changed, 181 insertions(+) diff --git a/nodejs/lancedb/table.ts b/nodejs/lancedb/table.ts index c5a14bdc4..495baddcb 100644 --- a/nodejs/lancedb/table.ts +++ b/nodejs/lancedb/table.ts @@ -542,6 +542,35 @@ export abstract class Table { * */ abstract stats(): Promise; + + /** + * Get the initial storage options that were passed in when opening this table. + * + * For dynamically refreshed options (e.g., credential vending), use + * {@link Table.latestStorageOptions}. + * + * Warning: This is an internal API and the return value is subject to change. + * + * @returns The storage options, or undefined if no storage options were configured. + */ + abstract initialStorageOptions(): Promise< + Record | null | undefined + >; + + /** + * Get the latest storage options, refreshing from provider if configured. + * + * This method is useful for credential vending scenarios where storage options + * may be refreshed dynamically. If no dynamic provider is configured, this + * returns the initial static options. + * + * Warning: This is an internal API and the return value is subject to change. + * + * @returns The storage options, or undefined if no storage options were configured. + */ + abstract latestStorageOptions(): Promise< + Record | null | undefined + >; } export class LocalTable extends Table { @@ -878,6 +907,18 @@ export class LocalTable extends Table { return await this.inner.stats(); } + async initialStorageOptions(): Promise< + Record | null | undefined + > { + return await this.inner.initialStorageOptions(); + } + + async latestStorageOptions(): Promise< + Record | null | undefined + > { + return await this.inner.latestStorageOptions(); + } + mergeInsert(on: string | string[]): MergeInsertBuilder { on = Array.isArray(on) ? on : [on]; return new MergeInsertBuilder(this.inner.mergeInsert(on), this.schema()); diff --git a/nodejs/src/table.rs b/nodejs/src/table.rs index 253182e70..272cc878c 100644 --- a/nodejs/src/table.rs +++ b/nodejs/src/table.rs @@ -166,6 +166,19 @@ impl Table { Ok(stats.into()) } + #[napi(catch_unwind)] + pub async fn initial_storage_options(&self) -> napi::Result>> { + Ok(self.inner_ref()?.initial_storage_options().await) + } + + #[napi(catch_unwind)] + pub async fn latest_storage_options(&self) -> napi::Result>> { + self.inner_ref()? + .latest_storage_options() + .await + .default_error() + } + #[napi(catch_unwind)] pub async fn update( &self, diff --git a/python/python/lancedb/_lancedb.pyi b/python/python/lancedb/_lancedb.pyi index f85b6673a..60a4aae60 100644 --- a/python/python/lancedb/_lancedb.pyi +++ b/python/python/lancedb/_lancedb.pyi @@ -180,6 +180,8 @@ class Table: delete_unverified: Optional[bool] = None, ) -> OptimizeStats: ... async def uri(self) -> str: ... + async def initial_storage_options(self) -> Optional[Dict[str, str]]: ... + async def latest_storage_options(self) -> Optional[Dict[str, str]]: ... @property def tags(self) -> Tags: ... def query(self) -> Query: ... diff --git a/python/python/lancedb/table.py b/python/python/lancedb/table.py index b18d6e032..969332355 100644 --- a/python/python/lancedb/table.py +++ b/python/python/lancedb/table.py @@ -2222,6 +2222,37 @@ class LanceTable(Table): def uri(self) -> str: return LOOP.run(self._table.uri()) + def initial_storage_options(self) -> Optional[Dict[str, str]]: + """Get the initial storage options that were passed in when opening this table. + + For dynamically refreshed options (e.g., credential vending), use + :meth:`latest_storage_options`. + + Warning: This is an internal API and the return value is subject to change. + + Returns + ------- + Optional[Dict[str, str]] + The storage options, or None if no storage options were configured. + """ + return LOOP.run(self._table.initial_storage_options()) + + def latest_storage_options(self) -> Optional[Dict[str, str]]: + """Get the latest storage options, refreshing from provider if configured. + + This method is useful for credential vending scenarios where storage options + may be refreshed dynamically. If no dynamic provider is configured, this + returns the initial static options. + + Warning: This is an internal API and the return value is subject to change. + + Returns + ------- + Optional[Dict[str, str]] + The storage options, or None if no storage options were configured. + """ + return LOOP.run(self._table.latest_storage_options()) + def create_scalar_index( self, column: str, @@ -3624,6 +3655,37 @@ class AsyncTable: """ return await self._inner.uri() + async def initial_storage_options(self) -> Optional[Dict[str, str]]: + """Get the initial storage options that were passed in when opening this table. + + For dynamically refreshed options (e.g., credential vending), use + :meth:`latest_storage_options`. + + Warning: This is an internal API and the return value is subject to change. + + Returns + ------- + Optional[Dict[str, str]] + The storage options, or None if no storage options were configured. + """ + return await self._inner.initial_storage_options() + + async def latest_storage_options(self) -> Optional[Dict[str, str]]: + """Get the latest storage options, refreshing from provider if configured. + + This method is useful for credential vending scenarios where storage options + may be refreshed dynamically. If no dynamic provider is configured, this + returns the initial static options. + + Warning: This is an internal API and the return value is subject to change. + + Returns + ------- + Optional[Dict[str, str]] + The storage options, or None if no storage options were configured. + """ + return await self._inner.latest_storage_options() + async def add( self, data: DATA, diff --git a/python/src/table.rs b/python/src/table.rs index af91a50d6..41e4df35f 100644 --- a/python/src/table.rs +++ b/python/src/table.rs @@ -502,6 +502,20 @@ impl Table { future_into_py(self_.py(), async move { inner.uri().await.infer_error() }) } + pub fn initial_storage_options(self_: PyRef<'_, Self>) -> PyResult> { + let inner = self_.inner_ref()?.clone(); + future_into_py(self_.py(), async move { + Ok(inner.initial_storage_options().await) + }) + } + + pub fn latest_storage_options(self_: PyRef<'_, Self>) -> PyResult> { + let inner = self_.inner_ref()?.clone(); + future_into_py(self_.py(), async move { + inner.latest_storage_options().await.infer_error() + }) + } + pub fn __repr__(&self) -> String { match &self.inner { None => format!("ClosedTable({})", self.name), diff --git a/rust/lancedb/src/remote/table.rs b/rust/lancedb/src/remote/table.rs index 11c43fde0..04d4ce489 100644 --- a/rust/lancedb/src/remote/table.rs +++ b/rust/lancedb/src/remote/table.rs @@ -1497,6 +1497,14 @@ impl BaseTable for RemoteTable { None } + async fn initial_storage_options(&self) -> Option> { + None + } + + async fn latest_storage_options(&self) -> Result>> { + Ok(None) + } + async fn stats(&self) -> Result { let request = self .client diff --git a/rust/lancedb/src/table.rs b/rust/lancedb/src/table.rs index 461736e3d..224d9b24b 100644 --- a/rust/lancedb/src/table.rs +++ b/rust/lancedb/src/table.rs @@ -527,7 +527,17 @@ pub trait BaseTable: std::fmt::Display + std::fmt::Debug + Send + Sync { /// Get the table URI (storage location) async fn uri(&self) -> Result; /// Get the storage options used when opening this table, if any. + #[deprecated(since = "0.25.0", note = "Use initial_storage_options() instead")] async fn storage_options(&self) -> Option>; + /// Get the initial storage options that were passed in when opening this table. + /// + /// For dynamically refreshed options (e.g., credential vending), use [`Self::latest_storage_options`]. + async fn initial_storage_options(&self) -> Option>; + /// Get the latest storage options, refreshing from provider if configured. + /// + /// Returns `Ok(Some(options))` if storage options are available (static or refreshed), + /// `Ok(None)` if no storage options were configured, or `Err(...)` if refresh failed. + async fn latest_storage_options(&self) -> Result>>; /// Poll until the columns are fully indexed. Will return Error::Timeout if the columns /// are not fully indexed within the timeout. async fn wait_for_index( @@ -1257,10 +1267,32 @@ impl Table { /// Get the storage options used when opening this table, if any. /// /// Warning: This is an internal API and the return value is subject to change. + #[deprecated(since = "0.25.0", note = "Use initial_storage_options() instead")] pub async fn storage_options(&self) -> Option> { + #[allow(deprecated)] self.inner.storage_options().await } + /// Get the initial storage options that were passed in when opening this table. + /// + /// For dynamically refreshed options (e.g., credential vending), use [`Self::latest_storage_options`]. + /// + /// Warning: This is an internal API and the return value is subject to change. + pub async fn initial_storage_options(&self) -> Option> { + self.inner.initial_storage_options().await + } + + /// Get the latest storage options, refreshing from provider if configured. + /// + /// This method is useful for credential vending scenarios where storage options + /// may be refreshed dynamically. If no dynamic provider is configured, this + /// returns the initial static options. + /// + /// Warning: This is an internal API and the return value is subject to change. + pub async fn latest_storage_options(&self) -> Result>> { + self.inner.latest_storage_options().await + } + /// Get statistics about an index. /// Returns None if the index does not exist. pub async fn index_stats( @@ -3142,6 +3174,10 @@ impl BaseTable for NativeTable { } async fn storage_options(&self) -> Option> { + self.initial_storage_options().await + } + + async fn initial_storage_options(&self) -> Option> { self.dataset .get() .await @@ -3149,6 +3185,11 @@ impl BaseTable for NativeTable { .and_then(|dataset| dataset.initial_storage_options().cloned()) } + async fn latest_storage_options(&self) -> Result>> { + let dataset = self.dataset.get().await?; + Ok(dataset.latest_storage_options().await?.map(|o| o.0)) + } + async fn index_stats(&self, index_name: &str) -> Result> { let stats = match self .dataset