mirror of
https://github.com/lancedb/lancedb.git
synced 2026-08-18 12:08:35 +00:00
feat(remote): add seekable blob range reads (#3750)
## Summary - Implements Cloud `fetch_blob_files`: returns real seekable `BlobFile` handles over HTTP Range instead of `NotSupported`. - Completes the second Cloud blob read verb after #3684 (`fetch_blobs` = eager whole bytes; this = lazy / partial / sequential reads). - Same public handle API as local (`read_range`, `read_up_to`, `seek`, `tell`, `close`), so one code path works for local and Cloud. Large blobs (video, audio, PDFs) should not require downloading the whole object to inspect a header or stream a slice. After search, callers open a handle and read only what they need: ```python hits = table.search(vec).select(["id", "video"]).limit(5).to_arrow() with table.fetch_blob_files("video", hits)[0] as f: header = f.read_range(0, 256) f.seek(keyframe_offset) chunk = f.read_up_to(1 << 20) ``` ### Behavior - Handle creation probes size with `bytes=0-0` (bounded concurrency, input order preserved). - `204` → null (`None`); `416` with `bytes */0` → valid empty blob; other `416` → error. - `read_range` validates `Content-Range` and body length; OOB ranges fail with `invalid_input` before the request (aligned with Lance). - `read_up_to` reuses one open-ended Range response across sequential reads; `seek` drops it. - Servers older than 0.5.0 get a clear `NotSupported` (does not suggest `fetch_blobs`, which they also lack). ## Testing - `cargo test --features remote -p lancedb remote_blob` - `cargo test --features remote -p lancedb test_blob` - `cargo clippy --features remote --tests --examples` (no new warnings from this change) --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
+25
-11
@@ -426,9 +426,11 @@ pub struct PyBlobFile {
|
||||
impl PyBlobFile {
|
||||
fn read_bytes(self_: PyRef<'_, Self>) -> PyResult<Py<PyBytes>> {
|
||||
let inner = self_.inner.clone();
|
||||
let bytes = block_on(async move { inner.read().await })
|
||||
let py = self_.py();
|
||||
let bytes = py
|
||||
.detach(move || block_on(async move { inner.read().await }))
|
||||
.map_err(|e| PyRuntimeError::new_err(format!("blob read failed: {e}")))?;
|
||||
Ok(PyBytes::new(self_.py(), bytes.as_ref()).unbind())
|
||||
Ok(PyBytes::new(py, bytes.as_ref()).unbind())
|
||||
}
|
||||
|
||||
pub fn read(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
|
||||
@@ -444,24 +446,32 @@ impl PyBlobFile {
|
||||
|
||||
fn close(self_: PyRef<'_, Self>) -> PyResult<()> {
|
||||
let inner = self_.inner.clone();
|
||||
block_on(async move { inner.close().await })
|
||||
self_
|
||||
.py()
|
||||
.detach(move || block_on(async move { inner.close().await }))
|
||||
.map_err(|e| PyRuntimeError::new_err(format!("blob close failed: {e}")))
|
||||
}
|
||||
|
||||
fn is_closed(self_: PyRef<'_, Self>) -> bool {
|
||||
let inner = self_.inner.clone();
|
||||
block_on(async move { inner.is_closed().await })
|
||||
self_
|
||||
.py()
|
||||
.detach(move || block_on(async move { inner.is_closed().await }))
|
||||
}
|
||||
|
||||
fn seek(self_: PyRef<'_, Self>, position: u64) -> PyResult<()> {
|
||||
let inner = self_.inner.clone();
|
||||
block_on(async move { inner.seek(position).await })
|
||||
self_
|
||||
.py()
|
||||
.detach(move || block_on(async move { inner.seek(position).await }))
|
||||
.map_err(|e| PyRuntimeError::new_err(format!("blob seek failed: {e}")))
|
||||
}
|
||||
|
||||
fn tell(self_: PyRef<'_, Self>) -> PyResult<u64> {
|
||||
let inner = self_.inner.clone();
|
||||
block_on(async move { inner.tell().await })
|
||||
self_
|
||||
.py()
|
||||
.detach(move || block_on(async move { inner.tell().await }))
|
||||
.map_err(|e| PyRuntimeError::new_err(format!("blob tell failed: {e}")))
|
||||
}
|
||||
|
||||
@@ -475,16 +485,20 @@ impl PyBlobFile {
|
||||
.checked_add(length as u64)
|
||||
.ok_or_else(|| PyValueError::new_err("offset + length overflowed"))?;
|
||||
let inner = self_.inner.clone();
|
||||
let bytes = block_on(async move { inner.read_range(offset..end).await })
|
||||
let py = self_.py();
|
||||
let bytes = py
|
||||
.detach(move || block_on(async move { inner.read_range(offset..end).await }))
|
||||
.map_err(|e| PyRuntimeError::new_err(format!("blob read_range failed: {e}")))?;
|
||||
Ok(PyBytes::new(self_.py(), bytes.as_ref()).unbind())
|
||||
Ok(PyBytes::new(py, bytes.as_ref()).unbind())
|
||||
}
|
||||
|
||||
fn read_up_to(self_: PyRef<'_, Self>, length: usize) -> PyResult<Py<PyBytes>> {
|
||||
let inner = self_.inner.clone();
|
||||
let bytes = block_on(async move { inner.read_up_to(length).await })
|
||||
.map_err(|e| PyRuntimeError::new_err(format!("blob read failed: {e}")))?;
|
||||
Ok(PyBytes::new(self_.py(), bytes.as_ref()).unbind())
|
||||
let py = self_.py();
|
||||
let bytes = py
|
||||
.detach(move || block_on(async move { inner.read_up_to(length).await }))
|
||||
.map_err(|e| PyRuntimeError::new_err(format!("blob read_up_to failed: {e}")))?;
|
||||
Ok(PyBytes::new(py, bytes.as_ref()).unbind())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user