Files
lancedb/python/src/table.rs
T
Xuanwo f6f8119ba4 feat: expose structured Function refresh options in SDKs (#5047)
## Problem

ENT-3267 still lacks typed refresh controls for `supervisor_resources`
and `planning_workers` in Python, Rust, and Node, and Node has no
`retry_policy` refresh option. Callers must drop to SQL for those
controls.

## Change

- Add canonical Rust `FunctionRefreshPodResources` and
`FunctionRefreshPlanningConfig`, and serialize them through
`RefreshColumnOptions`.
- Expose both options in the Python sync/async wrappers and PyO3
binding.
- Expose both options in the Node NAPI object and add a Node
`retryPolicy` object backed by the existing Rust `FunctionRetryPolicy`.
- Omitted options stay omitted; branch targeting continues through the
table handle. No server, dependency, or pin changes are included.

Client-side checks only reject non-positive structural values.
Deployment resource limits and column-kind validation remain
server-side.

## Validation

`cargo fmt --all -- --check`, `python3 -m py_compile`, and `git diff
--check` pass. No native build was run locally; CI compiles and tests
the change. Generated JS API pages are unchanged.
2026-11-14 20:59:44 +08:00

3186 lines
112 KiB
Rust

// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright The LanceDB Authors
use lancedb::table::computed_columns::ComputedColumnOptions;
use std::{
collections::HashMap,
sync::{
Arc,
atomic::{AtomicBool, Ordering},
},
};
use crate::runtime::{block_on, future_into_py, spawn_background};
use crate::{
connection::Connection,
error::PythonErrorExt,
expr::PyExpr,
index::{CodeAnalyzerOptions, IndexConfig, extract_index_params},
job::JobFailureInfo,
query::{Query, TakeQuery},
table::scannable::PyScannable,
};
use arrow::{
array::{Array, LargeBinaryArray, RecordBatchIterator, RecordBatchReader},
datatypes::{DataType, Schema},
ffi_stream::ArrowArrayStreamReader,
pyarrow::{FromPyArrow, PyArrowType, ToPyArrow},
};
use lancedb::blob::{BlobDescriptor, BlobFile, BlobRangeRequest};
use lancedb::connection::LanceFileVersion;
use lancedb::index::scalar::FtsIndexBuilder;
use lancedb::table::{
AddDataMode, ColumnAlteration, CompactionMode, CompactionOptions, Duration,
FieldMetadataUpdate, FtsToken as LanceDbFtsToken, FunctionRefreshPlanningConfig,
FunctionRefreshPodResources, FunctionWorkerResourceBudget, IndexRemapMode, NewColumnTransform,
Ref, RefreshColumnOptions, Table as LanceDbTable,
};
use lancedb::tokenize as lancedb_tokenize;
use pyo3::{
Bound, FromPyObject, Py, PyAny, PyRef, PyResult, Python,
exceptions::{PyRuntimeError, PyValueError},
pyclass, pyfunction, pymethods,
types::{
IntoPyDict, PyAnyMethods, PyBool, PyBytes, PyDict, PyDictMethods, PyList, PyListMethods,
},
};
mod scannable;
/// Keyword arguments shared by `refresh_column` and `refresh_column_async`.
struct RefreshColumnArgs {
where_clause: Option<String>,
batch_size: Option<i64>,
max_concurrency: Option<u32>,
publication_interval_rows: Option<i64>,
resource_budget: Option<FunctionWorkerResourceBudget>,
supervisor_resources: Option<FunctionRefreshPodResources>,
planning_workers: Option<FunctionRefreshPlanningConfig>,
gpu_class: Option<String>,
gpu_count: Option<i64>,
retry_policy: Option<String>,
error_policy: Option<String>,
max_failed_fraction: Option<f64>,
max_failed_rows: Option<i64>,
row_timeout_seconds: Option<f64>,
init_timeout_seconds: Option<f64>,
num_workers: Option<i64>,
max_executor_processes_per_worker: Option<i64>,
}
fn positive_count<T: TryFrom<i64>>(value: Option<i64>, name: &str) -> PyResult<Option<T>> {
value
.map(|v| {
T::try_from(v)
.ok()
.filter(|_| v > 0)
.ok_or_else(|| PyValueError::new_err(format!("{name} must be greater than zero")))
})
.transpose()
}
fn positive_u32_integer(value: Option<&Bound<'_, PyAny>>, name: &str) -> PyResult<Option<u32>> {
let Some(value) = value else {
return Ok(None);
};
if value.is_none() || value.is_instance_of::<PyBool>() {
return Err(PyValueError::new_err(format!(
"{name} must be an integer between 1 and 4294967295"
)));
}
let parsed = value.extract::<i64>().map_err(|_| {
PyValueError::new_err(format!(
"{name} must be an integer between 1 and 4294967295"
))
})?;
u32::try_from(parsed)
.ok()
.filter(|parsed| *parsed > 0)
.map(Some)
.ok_or_else(|| {
PyValueError::new_err(format!(
"{name} must be an integer between 1 and 4294967295"
))
})
}
fn resource_budget_integer(
value: &Bound<'_, PyAny>,
name: &str,
allow_zero: bool,
) -> PyResult<u64> {
if value.is_none() || value.is_instance_of::<PyBool>() {
return Err(PyValueError::new_err(format!(
"resource_budget.{name} must be an integer"
)));
}
let parsed = value
.extract::<u64>()
.map_err(|_| PyValueError::new_err(format!("resource_budget.{name} must be an integer")))?;
if !allow_zero && parsed == 0 {
return Err(PyValueError::new_err(format!(
"resource_budget.{name} must be greater than zero"
)));
}
Ok(parsed)
}
fn parse_resource_budget(
budget: Option<&Bound<'_, PyDict>>,
) -> PyResult<Option<FunctionWorkerResourceBudget>> {
let Some(budget) = budget else {
return Ok(None);
};
for key in budget.keys() {
let key = key
.extract::<String>()
.map_err(|_| PyValueError::new_err("resource_budget keys must be strings"))?;
if !matches!(key.as_str(), "cpu_millis" | "memory_bytes" | "gpu_count") {
return Err(PyValueError::new_err(format!(
"resource_budget contains unsupported key '{key}'"
)));
}
}
let cpu_millis = budget
.get_item("cpu_millis")?
.ok_or_else(|| PyValueError::new_err("resource_budget.cpu_millis is required"))?;
let cpu_millis = resource_budget_integer(&cpu_millis, "cpu_millis", false)?;
let memory_bytes = budget
.get_item("memory_bytes")?
.ok_or_else(|| PyValueError::new_err("resource_budget.memory_bytes is required"))?;
let memory_bytes = resource_budget_integer(&memory_bytes, "memory_bytes", false)?;
let gpu_count = match budget.get_item("gpu_count")? {
None => 0,
Some(value) if value.is_none() => 0,
Some(value) => {
u32::try_from(resource_budget_integer(&value, "gpu_count", true)?).map_err(|_| {
PyValueError::new_err("resource_budget.gpu_count must be no greater than u32::MAX")
})?
}
};
Ok(Some(
FunctionWorkerResourceBudget::new(cpu_millis, memory_bytes).with_gpu_count(gpu_count),
))
}
fn structured_integer(value: &Bound<'_, PyAny>, name: &str) -> PyResult<u64> {
if value.is_none() || value.is_instance_of::<PyBool>() {
return Err(PyValueError::new_err(format!("{name} must be an integer")));
}
let parsed = value
.extract::<u64>()
.map_err(|_| PyValueError::new_err(format!("{name} must be an integer")))?;
if parsed == 0 {
return Err(PyValueError::new_err(format!(
"{name} must be greater than zero"
)));
}
Ok(parsed)
}
fn parse_pod_resources(
resources: &Bound<'_, PyDict>,
prefix: &str,
) -> PyResult<FunctionRefreshPodResources> {
for key in resources.keys() {
let key = key
.extract::<String>()
.map_err(|_| PyValueError::new_err(format!("{prefix} keys must be strings")))?;
if !matches!(
key.as_str(),
"cpu_millis" | "memory_bytes" | "ephemeral_storage_bytes"
) {
return Err(PyValueError::new_err(format!(
"{prefix} contains unsupported key '{key}'"
)));
}
}
let get = |name: &str| -> PyResult<u64> {
let value = resources
.get_item(name)?
.ok_or_else(|| PyValueError::new_err(format!("{prefix}.{name} is required")))?;
structured_integer(&value, &format!("{prefix}.{name}"))
};
Ok(FunctionRefreshPodResources::new(
get("cpu_millis")?,
get("memory_bytes")?,
get("ephemeral_storage_bytes")?,
))
}
fn parse_supervisor_resources(
resources: Option<&Bound<'_, PyDict>>,
) -> PyResult<Option<FunctionRefreshPodResources>> {
resources
.map(|resources| parse_pod_resources(resources, "supervisor_resources"))
.transpose()
}
fn parse_planning_workers(
planning: Option<&Bound<'_, PyDict>>,
) -> PyResult<Option<FunctionRefreshPlanningConfig>> {
let Some(planning) = planning else {
return Ok(None);
};
for key in planning.keys() {
let key = key
.extract::<String>()
.map_err(|_| PyValueError::new_err("planning_workers keys must be strings"))?;
if !matches!(
key.as_str(),
"resources" | "max_workers" | "rows_per_worker" | "fragments_per_worker"
) {
return Err(PyValueError::new_err(format!(
"planning_workers contains unsupported key '{key}'"
)));
}
}
let resources = planning
.get_item("resources")?
.ok_or_else(|| PyValueError::new_err("planning_workers.resources is required"))?;
let resources = resources
.cast::<PyDict>()
.map_err(|_| PyValueError::new_err("planning_workers.resources must be a dict"))?;
let resources = parse_pod_resources(resources, "planning_workers.resources")?;
let get = |name: &str| -> PyResult<u64> {
let value = planning
.get_item(name)?
.ok_or_else(|| PyValueError::new_err(format!("planning_workers.{name} is required")))?;
structured_integer(&value, &format!("planning_workers.{name}"))
};
let max_workers = usize::try_from(get("max_workers")?)
.map_err(|_| PyValueError::new_err("planning_workers.max_workers is too large"))?;
Ok(Some(FunctionRefreshPlanningConfig::new(
resources,
max_workers,
get("rows_per_worker")?,
get("fragments_per_worker")?,
)))
}
fn refresh_column_options(args: RefreshColumnArgs) -> PyResult<RefreshColumnOptions> {
let RefreshColumnArgs {
where_clause,
batch_size,
max_concurrency,
publication_interval_rows,
resource_budget,
supervisor_resources,
planning_workers,
gpu_class,
gpu_count,
retry_policy,
error_policy,
max_failed_fraction,
max_failed_rows,
row_timeout_seconds,
init_timeout_seconds,
num_workers,
max_executor_processes_per_worker,
} = args;
let mut options = RefreshColumnOptions::default();
options.where_clause = where_clause;
options.max_failed_fraction = max_failed_fraction;
options.max_failed_rows = max_failed_rows
.map(u64::try_from)
.transpose()
.map_err(|_| PyValueError::new_err("max_failed_rows must be a non-negative integer"))?;
options.row_timeout_seconds = row_timeout_seconds;
options.init_timeout_seconds = init_timeout_seconds;
options.num_workers = positive_count(num_workers, "num_workers")?;
options.max_executor_processes_per_worker = positive_count(
max_executor_processes_per_worker,
"max_executor_processes_per_worker",
)?;
options.retry_policy = retry_policy
.map(|json| {
serde_json::from_str(&json)
.map_err(|error| PyValueError::new_err(format!("invalid retry_policy: {error}")))
})
.transpose()?;
options.error_policy = error_policy
.map(|value| {
serde_json::from_value(serde_json::Value::String(value))
.map_err(|error| PyValueError::new_err(format!("invalid error_policy: {error}")))
})
.transpose()?;
options.batch_size = batch_size
.map(usize::try_from)
.transpose()
.map_err(|_| PyValueError::new_err("batch_size must be greater than zero"))?;
options.max_concurrency = max_concurrency;
options.publication_interval_rows =
positive_count(publication_interval_rows, "publication_interval_rows")?;
options.resource_budget = resource_budget;
options.supervisor_resources = supervisor_resources;
options.planning_workers = planning_workers;
options.gpu_class = gpu_class;
options.gpu_count = gpu_count
.map(|count| {
u32::try_from(count)
.ok()
.filter(|count| *count > 0)
.ok_or_else(|| {
PyValueError::new_err(
"gpu_count must be a positive integer no greater than u32::MAX",
)
})
})
.transpose()?;
options.validate().infer_error()?;
Ok(options)
}
/// Convert `RTEStats` to a Python dict, preserving the per-table-shard list.
///
/// Deliberately not flattened to a table-level summary: a table is N
/// table shards on one node, and the per-shard detail is the reason the
/// endpoint exists — flattening hides the single hot table shard someone opened
/// it to find.
fn rte_stats_to_py(py: Python<'_>, stats: &lancedb::table::RTEStats) -> PyResult<Py<PyDict>> {
let out = PyDict::new(py);
let table_shards = PyList::empty(py);
for b in &stats.table_shards {
let e = PyDict::new(py);
e.set_item("shard_id", &b.shard_id)?;
e.set_item("status", &b.status)?;
e.set_item("writer_epoch", b.writer_epoch)?;
e.set_item("manifest_version", b.manifest_version)?;
e.set_item("current_generation", b.current_generation)?;
e.set_item(
"replay_after_wal_entry_position",
b.replay_after_wal_entry_position,
)?;
e.set_item(
"wal_entry_position_last_seen",
b.wal_entry_position_last_seen,
)?;
let sstables = PyList::empty(py);
for g in &b.sstables {
let ge = PyDict::new(py);
ge.set_item("generation", g.generation)?;
ge.set_item("bytes", g.bytes)?;
ge.set_item("rows", g.rows)?;
sstables.append(ge)?;
}
e.set_item("sstables", sstables)?;
e.set_item("compacting", b.compacting)?;
e.set_item(
"memtables",
b.memtables
.as_ref()
.map(|ms| {
let l = PyList::empty(py);
for m in ms {
let d = PyDict::new(py);
d.set_item("generation", m.generation)?;
d.set_item("rows", m.rows)?;
d.set_item("bytes", m.bytes)?;
d.set_item("batches", m.batches)?;
d.set_item("indexes", m.indexes.clone())?;
l.append(d)?;
}
PyResult::Ok(l.unbind())
})
.transpose()?,
)?;
table_shards.append(e)?;
}
out.set_item("table_shards", table_shards)?;
Ok(out.unbind())
}
#[derive(FromPyObject)]
enum PredicateArg {
Expr(PyExpr),
Sql(String),
}
fn validate_positive_u32(value: u64, name: &str) -> PyResult<usize> {
if !(1..=u32::MAX as u64).contains(&value) {
return Err(PyValueError::new_err(format!(
"{name} must be between 1 and {}",
u32::MAX
)));
}
Ok(value as usize)
}
fn positive_u32(value: &Bound<'_, PyAny>, name: &str) -> PyResult<usize> {
validate_positive_u32(value.extract()?, name)
}
fn optional_positive_u32(value: &Bound<'_, PyAny>, name: &str) -> PyResult<Option<usize>> {
value
.extract::<Option<u64>>()?
.map(|value| validate_positive_u32(value, name))
.transpose()
}
fn optional_positive_usize(value: &Bound<'_, PyAny>, name: &str) -> PyResult<Option<usize>> {
let value: Option<usize> = value.extract()?;
if value == Some(0) {
return Err(PyValueError::new_err(format!(
"{name} must be greater than 0"
)));
}
Ok(value)
}
fn optional_positive_u64(value: &Bound<'_, PyAny>, name: &str) -> PyResult<Option<u64>> {
let value: Option<u64> = value.extract()?;
if value == Some(0) {
return Err(PyValueError::new_err(format!(
"{name} must be greater than 0"
)));
}
Ok(value)
}
fn u32_list(value: &Bound<'_, PyAny>, name: &str) -> PyResult<Vec<u32>> {
value
.extract::<Vec<i64>>()?
.into_iter()
.map(|value| {
u32::try_from(value).map_err(|_| {
PyValueError::new_err(format!(
"{name} must contain values between 0 and {}",
u32::MAX
))
})
})
.collect()
}
fn optional_i64_bounded_u64(value: &Bound<'_, PyAny>, name: &str) -> PyResult<Option<u64>> {
let value: Option<u64> = value.extract()?;
if value.is_some_and(|value| value > i64::MAX as u64) {
return Err(PyValueError::new_err(format!(
"{name} must be at most {}",
i64::MAX
)));
}
Ok(value)
}
fn parse_compaction_options(options: Option<&Bound<'_, PyDict>>) -> PyResult<CompactionOptions> {
let mut parsed = CompactionOptions::default();
let Some(options) = options else {
return Ok(parsed);
};
for (key, value) in options.iter() {
let key: String = key.extract()?;
match key.as_str() {
"target_rows_per_fragment" => {
parsed.target_rows_per_fragment = positive_u32(&value, &key)?
}
"max_rows_per_group" => parsed.max_rows_per_group = positive_u32(&value, &key)?,
"max_bytes_per_file" => parsed.max_bytes_per_file = value.extract()?,
"materialize_deletions" => parsed.materialize_deletions = value.extract()?,
"materialize_deletions_threshold" => {
parsed.materialize_deletions_threshold = value.extract()?
}
"num_threads" => parsed.num_threads = optional_positive_usize(&value, &key)?,
"batch_size" => parsed.batch_size = optional_positive_u32(&value, &key)?,
"io_buffer_size" => parsed.io_buffer_size = optional_i64_bounded_u64(&value, &key)?,
"defer_index_remap" => parsed.defer_index_remap = value.extract()?,
"index_remap_mode" => {
let mode: String = value.extract()?;
parsed.index_remap_mode = IndexRemapMode::try_from(mode.as_str())
.map_err(|err| PyValueError::new_err(err.to_string()))?;
}
"compaction_mode" => {
let mode: Option<String> = value.extract()?;
parsed.compaction_mode = mode
.map(|mode| {
CompactionMode::try_from(mode.as_str())
.map_err(|err| PyValueError::new_err(err.to_string()))
})
.transpose()?;
}
"binary_copy_read_batch_bytes" => {
parsed.binary_copy_read_batch_bytes = value.extract()?
}
"max_source_fragments" => parsed.max_source_fragments = value.extract()?,
"max_source_rows" => parsed.max_source_rows = optional_positive_usize(&value, &key)?,
"max_source_bytes" => parsed.max_source_bytes = optional_positive_u64(&value, &key)?,
"excluded_fragment_ids" => parsed.excluded_fragment_ids = u32_list(&value, &key)?,
"max_overlays_per_fragment" => parsed.max_overlays_per_fragment = value.extract()?,
"data_storage_version" => {
let version: String = value.extract()?;
parsed.data_storage_version = Some(
version
.parse::<LanceFileVersion>()
.map_err(|err| PyValueError::new_err(err.to_string()))?,
);
}
_ => {
return Err(PyValueError::new_err(format!(
"Invalid compaction option: {key}"
)));
}
}
}
Ok(parsed)
}
/// Statistics about a compaction operation.
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct CompactionStats {
/// The number of fragments removed
pub fragments_removed: u64,
/// The number of new, compacted fragments added
pub fragments_added: u64,
/// The number of data files removed
pub files_removed: u64,
/// The number of new, compacted data files added
pub files_added: u64,
}
/// Statistics about a cleanup operation
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct RemovalStats {
/// The number of bytes removed
pub bytes_removed: u64,
/// The number of old versions removed
pub old_versions_removed: u64,
}
/// Statistics about an optimize operation
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct OptimizeStats {
/// Statistics about the compaction operation
pub compaction: CompactionStats,
/// Statistics about the removal operation
pub prune: RemovalStats,
}
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct UpdateResult {
pub rows_updated: u64,
pub version: u64,
}
#[pymethods]
impl UpdateResult {
pub fn __repr__(&self) -> String {
format!(
"UpdateResult(rows_updated={}, version={})",
self.rows_updated, self.version
)
}
}
impl From<lancedb::table::UpdateResult> for UpdateResult {
fn from(result: lancedb::table::UpdateResult) -> Self {
Self {
rows_updated: result.rows_updated,
version: result.version,
}
}
}
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct AddResult {
pub version: u64,
}
#[pymethods]
impl AddResult {
pub fn __repr__(&self) -> String {
format!("AddResult(version={})", self.version)
}
}
impl From<lancedb::table::AddResult> for AddResult {
fn from(result: lancedb::table::AddResult) -> Self {
Self {
version: result.version,
}
}
}
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct DeleteResult {
pub num_deleted_rows: u64,
pub version: u64,
}
#[pymethods]
impl DeleteResult {
pub fn __repr__(&self) -> String {
format!(
"DeleteResult(num_deleted_rows={}, version={})",
self.num_deleted_rows, self.version
)
}
}
impl From<lancedb::table::DeleteResult> for DeleteResult {
fn from(result: lancedb::table::DeleteResult) -> Self {
Self {
num_deleted_rows: result.num_deleted_rows,
version: result.version,
}
}
}
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct MergeResult {
pub version: u64,
pub num_updated_rows: u64,
pub num_inserted_rows: u64,
pub num_deleted_rows: u64,
pub num_attempts: u32,
pub num_rows: u64,
}
#[pymethods]
impl MergeResult {
pub fn __repr__(&self) -> String {
format!(
"MergeResult(version={}, num_updated_rows={}, num_inserted_rows={}, num_deleted_rows={}, num_attempts={}, num_rows={})",
self.version,
self.num_updated_rows,
self.num_inserted_rows,
self.num_deleted_rows,
self.num_attempts,
self.num_rows
)
}
}
impl From<lancedb::table::MergeResult> for MergeResult {
fn from(result: lancedb::table::MergeResult) -> Self {
Self {
version: result.version,
num_updated_rows: result.num_updated_rows,
num_inserted_rows: result.num_inserted_rows,
num_deleted_rows: result.num_deleted_rows,
num_attempts: result.num_attempts,
num_rows: result.num_rows,
}
}
}
/// Render for `__repr__`, so the default reads as Python's `None` rather than
/// Rust's `Some([..])`.
fn fmt_maintained(maintained: &Option<Vec<String>>) -> String {
match maintained {
Some(names) => format!("{:?}", names),
None => "None".to_string(),
}
}
/// Specification enabling the realtime engine (RTE), which routes
/// `merge_insert` through Lance's MemWAL LSM-style write path.
///
/// Constructed via the `bucket(...)`, `identity(...)`, or `unsharded()`
/// classmethods, then optionally chain `with_maintained_indexes(...)` and
/// `with_writer_config_defaults(...)`. A fresh spec maintains every index the
/// table has, including ones created later.
#[pyclass(module = "lancedb._lancedb", from_py_object)]
#[derive(Clone, Debug)]
pub struct RTEWriteSpec {
inner: lancedb::table::RTEWriteSpec,
}
#[pymethods]
impl RTEWriteSpec {
/// Hash-bucket sharding by the unenforced primary key column.
#[staticmethod]
pub fn bucket(column: String, num_buckets: u32) -> Self {
Self {
inner: lancedb::table::RTEWriteSpec::bucket(column, num_buckets),
}
}
/// Identity sharding — shard by the raw value of `column`.
///
/// `column` must be a deterministic function of the unenforced primary
/// key: every row with a given primary key must always produce the same
/// `column` value, or upserts of that key can land in different shards
/// and a stale version can win. Typically `column` is the primary key
/// itself or a stable attribute of it.
#[staticmethod]
pub fn identity(column: String) -> Self {
Self {
inner: lancedb::table::RTEWriteSpec::identity(column),
}
}
/// No sharding — every `merge_insert` call writes to a single
/// RTE shard.
#[staticmethod]
pub fn unsharded() -> Self {
Self {
inner: lancedb::table::RTEWriteSpec::unsharded(),
}
}
/// Set which indexes the RTE maintains. `None` (the default) is
/// every index the table has, including ones created later; a list is
/// verbatim, and an empty list maintains nothing.
#[pyo3(signature = (indexes))]
pub fn with_maintained_indexes(&self, indexes: Option<Vec<String>>) -> Self {
Self {
inner: self.inner.clone().with_maintained_indexes(indexes),
}
}
/// Replace the default `ShardWriter` configuration recorded in the
/// MemWAL index, so every writer starts from the same defaults.
pub fn with_writer_config_defaults(&self, defaults: HashMap<String, String>) -> Self {
Self {
inner: self.inner.clone().with_writer_config_defaults(defaults),
}
}
pub fn __repr__(&self) -> String {
match &self.inner {
lancedb::table::RTEWriteSpec::Bucket {
column,
num_buckets,
maintained_indexes,
writer_config_defaults,
} => format!(
"RTEWriteSpec.bucket(column={:?}, num_buckets={}, maintained_indexes={}, writer_config_defaults={:?})",
column,
num_buckets,
fmt_maintained(maintained_indexes),
writer_config_defaults,
),
lancedb::table::RTEWriteSpec::Identity {
column,
maintained_indexes,
writer_config_defaults,
} => format!(
"RTEWriteSpec.identity(column={:?}, maintained_indexes={}, writer_config_defaults={:?})",
column,
fmt_maintained(maintained_indexes),
writer_config_defaults,
),
lancedb::table::RTEWriteSpec::Unsharded {
maintained_indexes,
writer_config_defaults,
} => format!(
"RTEWriteSpec.unsharded(maintained_indexes={}, writer_config_defaults={:?})",
fmt_maintained(maintained_indexes),
writer_config_defaults,
),
}
}
/// Discriminator string identifying the variant ("bucket", "identity",
/// or "unsharded").
#[getter]
pub fn spec_type(&self) -> &'static str {
match &self.inner {
lancedb::table::RTEWriteSpec::Bucket { .. } => "bucket",
lancedb::table::RTEWriteSpec::Identity { .. } => "identity",
lancedb::table::RTEWriteSpec::Unsharded { .. } => "unsharded",
}
}
/// Bucket and identity variants: the sharding column. `None` for unsharded.
#[getter]
pub fn column(&self) -> Option<String> {
match &self.inner {
lancedb::table::RTEWriteSpec::Bucket { column, .. }
| lancedb::table::RTEWriteSpec::Identity { column, .. } => Some(column.clone()),
lancedb::table::RTEWriteSpec::Unsharded { .. } => None,
}
}
/// Bucket variant only: the number of buckets.
#[getter]
pub fn num_buckets(&self) -> Option<u32> {
match &self.inner {
lancedb::table::RTEWriteSpec::Bucket { num_buckets, .. } => Some(*num_buckets),
_ => None,
}
}
/// Indexes the RTE keeps up to date, or `None` for every supported one.
#[getter]
pub fn maintained_indexes(&self) -> Option<Vec<String>> {
self.inner.maintained_indexes().map(<[String]>::to_vec)
}
/// Default `ShardWriter` configuration recorded by this spec.
#[getter]
pub fn writer_config_defaults(&self) -> HashMap<String, String> {
self.inner.writer_config_defaults().clone()
}
}
impl From<RTEWriteSpec> for lancedb::table::RTEWriteSpec {
fn from(spec: RTEWriteSpec) -> Self {
spec.inner
}
}
impl From<lancedb::table::RTEWriteSpec> for RTEWriteSpec {
fn from(inner: lancedb::table::RTEWriteSpec) -> Self {
Self { inner }
}
}
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct AddColumnsResult {
pub version: u64,
}
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct RefreshColumnResult {
pub rows_filled: u64,
pub version: u64,
}
#[pymethods]
impl RefreshColumnResult {
pub fn __repr__(&self) -> String {
format!(
"RefreshColumnResult(rows_filled={}, version={})",
self.rows_filled, self.version
)
}
}
impl From<lancedb::table::RefreshColumnResult> for RefreshColumnResult {
fn from(result: lancedb::table::RefreshColumnResult) -> Self {
Self {
rows_filled: result.rows_filled,
version: result.version,
}
}
}
/// One row a Function refresh skipped, as the server recorded it.
#[pyclass(module = "lancedb._lancedb", get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct FunctionErrorRecord {
pub attempts: Option<u32>,
pub job_id: String,
pub fragment_id: u64,
pub row_offset: Option<u32>,
pub row_id: Option<u64>,
pub row_address: Option<u64>,
pub column: String,
pub function: String,
pub function_version: String,
pub table_version: u64,
pub error_type: String,
pub error_message: String,
/// User-code traceback, possibly truncated by the server.
pub traceback: Option<String>,
pub created_at_millis: i64,
}
#[pymethods]
impl FunctionErrorRecord {
pub fn __repr__(&self) -> String {
let row_offset = self
.row_offset
.map_or_else(|| "None".to_string(), |value| value.to_string());
let row_address = self
.row_address
.map_or_else(|| "None".to_string(), |value| value.to_string());
format!(
"FunctionErrorRecord(job_id={:?}, fragment_id={}, row_offset={}, row_address={}, \
column={:?}, error_type={:?}, error_message={:?})",
self.job_id,
self.fragment_id,
row_offset,
row_address,
self.column,
self.error_type,
self.error_message
)
}
}
impl From<lancedb::function::FunctionErrorRecord> for FunctionErrorRecord {
fn from(record: lancedb::function::FunctionErrorRecord) -> Self {
Self {
attempts: record.attempts,
job_id: record.job_id,
fragment_id: record.fragment_id,
row_offset: record.row_offset,
row_id: record.row_id,
row_address: record.row_address,
column: record.column,
function: record.function,
function_version: record.function_version,
table_version: record.table_version,
error_type: record.error_type,
error_message: record.error_message,
traceback: record.traceback,
created_at_millis: record.created_at_millis,
}
}
}
/// A fragment whose per-row error detail was capped.
#[pyclass(module = "lancedb._lancedb", get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct FunctionErrorFragment {
pub job_id: String,
pub fragment_id: u64,
pub rows_skipped: u64,
pub rows_recorded: u64,
}
#[pymethods]
impl FunctionErrorFragment {
pub fn __repr__(&self) -> String {
format!(
"FunctionErrorFragment(job_id={:?}, fragment_id={}, rows_skipped={}, rows_recorded={})",
self.job_id, self.fragment_id, self.rows_skipped, self.rows_recorded
)
}
}
impl From<lancedb::function::FunctionErrorFragment> for FunctionErrorFragment {
fn from(fragment: lancedb::function::FunctionErrorFragment) -> Self {
Self {
job_id: fragment.job_id,
fragment_id: fragment.fragment_id,
rows_skipped: fragment.rows_skipped,
rows_recorded: fragment.rows_recorded,
}
}
}
/// A table's per-row Function errors.
#[pyclass(module = "lancedb._lancedb", get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct FunctionErrors {
pub records: Vec<FunctionErrorRecord>,
pub fragments: Vec<FunctionErrorFragment>,
pub truncated: bool,
}
#[pymethods]
impl FunctionErrors {
pub fn __repr__(&self) -> String {
format!(
"FunctionErrors(records={}, fragments={}, truncated={})",
self.records.len(),
self.fragments.len(),
self.truncated
)
}
}
impl From<lancedb::function::FunctionErrors> for FunctionErrors {
fn from(errors: lancedb::function::FunctionErrors) -> Self {
Self {
records: errors.records.into_iter().map(Into::into).collect(),
fragments: errors.fragments.into_iter().map(Into::into).collect(),
truncated: errors.truncated,
}
}
}
/// Exact immutable Function revision attached to a table output.
#[pyclass(module = "lancedb._lancedb", get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct FunctionColumnVersionStatus {
pub name: String,
pub namespace_path: Vec<String>,
pub object_id: String,
pub location: String,
pub version: String,
pub manifest_digest: String,
}
impl From<lancedb::function::FunctionVersionRef> for FunctionColumnVersionStatus {
fn from(function: lancedb::function::FunctionVersionRef) -> Self {
Self {
name: function.name,
namespace_path: function.namespace_path,
object_id: function.object_id,
location: function.location,
version: function.version,
manifest_digest: function.manifest_digest,
}
}
}
/// Latest matching refresh Job projected for a Function output.
#[pyclass(module = "lancedb._lancedb", get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct FunctionColumnLastRefreshJob {
pub job_id: String,
pub state: String,
pub automatic: Option<bool>,
pub completed_at_millis: Option<i64>,
pub failure: Option<JobFailureInfo>,
}
impl From<lancedb::table::FunctionColumnLastRefreshJob> for FunctionColumnLastRefreshJob {
fn from(job: lancedb::table::FunctionColumnLastRefreshJob) -> Self {
Self {
job_id: job.job_id,
state: job.state,
automatic: job.automatic,
completed_at_millis: job.completed_at_millis,
failure: job.failure.map(Into::into),
}
}
}
/// Automatic-refresh projection for a Function output.
#[pyclass(module = "lancedb._lancedb", get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct FunctionColumnAutoRefreshStatus {
pub enabled: bool,
pub interval_seconds: Option<u64>,
pub active_job_id: Option<String>,
pub state: String,
pub failure_streak: u32,
pub effective_cooldown_millis: Option<u64>,
pub blocked_reason: Option<String>,
pub next_run_at_millis: Option<i64>,
pub next_run_at_reason: Option<String>,
}
impl From<lancedb::table::FunctionColumnAutoRefreshStatus> for FunctionColumnAutoRefreshStatus {
fn from(status: lancedb::table::FunctionColumnAutoRefreshStatus) -> Self {
Self {
enabled: status.enabled,
interval_seconds: status.interval_seconds,
active_job_id: status.active_job_id,
state: status.state.as_str().to_string(),
failure_streak: status.failure_streak,
effective_cooldown_millis: status.effective_cooldown_millis,
blocked_reason: status
.blocked_reason
.map(|reason| reason.as_str().to_string()),
next_run_at_millis: status.next_run_at_millis,
next_run_at_reason: status.next_run_at_reason,
}
}
}
/// Exact four-way status of one Function output at one table version.
#[pyclass(module = "lancedb._lancedb", get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct FunctionColumnStatus {
pub column: String,
pub binding_id: String,
pub function: FunctionColumnVersionStatus,
pub table_version: u64,
pub rows: u64,
pub fresh: u64,
pub stale: u64,
pub never_computed: u64,
pub failed: u64,
pub last_refresh_job: Option<FunctionColumnLastRefreshJob>,
pub last_failure_job: Option<FunctionColumnLastRefreshJob>,
pub auto_refresh: FunctionColumnAutoRefreshStatus,
}
#[pymethods]
impl FunctionColumnStatus {
pub fn __repr__(&self) -> String {
format!(
"FunctionColumnStatus(column={:?}, table_version={}, rows={}, fresh={}, stale={}, never_computed={}, failed={})",
self.column,
self.table_version,
self.rows,
self.fresh,
self.stale,
self.never_computed,
self.failed
)
}
}
impl From<lancedb::table::FunctionColumnStatus> for FunctionColumnStatus {
fn from(status: lancedb::table::FunctionColumnStatus) -> Self {
Self {
column: status.column,
binding_id: status.binding_id,
function: status.function.into(),
table_version: status.table_version,
rows: status.rows,
fresh: status.fresh,
stale: status.stale,
never_computed: status.never_computed,
failed: status.failed,
last_refresh_job: status.last_refresh_job.map(Into::into),
last_failure_job: status.last_failure_job.map(Into::into),
auto_refresh: status.auto_refresh.into(),
}
}
}
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct RefreshMaterializedViewResult {
pub mode: Option<String>,
pub rows_written: Option<u64>,
pub source_version: Option<u64>,
pub version: u64,
}
#[pymethods]
impl RefreshMaterializedViewResult {
#[staticmethod]
pub fn from_json(value: &str) -> PyResult<Self> {
let result: lancedb::RefreshMaterializedViewResult =
serde_json::from_str(value).map_err(|err| {
PyValueError::new_err(format!(
"failed to decode materialized-view refresh result: {err}"
))
})?;
Ok(Self::from(result))
}
pub fn __repr__(&self) -> String {
format!(
"RefreshMaterializedViewResult(mode={}, rows_written={}, source_version={}, version={})",
self.mode.as_deref().unwrap_or("None"),
self.rows_written
.map_or_else(|| "None".into(), |value| value.to_string()),
self.source_version
.map_or_else(|| "None".into(), |value| value.to_string()),
self.version
)
}
}
impl From<lancedb::RefreshMaterializedViewResult> for RefreshMaterializedViewResult {
fn from(result: lancedb::RefreshMaterializedViewResult) -> Self {
let mode = result.mode.map(|mode| match mode {
lancedb::RefreshMode::Rebuild => "rebuild",
});
Self {
mode: mode.map(str::to_string),
rows_written: result.rows_written,
source_version: result.source_version,
version: result.version,
}
}
}
#[pymethods]
impl AddColumnsResult {
pub fn __repr__(&self) -> String {
format!("AddColumnsResult(version={})", self.version)
}
}
impl From<lancedb::table::AddColumnsResult> for AddColumnsResult {
fn from(result: lancedb::table::AddColumnsResult) -> Self {
Self {
version: result.version,
}
}
}
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct AlterColumnsResult {
pub version: u64,
}
#[pymethods]
impl AlterColumnsResult {
pub fn __repr__(&self) -> String {
format!("AlterColumnsResult(version={})", self.version)
}
}
impl From<lancedb::table::AlterColumnsResult> for AlterColumnsResult {
fn from(result: lancedb::table::AlterColumnsResult) -> Self {
Self {
version: result.version,
}
}
}
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct UpdateFieldMetadataResult {
pub version: u64,
}
#[pymethods]
impl UpdateFieldMetadataResult {
pub fn __repr__(&self) -> String {
format!("UpdateFieldMetadataResult(version={})", self.version)
}
}
impl From<lancedb::table::UpdateFieldMetadataResult> for UpdateFieldMetadataResult {
fn from(result: lancedb::table::UpdateFieldMetadataResult) -> Self {
Self {
version: result.version,
}
}
}
#[pyclass(get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct DropColumnsResult {
pub version: u64,
}
#[pymethods]
impl DropColumnsResult {
pub fn __repr__(&self) -> String {
format!("DropColumnsResult(version={})", self.version)
}
}
impl From<lancedb::table::DropColumnsResult> for DropColumnsResult {
fn from(result: lancedb::table::DropColumnsResult) -> Self {
Self {
version: result.version,
}
}
}
/// Lazy blob handle from ``Table.fetch_blob_files``.
#[pyclass(name = "BlobFile")]
pub struct PyBlobFile {
inner: Arc<BlobFile>,
closed: AtomicBool,
}
impl PyBlobFile {
fn ensure_open(&self) -> PyResult<()> {
if self.closed.load(Ordering::Acquire) {
Err(PyRuntimeError::new_err("blob file is already closed"))
} else {
Ok(())
}
}
}
fn blob_file_error(operation: &str, error: lance_core::Error) -> pyo3::PyErr {
let message = format!("blob {operation} failed: {error}");
match error {
lance_core::Error::InvalidInput { .. } => PyValueError::new_err(message),
_ => PyRuntimeError::new_err(message),
}
}
#[pymethods]
impl PyBlobFile {
fn read_bytes(self_: PyRef<'_, Self>) -> PyResult<Py<PyBytes>> {
self_.ensure_open()?;
let inner = self_.inner.clone();
let py = self_.py();
let bytes = py
.detach(move || block_on(async move { inner.read().await }))
.map_err(|e| blob_file_error("read", e))?;
Ok(PyBytes::new(py, bytes.as_ref()).unbind())
}
pub fn read(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
self_.ensure_open()?;
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let bytes = inner.read().await.map_err(|e| blob_file_error("read", e))?;
Python::attach(|py| Ok(PyBytes::new(py, bytes.as_ref()).unbind()))
})
}
fn close(self_: PyRef<'_, Self>) -> PyResult<()> {
if self_.closed.swap(true, Ordering::AcqRel) {
return Ok(());
}
let inner = self_.inner.clone();
if tokio::runtime::Handle::try_current().is_ok() {
// IOBase.__del__ can call close while cyclic GC runs on a worker.
// The status changes immediately; release Lance's async state there.
spawn_background(async move {
if let Err(error) = inner.close().await {
log::warn!("blob close failed: {error}");
}
});
return Ok(());
}
self_
.py()
.detach(move || block_on(async move { inner.close().await }))
.map_err(|e| blob_file_error("close", e))
}
fn is_closed(self_: PyRef<'_, Self>) -> bool {
self_.closed.load(Ordering::Acquire)
}
fn seek(self_: PyRef<'_, Self>, position: u64) -> PyResult<()> {
self_.ensure_open()?;
let inner = self_.inner.clone();
self_
.py()
.detach(move || block_on(async move { inner.seek(position).await }))
.map_err(|e| blob_file_error("seek", e))
}
fn tell(self_: PyRef<'_, Self>) -> PyResult<u64> {
self_.ensure_open()?;
let inner = self_.inner.clone();
self_
.py()
.detach(move || block_on(async move { inner.tell().await }))
.map_err(|e| blob_file_error("tell", e))
}
fn set_buffer_size(self_: PyRef<'_, Self>, size: usize) -> PyResult<()> {
self_.ensure_open()?;
let inner = self_.inner.clone();
self_
.py()
.detach(move || block_on(async move { inner.set_buffer_size(size).await }))
.map_err(|e| blob_file_error("set_buffer_size", e))
}
fn size(self_: PyRef<'_, Self>) -> u64 {
self_.inner.size()
}
/// Read a blob-local byte range without moving the cursor.
fn read_range(self_: PyRef<'_, Self>, offset: u64, length: usize) -> PyResult<Py<PyBytes>> {
self_.ensure_open()?;
let end = offset
.checked_add(length as u64)
.ok_or_else(|| PyValueError::new_err("offset + length overflowed"))?;
let inner = self_.inner.clone();
let py = self_.py();
let bytes = py
.detach(move || block_on(async move { inner.read_range(offset..end).await }))
.map_err(|e| blob_file_error("read_range", e))?;
Ok(PyBytes::new(py, bytes.as_ref()).unbind())
}
/// Read blob-local byte ranges without moving the cursor.
fn read_ranges(self_: PyRef<'_, Self>, ranges: Vec<(u64, u64)>) -> PyResult<Vec<Py<PyBytes>>> {
let ranges = ranges
.into_iter()
.map(|(offset, length)| {
offset
.checked_add(length)
.map(|end| offset..end)
.ok_or_else(|| PyValueError::new_err("offset + length overflowed"))
})
.collect::<PyResult<Vec<_>>>()?;
let inner = self_.inner.clone();
let py = self_.py();
let bytes = py
.detach(move || block_on(async move { inner.read_ranges(&ranges).await }))
.map_err(|e| blob_file_error("read_ranges", e))?;
Ok(bytes
.iter()
.map(|bytes| PyBytes::new(py, bytes.as_ref()).unbind())
.collect())
}
fn read_up_to(self_: PyRef<'_, Self>, length: usize) -> PyResult<Py<PyBytes>> {
self_.ensure_open()?;
let inner = self_.inner.clone();
let py = self_.py();
let bytes = py
.detach(move || block_on(async move { inner.read_up_to(length).await }))
.map_err(|e| blob_file_error("read_up_to", e))?;
Ok(PyBytes::new(py, bytes.as_ref()).unbind())
}
}
#[pyclass(module = "lancedb._lancedb", get_all, from_py_object)]
#[derive(Clone, Debug)]
pub struct FtsToken {
pub text: String,
pub position: u32,
}
#[pymethods]
impl FtsToken {
pub fn __repr__(&self) -> String {
format!("FtsToken(text={:?}, position={})", self.text, self.position)
}
}
impl From<LanceDbFtsToken> for FtsToken {
fn from(token: LanceDbFtsToken) -> Self {
Self {
text: token.text,
position: token.position,
}
}
}
#[pyfunction(signature = (
query,
*,
base_tokenizer = "simple".to_string(),
language = "English".to_string(),
max_token_length = Some(40),
lower_case = true,
stem = None,
remove_stop_words = None,
custom_stop_words = None,
ascii_folding = true,
ngram_min_length = 3,
ngram_max_length = 3,
prefix_only = false,
analyzer = None,
split_identifiers = None,
split_on_numerics = None,
preserve_original = None,
index_operators = None
))]
#[allow(clippy::too_many_arguments)]
pub fn tokenize(
query: String,
base_tokenizer: String,
language: String,
max_token_length: Option<u32>,
lower_case: bool,
stem: Option<bool>,
remove_stop_words: Option<bool>,
custom_stop_words: Option<Vec<String>>,
ascii_folding: bool,
ngram_min_length: u32,
ngram_max_length: u32,
prefix_only: bool,
analyzer: Option<String>,
split_identifiers: Option<bool>,
split_on_numerics: Option<bool>,
preserve_original: Option<bool>,
index_operators: Option<bool>,
) -> PyResult<Vec<FtsToken>> {
let code_options = CodeAnalyzerOptions {
analyzer,
split_identifiers,
split_on_numerics,
preserve_original,
index_operators,
};
let base_tokenizer = code_options.resolve_base_tokenizer(base_tokenizer)?;
let mut params = FtsIndexBuilder::default()
.base_tokenizer(base_tokenizer)
.language(&language)
.map_err(|_| {
PyValueError::new_err(format!(
"LanceDB does not support the requested language: '{}'",
language
))
})?
.max_token_length(max_token_length.map(|value| value as usize))
.lower_case(lower_case)
.ascii_folding(ascii_folding)
.ngram_min_length(ngram_min_length)
.ngram_max_length(ngram_max_length)
.ngram_prefix_only(prefix_only)
.custom_stop_words(custom_stop_words);
if let Some(stem) = stem {
params = params.stem(stem);
}
if let Some(remove_stop_words) = remove_stop_words {
params = params.remove_stop_words(remove_stop_words);
}
let params = code_options.apply(params);
let tokens = lancedb_tokenize(&query, &params).infer_error()?;
Ok(tokens.into_iter().map(FtsToken::from).collect())
}
#[pyclass]
pub struct Table {
// We keep a copy of the name to use if the inner table is dropped
name: String,
inner: Option<LanceDbTable>,
}
#[pymethods]
impl OptimizeStats {
pub fn __repr__(&self) -> String {
format!(
"OptimizeStats(compaction={:?}, prune={:?})",
self.compaction, self.prune
)
}
}
impl Table {
pub(crate) fn new(inner: LanceDbTable) -> Self {
Self {
name: inner.name().to_string(),
inner: Some(inner),
}
}
}
impl Table {
pub(crate) fn inner_ref(&self) -> PyResult<&LanceDbTable> {
self.inner
.as_ref()
.ok_or_else(|| PyRuntimeError::new_err(format!("Table {} is closed", self.name)))
}
}
#[pymethods]
impl Table {
pub fn name(&self) -> String {
self.name.clone()
}
/// Returns True if the table is open, False if it is closed.
pub fn is_open(&self) -> bool {
self.inner.is_some()
}
/// Whether this table has a local Lance dataset that the client may open.
pub fn _is_native(&self) -> PyResult<bool> {
Ok(self.inner_ref()?.as_native().is_some())
}
/// Closes the table, releasing any resources associated with it.
pub fn close(&mut self) {
self.inner.take();
}
pub fn database(&self) -> PyResult<Connection> {
let inner = self.inner_ref()?.clone();
let inner_connection =
lancedb::Connection::new(inner.database().clone(), inner.embedding_registry().clone());
Ok(Connection::new(inner_connection))
}
pub fn schema(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let schema = inner.schema().await.infer_error()?;
Python::attach(|py| schema.to_pyarrow(py).map(|obj| obj.unbind()))
})
}
#[pyo3(signature = (data, mode, progress=None, write_parallelism=None, allow_external_blob_outside_bases=false, retries=None))]
pub fn add<'a>(
self_: PyRef<'a, Self>,
data: PyScannable,
mode: String,
progress: Option<Py<PyAny>>,
write_parallelism: Option<usize>,
allow_external_blob_outside_bases: bool,
retries: Option<u8>,
) -> PyResult<Bound<'a, PyAny>> {
let mut op = self_
.inner_ref()?
.add(data)
.allow_external_blob_outside_bases(allow_external_blob_outside_bases);
if mode == "append" {
op = op.mode(AddDataMode::Append);
} else if mode == "overwrite" {
op = op.mode(AddDataMode::Overwrite);
} else {
return Err(PyValueError::new_err(format!("Invalid mode: {}", mode)));
}
if let Some(retries) = retries {
op = op.retries(retries);
}
if let Some(write_parallelism) = write_parallelism {
op = op.write_parallelism(write_parallelism);
}
if let Some(progress_obj) = progress {
let is_callable = Python::attach(|py| progress_obj.bind(py).is_callable());
if is_callable {
// Callback: call with a dict of progress info.
op = op.progress(move |p| {
Python::attach(|py| {
let dict = PyDict::new(py);
if let Err(e) = dict
.set_item("output_rows", p.output_rows())
.and_then(|_| dict.set_item("output_bytes", p.output_bytes()))
.and_then(|_| dict.set_item("total_rows", p.total_rows()))
.and_then(|_| {
dict.set_item("elapsed_seconds", p.elapsed().as_secs_f64())
})
.and_then(|_| dict.set_item("active_tasks", p.active_tasks()))
.and_then(|_| dict.set_item("total_tasks", p.total_tasks()))
.and_then(|_| dict.set_item("done", p.done()))
{
log::warn!("progress dict error: {e}");
return;
}
if let Err(e) = progress_obj.call1(py, (dict,)) {
log::warn!("progress callback error: {e}");
}
});
});
} else {
// tqdm-like: has update() method.
let mut last_rows: usize = 0;
let mut total_set = false;
op = op.progress(move |p| {
let current = p.output_rows();
let prev = last_rows;
last_rows = current;
Python::attach(|py| {
if let Some(total) = p.total_rows()
&& !total_set
{
if let Err(e) = progress_obj.setattr(py, "total", total) {
log::warn!("progress setattr error: {e}");
}
total_set = true;
}
let delta = current.saturating_sub(prev);
if delta > 0 {
if let Err(e) = progress_obj.call_method1(py, "update", (delta,)) {
log::warn!("progress update error: {e}");
}
// Show throughput and active workers in tqdm postfix.
let elapsed = p.elapsed().as_secs_f64();
if elapsed > 0.0 {
let mb_per_sec = p.output_bytes() as f64 / elapsed / 1_000_000.0;
let postfix = format!(
"{:.1} MB/s | {}/{} workers",
mb_per_sec,
p.active_tasks(),
p.total_tasks()
);
if let Err(e) =
progress_obj.call_method1(py, "set_postfix_str", (postfix,))
{
log::warn!("progress set_postfix_str error: {e}");
}
}
}
if p.done() {
// Force a final refresh so the bar shows completion.
if let Err(e) = progress_obj.call_method0(py, "refresh") {
log::warn!("progress refresh error: {e}");
}
}
});
});
}
}
future_into_py(self_.py(), async move {
let result = op.execute().await.infer_error()?;
Ok(AddResult::from(result))
})
}
#[allow(private_interfaces)]
pub fn delete(self_: PyRef<'_, Self>, condition: PredicateArg) -> PyResult<Bound<'_, PyAny>> {
// Do not hold the Python borrow across the await. The cloned Rust table
// handle is thread-safe and allows deletes on the same Python table to
// run concurrently without PyO3 reporting "Already borrowed".
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let result = match &condition {
PredicateArg::Expr(e) => inner.delete(&e.0).await,
PredicateArg::Sql(s) => inner.delete(s.as_str()).await,
}
.infer_error()?;
Ok(DeleteResult::from(result))
})
}
#[pyo3(signature = (updates, r#where=None))]
pub fn update<'a>(
self_: PyRef<'a, Self>,
updates: &Bound<'_, PyDict>,
r#where: Option<String>,
) -> PyResult<Bound<'a, PyAny>> {
let mut op = self_.inner_ref()?.update();
if let Some(only_if) = r#where {
op = op.only_if(only_if);
}
for (column_name, value) in updates.into_iter() {
let column_name: String = column_name.extract()?;
let value: String = value.extract()?;
op = op.column(column_name, value);
}
future_into_py(self_.py(), async move {
let result = op.execute().await.infer_error()?;
Ok(UpdateResult::from(result))
})
}
#[pyo3(signature = (filter=None))]
pub fn count_rows(
self_: PyRef<'_, Self>,
filter: Option<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.count_rows(filter).await.infer_error()
})
}
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None, exist_ok=false))]
pub fn create_index<'a>(
self_: PyRef<'a, Self>,
column: String,
index: Option<Bound<'_, PyAny>>,
replace: Option<bool>,
wait_timeout: Option<Bound<'_, PyAny>>,
name: Option<String>,
train: Option<bool>,
exist_ok: bool,
) -> PyResult<Bound<'a, PyAny>> {
let index = extract_index_params(&index)?;
let timeout = wait_timeout
.map(|t| t.extract::<std::time::Duration>())
.transpose()?;
let mut op = self_
.inner_ref()?
.create_index_with_timeout(&[column], index, timeout);
op = op.exist_ok(exist_ok);
if let Some(replace) = replace {
op = op.replace(replace);
}
if let Some(name) = name {
op = op.name(name);
}
if let Some(train) = train {
op = op.train(train);
}
future_into_py(self_.py(), async move {
op.execute().await.infer_error()?;
Ok(())
})
}
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (column, index=None, replace=None, wait_timeout=None, *, name=None, train=None, exist_ok=false))]
pub fn create_index_async<'a>(
self_: PyRef<'a, Self>,
column: String,
index: Option<Bound<'_, PyAny>>,
replace: Option<bool>,
wait_timeout: Option<Bound<'_, PyAny>>,
name: Option<String>,
train: Option<bool>,
exist_ok: bool,
) -> PyResult<Bound<'a, PyAny>> {
let index = extract_index_params(&index)?;
let timeout = wait_timeout
.map(|t| t.extract::<std::time::Duration>())
.transpose()?;
let mut op = self_
.inner_ref()?
.create_index_with_timeout(&[column], index, timeout);
op = op.exist_ok(exist_ok);
if let Some(replace) = replace {
op = op.replace(replace);
}
if let Some(name) = name {
op = op.name(name);
}
if let Some(train) = train {
op = op.train(train);
}
future_into_py(self_.py(), async move {
let job = op.execute_async().await.infer_error()?;
Ok(crate::job::Job::new(job))
})
}
pub fn drop_index(self_: PyRef<'_, Self>, index_name: String) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.drop_index(&index_name).await.infer_error()?;
Ok(())
})
}
pub fn wait_for_index<'a>(
self_: PyRef<'a, Self>,
index_names: Vec<String>,
timeout: Bound<'_, PyAny>,
) -> PyResult<Bound<'a, PyAny>> {
let inner = self_.inner_ref()?.clone();
let timeout = timeout.extract::<std::time::Duration>()?;
future_into_py(self_.py(), async move {
let index_refs = index_names
.iter()
.map(String::as_str)
.collect::<Vec<&str>>();
inner
.wait_for_index(&index_refs, timeout)
.await
.infer_error()?;
Ok(())
})
}
pub fn prewarm_index(self_: PyRef<'_, Self>, index_name: String) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.prewarm_index(&index_name).await.infer_error()?;
Ok(())
})
}
pub fn prewarm_data(
self_: PyRef<'_, Self>,
columns: Option<Vec<String>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.prewarm_data(columns).await.infer_error()?;
Ok(())
})
}
pub fn list_indices(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let indices = inner.list_indices().await.infer_error()?;
Python::attach(|py| {
Ok(indices
.into_iter()
.map(|idx| IndexConfig::from_lancedb(py, idx))
.collect::<Vec<_>>())
})
})
}
#[pyo3(signature = (query, *, column=None, index_name=None))]
pub fn tokenize(
self_: PyRef<'_, Self>,
query: String,
column: Option<String>,
index_name: Option<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let tokens = match (column.as_deref(), index_name.as_deref()) {
(Some(_), Some(_)) | (None, None) => {
return Err(PyValueError::new_err(
"Specify exactly one of 'column' or 'index_name'",
));
}
(Some(column), None) => inner.tokenize_with_column(&query, column).await,
(None, Some(index_name)) => inner.tokenize(&query, index_name).await,
}
.infer_error()?;
Ok(tokens.into_iter().map(FtsToken::from).collect::<Vec<_>>())
})
}
pub fn index_stats(self_: PyRef<'_, Self>, index_name: String) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let stats = inner.index_stats(&index_name).await.infer_error()?;
if let Some(stats) = stats {
Python::attach(|py| {
let dict = PyDict::new(py);
dict.set_item("num_indexed_rows", stats.num_indexed_rows)?;
dict.set_item("num_unindexed_rows", stats.num_unindexed_rows)?;
dict.set_item("index_type", stats.index_type.to_string())?;
if let Some(distance_type) = stats.distance_type {
dict.set_item("distance_type", distance_type.to_string())?;
}
if let Some(num_indices) = stats.num_indices {
dict.set_item("num_indices", num_indices)?;
}
Ok(Some(dict.unbind()))
})
} else {
Ok(None)
}
})
}
pub fn stats(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let stats = inner.stats().await.infer_error()?;
Python::attach(|py| {
let dict = PyDict::new(py);
dict.set_item("total_bytes", stats.total_bytes)?;
dict.set_item("num_rows", stats.num_rows)?;
dict.set_item("num_deleted_rows", stats.num_deleted_rows)?;
dict.set_item("num_indices", stats.num_indices)?;
let fragment_stats = PyDict::new(py);
fragment_stats.set_item("num_fragments", stats.fragment_stats.num_fragments)?;
fragment_stats.set_item(
"num_small_fragments",
stats.fragment_stats.num_small_fragments,
)?;
let fragment_lengths = PyDict::new(py);
fragment_lengths.set_item("min", stats.fragment_stats.lengths.min)?;
fragment_lengths.set_item("max", stats.fragment_stats.lengths.max)?;
fragment_lengths.set_item("mean", stats.fragment_stats.lengths.mean)?;
fragment_lengths.set_item("p25", stats.fragment_stats.lengths.p25)?;
fragment_lengths.set_item("p50", stats.fragment_stats.lengths.p50)?;
fragment_lengths.set_item("p75", stats.fragment_stats.lengths.p75)?;
fragment_lengths.set_item("p99", stats.fragment_stats.lengths.p99)?;
fragment_stats.set_item("lengths", fragment_lengths)?;
dict.set_item("fragment_stats", fragment_stats)?;
Ok(Some(dict.unbind()))
})
})
}
pub fn uri(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move { inner.uri().await.infer_error() })
}
pub fn initial_storage_options(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
Ok(inner.initial_storage_options().await)
})
}
pub fn latest_storage_options(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.latest_storage_options().await.infer_error()
})
}
pub fn __repr__(&self) -> String {
match &self.inner {
None => format!("ClosedTable({})", self.name),
Some(inner) => inner.to_string(),
}
}
pub fn version(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(
self_.py(),
async move { inner.version().await.infer_error() },
)
}
pub fn list_versions(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let versions = inner.list_versions().await.infer_error()?;
Python::attach(|py| {
versions
.iter()
.map(|v| {
let dict = PyDict::new(py);
dict.set_item("version", v.version).unwrap();
dict.set_item(
"timestamp",
v.timestamp.timestamp_nanos_opt().unwrap_or_default(),
)
.unwrap();
let tup: Vec<(&String, &String)> = v.metadata.iter().collect();
dict.set_item("metadata", tup.into_py_dict(py)?).unwrap();
Ok(dict.unbind())
})
.collect::<PyResult<Vec<_>>>()
})
})
}
pub fn checkout(self_: PyRef<'_, Self>, version: LanceVersion) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
let py = self_.py();
future_into_py(py, async move {
match version {
LanceVersion::Version(version_num) => {
inner.checkout(version_num).await.infer_error()
}
LanceVersion::Tag(tag) => inner.checkout_tag(&tag).await.infer_error(),
}
})
}
pub fn checkout_latest(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.checkout_latest().await.infer_error()
})
}
#[pyo3(signature = (version=None))]
pub fn restore(
self_: PyRef<'_, Self>,
version: Option<LanceVersion>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
let py = self_.py();
future_into_py(py, async move {
if let Some(version) = version {
match version {
LanceVersion::Version(num) => inner.checkout(num).await.infer_error()?,
LanceVersion::Tag(tag) => inner.checkout_tag(&tag).await.infer_error()?,
}
}
inner.restore().await.infer_error()
})
}
pub fn query(&self) -> PyResult<Query> {
Ok(Query::new(self.inner_ref()?.query()))
}
#[getter]
pub fn tags(&self) -> PyResult<Tags> {
Ok(Tags::new(self.inner_ref()?.clone()))
}
pub fn current_branch(&self) -> PyResult<Option<String>> {
Ok(self.inner_ref()?.current_branch())
}
#[getter]
pub fn branches(&self) -> PyResult<Branches> {
Ok(Branches::new(self.inner_ref()?.clone()))
}
#[pyo3(signature = (offsets))]
pub fn take_offsets(self_: PyRef<'_, Self>, offsets: Vec<u64>) -> PyResult<TakeQuery> {
Ok(TakeQuery::new(
self_.inner_ref()?.clone().take_offsets(offsets),
))
}
#[pyo3(signature = (row_ids))]
pub fn take_row_ids(self_: PyRef<'_, Self>, row_ids: Vec<u64>) -> PyResult<TakeQuery> {
Ok(TakeQuery::new(
self_.inner_ref()?.clone().take_row_ids(row_ids),
))
}
/// Names of the blob v2 columns declared on this table, in declaration order.
pub fn blob_columns(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.blob_columns().await.infer_error()
})
}
/// Read blob bytes for `row_ids` from blob v2 column `column`.
#[pyo3(signature = (column, row_ids, *, version=None, snapshot_identity=None))]
pub fn fetch_blobs(
self_: PyRef<'_, Self>,
column: String,
row_ids: Vec<u64>,
version: Option<u64>,
snapshot_identity: Option<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let inner = match (version, snapshot_identity) {
(None, None) => inner,
(Some(version), Some(identity)) => inner
.query_snapshot_with_identity(version, &identity)
.await
.infer_error()?,
_ => {
return Err(PyValueError::new_err(
"deferred blob reads require both query version and snapshot identity",
));
}
};
let blobs: LargeBinaryArray = inner
.fetch_blobs(column.as_str(), &row_ids)
.await
.infer_error()?;
Python::attach(|py| blobs.to_data().to_pyarrow(py).map(|obj| obj.unbind()))
})
}
/// Read row-specific blob-local byte ranges in one planned operation.
#[pyo3(signature = (column, requests))]
pub fn fetch_blob_ranges(
self_: PyRef<'_, Self>,
column: String,
requests: Vec<Vec<i128>>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
let requests = requests
.into_iter()
.map(|values| {
let [row_id, offset, length] = <[i128; 3]>::try_from(values).map_err(|_| {
PyValueError::new_err("each blob range request must have three values")
})?;
let to_u64 = |value, name| {
u64::try_from(value).map_err(|_| {
PyValueError::new_err(format!(
"blob range {name} must be a non-negative 64-bit integer"
))
})
};
Ok(BlobRangeRequest::new(
to_u64(row_id, "row id")?,
to_u64(offset, "offset")?,
to_u64(length, "length")?,
))
})
.collect::<PyResult<Vec<_>>>()?;
future_into_py(self_.py(), async move {
let blobs: LargeBinaryArray = inner
.fetch_blob_ranges(column, requests)
.await
.infer_error()?;
Python::attach(|py| blobs.to_data().to_pyarrow(py).map(|obj| obj.unbind()))
})
}
/// Open lazy blob handles for `row_ids` from blob v2 column `column`.
#[pyo3(signature = (column, row_ids, buffer_size=None))]
pub fn fetch_blob_files(
self_: PyRef<'_, Self>,
column: String,
row_ids: Vec<u64>,
buffer_size: Option<usize>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let handles = inner
.fetch_blob_files(column.as_str(), &row_ids)
.await
.infer_error()?;
if let Some(size) = buffer_size {
for file in handles.iter().flatten() {
file.set_buffer_size(size)
.await
.map_err(|e| blob_file_error("set_buffer_size", e))?;
}
}
Ok(handles
.into_iter()
.map(|handle| {
handle.map(|file| PyBlobFile {
inner: Arc::new(file),
closed: AtomicBool::new(false),
})
})
.collect::<Vec<_>>())
})
}
#[pyo3(signature = (descriptor, buffer_size=None))]
pub fn open_blob_file<'py>(
self_: PyRef<'py, Self>,
descriptor: &str,
buffer_size: Option<usize>,
) -> PyResult<Bound<'py, PyAny>> {
let descriptor: BlobDescriptor = serde_json::from_str(descriptor)
.map_err(|error| PyValueError::new_err(format!("invalid blob descriptor: {error}")))?;
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let file = inner.open_blob_file(&descriptor).await.infer_error()?;
if let Some(size) = buffer_size {
file.set_buffer_size(size)
.await
.map_err(|e| blob_file_error("set_buffer_size", e))?;
}
Ok(PyBlobFile {
inner: Arc::new(file),
closed: AtomicBool::new(false),
})
})
}
/// Optimize the on-disk data by compacting and pruning old data, for better performance.
#[pyo3(signature = (cleanup_since_ms=None, delete_unverified=None, compaction_options=None))]
pub fn optimize<'py>(
self_: PyRef<'py, Self>,
cleanup_since_ms: Option<u64>,
delete_unverified: Option<bool>,
compaction_options: Option<&Bound<'py, PyDict>>,
) -> PyResult<Bound<'py, PyAny>> {
let inner = self_.inner_ref()?.clone();
let target_specified = compaction_options
.map(|options| options.contains("target_rows_per_fragment"))
.transpose()?
.unwrap_or(false);
let compaction_options = parse_compaction_options(compaction_options)?;
let older_than = if let Some(ms) = cleanup_since_ms {
if ms > i64::MAX as u64 {
return Err(PyValueError::new_err(format!(
"cleanup_since_ms must be between {} and -{}",
i32::MAX,
i32::MAX
)));
}
Duration::try_milliseconds(ms as i64)
} else {
None
};
future_into_py(self_.py(), async move {
let stats = if inner.as_native().is_some() {
inner
.optimize_full(compaction_options, older_than, None, delete_unverified)
.await
} else {
let mut overrides =
lancedb::table::optimize::CompactionOverrides::try_from(compaction_options)
.infer_error()?;
if !target_specified {
overrides.target_rows_per_fragment = None;
}
inner
.optimize_full_with_overrides(overrides, older_than, None, delete_unverified)
.await
}
.infer_error()?;
let compaction_stats = stats.compaction.unwrap();
let prune_stats = stats.prune.unwrap();
Ok(OptimizeStats {
compaction: CompactionStats {
files_added: compaction_stats.files_added as u64,
files_removed: compaction_stats.files_removed as u64,
fragments_added: compaction_stats.fragments_added as u64,
fragments_removed: compaction_stats.fragments_removed as u64,
},
prune: RemovalStats {
bytes_removed: prune_stats.bytes_removed,
old_versions_removed: prune_stats.old_versions,
},
})
})
}
pub fn execute_merge_insert<'a>(
self_: PyRef<'a, Self>,
data: Bound<'a, PyAny>,
parameters: MergeInsertParams,
) -> PyResult<Bound<'a, PyAny>> {
let batches: ArrowArrayStreamReader = ArrowArrayStreamReader::from_pyarrow_bound(&data)?;
let on = parameters.on.iter().map(|s| s.as_str()).collect::<Vec<_>>();
let mut builder = self_.inner_ref()?.merge_insert(&on);
if parameters.when_matched_update_all {
builder.when_matched_update_all(parameters.when_matched_update_all_condition);
}
if parameters.when_not_matched_insert_all {
builder.when_not_matched_insert_all();
}
if parameters.when_not_matched_by_source_delete {
if let Some(e) = parameters.when_not_matched_by_source_condition_expr {
builder.when_not_matched_by_source_delete_expr(e.0);
} else {
builder.when_not_matched_by_source_delete(
parameters.when_not_matched_by_source_condition,
);
}
}
if let Some(timeout) = parameters.timeout {
builder.timeout(timeout);
}
if let Some(use_index) = parameters.use_index {
builder.use_index(use_index);
}
if let Some(use_rte) = parameters.use_rte {
builder.use_rte(use_rte);
}
if let Some(validate_single_shard) = parameters.validate_single_shard {
builder.validate_single_shard(validate_single_shard);
}
future_into_py(self_.py(), async move {
let res = builder.execute(Box::new(batches)).await.infer_error()?;
Ok(MergeResult::from(res))
})
}
pub fn set_unenforced_primary_key<'a>(
self_: PyRef<'a, Self>,
columns: Vec<String>,
) -> PyResult<Bound<'a, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner
.set_unenforced_primary_key(columns)
.await
.infer_error()
})
}
pub fn enable_rte<'a>(
self_: PyRef<'a, Self>,
spec: RTEWriteSpec,
) -> PyResult<Bound<'a, PyAny>> {
let inner = self_.inner_ref()?.clone();
let native_spec = lancedb::table::RTEWriteSpec::from(spec);
future_into_py(self_.py(), async move {
inner.enable_rte(native_spec).await.infer_error()
})
}
pub fn disable_rte(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.disable_rte().await.infer_error()
})
}
pub fn get_rte_spec(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let spec = inner.get_rte_spec().await.infer_error()?;
Ok(spec.map(RTEWriteSpec::from))
})
}
pub fn get_table_overrides(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let overrides = inner.get_table_overrides().await.infer_error()?;
serde_json::to_string(&overrides).map_err(|err| {
PyRuntimeError::new_err(format!("Failed to serialize table overrides: {err}"))
})
})
}
pub fn update_table_overrides<'a>(
self_: PyRef<'a, Self>,
overrides_json: String,
) -> PyResult<Bound<'a, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let overrides = serde_json::from_str(&overrides_json).map_err(|err| {
PyValueError::new_err(format!("Invalid table overrides JSON: {err}"))
})?;
let overrides = inner
.update_table_overrides(overrides)
.await
.infer_error()?;
serde_json::to_string(&overrides).map_err(|err| {
PyRuntimeError::new_err(format!("Failed to serialize table overrides: {err}"))
})
})
}
pub fn reset_table_overrides(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let overrides = inner.reset_table_overrides().await.infer_error()?;
serde_json::to_string(&overrides).map_err(|err| {
PyRuntimeError::new_err(format!("Failed to serialize table overrides: {err}"))
})
})
}
/// Whether a hybrid query on this table has already been refused `_rowid`.
/// Learned from a previous refusal, never probed, so this is free.
///
/// WAL-PK-FUSION: delete this and `note_hybrid_pk_fusion`.
pub fn hybrid_pk_fusion_learned(self_: PyRef<'_, Self>) -> PyResult<bool> {
Ok(self_.inner_ref()?.base_table().hybrid_pk_fusion_learned())
}
/// Remember that this table refused `_rowid`, so later hybrid queries skip
/// straight to the primary-key fusion.
pub fn note_hybrid_pk_fusion(self_: PyRef<'_, Self>) -> PyResult<()> {
self_.inner_ref()?.base_table().note_hybrid_pk_fusion();
Ok(())
}
/// Converge the table's RTE write path into its base table.
///
/// Best-effort: with writes flowing, new rows may land after the last
/// pass. Errors if the table stops making progress.
pub fn checkpoint_rte(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.checkpoint_rte().await.infer_error()
})
}
/// Freeze every table shard's active memtable into an SSTable.
pub fn flush_rte(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(
self_.py(),
async move { inner.flush_rte().await.infer_error() },
)
}
/// Trigger a background SSTable compaction pass per table shard. Returns once the
/// passes are dispatched, not once they finish — watch `get_rte_stats`.
pub fn compact_rte(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.compact_rte().await.infer_error()
})
}
/// Live RTE state, or `None` when the RTE write path is not enabled.
#[pyo3(signature = (include_sstable_rows=false))]
pub fn get_rte_stats(
self_: PyRef<'_, Self>,
include_sstable_rows: bool,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let stats = inner
.get_rte_stats(include_sstable_rows)
.await
.infer_error()?;
Python::attach(|py| stats.map(|s| rte_stats_to_py(py, &s)).transpose())
})
}
pub fn close_rte_writers(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.close_rte_writers().await.infer_error()
})
}
pub fn uses_v2_manifest_paths(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner
.as_native()
.ok_or_else(|| lancedb::Error::NotSupported {
message: "uses_v2_manifest_paths is not supported in LanceDB Enterprise."
.into(),
})
.infer_error()?
.uses_v2_manifest_paths()
.await
.infer_error()
})
}
pub fn migrate_manifest_paths_v2(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner
.as_native()
.ok_or_else(|| lancedb::Error::NotSupported {
message: "migrate_manifest_paths_v2 is not supported in LanceDB Enterprise."
.into(),
})
.infer_error()?
.migrate_manifest_paths_v2()
.await
.infer_error()
})
}
pub fn add_columns(
self_: PyRef<'_, Self>,
definitions: Vec<(String, String)>,
) -> PyResult<Bound<'_, PyAny>> {
let definitions = NewColumnTransform::SqlExpressions(definitions);
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let result = inner
.add_columns()
.transform(definitions)
.execute()
.await
.infer_error()?;
Ok(AddColumnsResult::from(result))
})
}
pub fn add_columns_from_reader<'a>(
self_: PyRef<'a, Self>,
data: Bound<'a, PyAny>,
) -> PyResult<Bound<'a, PyAny>> {
let reader = ArrowArrayStreamReader::from_pyarrow_bound(&data)?;
let schema = reader.schema();
// Lance's surplus-row check treats trailing empty batches as extra data.
let reader = RecordBatchIterator::new(
reader.filter(|batch| !batch.as_ref().is_ok_and(|batch| batch.num_rows() == 0)),
schema,
);
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let result = inner
.add_columns()
.transform(NewColumnTransform::Reader(Box::new(reader)))
.execute()
.await
.infer_error()?;
Ok(AddColumnsResult::from(result))
})
}
#[pyo3(signature = (columns, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, auto_refresh_resource_budget=None))]
pub fn add_computed_columns<'py>(
self_: PyRef<'py, Self>,
columns: Vec<(String, String)>,
auto_refresh_enabled: Option<bool>,
auto_refresh_interval_seconds: Option<u64>,
auto_refresh_resource_budget: Option<Bound<'py, PyDict>>,
) -> PyResult<Bound<'py, PyAny>> {
let inner = self_.inner_ref()?.clone();
let options = ComputedColumnOptions {
auto_refresh_enabled: auto_refresh_enabled.unwrap_or(true),
auto_refresh_interval_seconds,
auto_refresh_resource_budget: parse_resource_budget(
auto_refresh_resource_budget.as_ref(),
)?,
};
options.validate().infer_error()?;
future_into_py(self_.py(), async move {
let mut builder = inner.add_columns();
for (name, expression) in columns {
builder = builder.computed(name, expression);
}
let result = builder
.computed_options(options)
.execute()
.await
.infer_error()?;
Ok(AddColumnsResult::from(result))
})
}
#[pyo3(signature = (application_json, output_name, auto_refresh_enabled=None, auto_refresh_interval_seconds=None, auto_refresh_resource_budget=None))]
pub fn add_function_columns<'py>(
self_: PyRef<'py, Self>,
application_json: String,
output_name: Option<String>,
auto_refresh_enabled: Option<bool>,
auto_refresh_interval_seconds: Option<u64>,
auto_refresh_resource_budget: Option<Bound<'py, PyDict>>,
) -> PyResult<Bound<'py, PyAny>> {
let application =
lancedb::function::FunctionApplication::from_json(&application_json).infer_error()?;
let inner = self_.inner_ref()?.clone();
let options = ComputedColumnOptions {
auto_refresh_enabled: auto_refresh_enabled.unwrap_or(true),
auto_refresh_interval_seconds,
auto_refresh_resource_budget: parse_resource_budget(
auto_refresh_resource_budget.as_ref(),
)?,
};
options.validate().infer_error()?;
future_into_py(self_.py(), async move {
let builder = match output_name {
Some(name) => inner.add_columns().function_as(name, application),
None => inner.add_columns().function(application),
};
let result = builder
.computed_options(options)
.execute()
.await
.infer_error()?;
Ok(AddColumnsResult::from(result))
})
}
#[pyo3(signature = (column, *, r#where=None, batch_size=None, max_concurrency=None, publication_interval_rows=None, resource_budget=None, supervisor_resources=None, planning_workers=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None, max_failed_fraction=None, max_failed_rows=None, row_timeout_seconds=None, init_timeout_seconds=None, num_workers=None, max_executor_processes_per_worker=None))]
#[allow(clippy::too_many_arguments)]
pub fn refresh_column<'py>(
self_: PyRef<'py, Self>,
column: String,
r#where: Option<String>,
batch_size: Option<i64>,
max_concurrency: Option<Bound<'py, PyAny>>,
publication_interval_rows: Option<i64>,
resource_budget: Option<Bound<'py, PyDict>>,
supervisor_resources: Option<Bound<'py, PyDict>>,
planning_workers: Option<Bound<'py, PyDict>>,
gpu_class: Option<String>,
gpu_count: Option<i64>,
retry_policy: Option<String>,
error_policy: Option<String>,
max_failed_fraction: Option<f64>,
max_failed_rows: Option<i64>,
row_timeout_seconds: Option<f64>,
init_timeout_seconds: Option<f64>,
num_workers: Option<i64>,
max_executor_processes_per_worker: Option<i64>,
) -> PyResult<Bound<'py, PyAny>> {
let options = refresh_column_options(RefreshColumnArgs {
where_clause: r#where,
batch_size,
max_concurrency: positive_u32_integer(max_concurrency.as_ref(), "max_concurrency")?,
publication_interval_rows,
resource_budget: parse_resource_budget(resource_budget.as_ref())?,
supervisor_resources: parse_supervisor_resources(supervisor_resources.as_ref())?,
planning_workers: parse_planning_workers(planning_workers.as_ref())?,
gpu_class,
gpu_count,
retry_policy,
error_policy,
max_failed_fraction,
max_failed_rows,
row_timeout_seconds,
init_timeout_seconds,
num_workers,
max_executor_processes_per_worker,
})?;
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let result = inner
.refresh_column_with_options(column, options)
.await
.infer_error()?;
Ok(RefreshColumnResult::from(result))
})
}
#[pyo3(signature = (column, *, r#where=None, batch_size=None, max_concurrency=None, publication_interval_rows=None, resource_budget=None, supervisor_resources=None, planning_workers=None, gpu_class=None, gpu_count=None, retry_policy=None, error_policy=None, max_failed_fraction=None, max_failed_rows=None, row_timeout_seconds=None, init_timeout_seconds=None, num_workers=None, max_executor_processes_per_worker=None))]
#[allow(clippy::too_many_arguments)]
pub fn refresh_column_async<'py>(
self_: PyRef<'py, Self>,
column: String,
r#where: Option<String>,
batch_size: Option<i64>,
max_concurrency: Option<Bound<'py, PyAny>>,
publication_interval_rows: Option<i64>,
resource_budget: Option<Bound<'py, PyDict>>,
supervisor_resources: Option<Bound<'py, PyDict>>,
planning_workers: Option<Bound<'py, PyDict>>,
gpu_class: Option<String>,
gpu_count: Option<i64>,
retry_policy: Option<String>,
error_policy: Option<String>,
max_failed_fraction: Option<f64>,
max_failed_rows: Option<i64>,
row_timeout_seconds: Option<f64>,
init_timeout_seconds: Option<f64>,
num_workers: Option<i64>,
max_executor_processes_per_worker: Option<i64>,
) -> PyResult<Bound<'py, PyAny>> {
let options = refresh_column_options(RefreshColumnArgs {
where_clause: r#where,
batch_size,
max_concurrency: positive_u32_integer(max_concurrency.as_ref(), "max_concurrency")?,
publication_interval_rows,
resource_budget: parse_resource_budget(resource_budget.as_ref())?,
supervisor_resources: parse_supervisor_resources(supervisor_resources.as_ref())?,
planning_workers: parse_planning_workers(planning_workers.as_ref())?,
gpu_class,
gpu_count,
retry_policy,
error_policy,
max_failed_fraction,
max_failed_rows,
row_timeout_seconds,
init_timeout_seconds,
num_workers,
max_executor_processes_per_worker,
})?;
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let job = inner
.refresh_column_async_with_options(column, options)
.await
.infer_error()?;
Ok(crate::job::Job::new_typed(job))
})
}
#[pyo3(signature = (job_id=None, column=None, limit=None))]
pub fn function_errors(
self_: PyRef<'_, Self>,
job_id: Option<String>,
column: Option<String>,
limit: Option<usize>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
let request = lancedb::function::FunctionErrorsRequest {
job_id,
column,
limit,
};
future_into_py(self_.py(), async move {
let errors = inner.function_errors(request).await.infer_error()?;
Ok(FunctionErrors::from(errors))
})
}
#[pyo3(signature = (column=None))]
pub fn column_statuses(
self_: PyRef<'_, Self>,
column: Option<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let statuses = inner
.function_column_statuses(column.as_deref())
.await
.infer_error()?;
Ok(statuses
.into_iter()
.map(FunctionColumnStatus::from)
.collect::<Vec<_>>())
})
}
pub fn column_status(self_: PyRef<'_, Self>, column: String) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let status = inner.function_column_status(&column).await.infer_error()?;
Ok(FunctionColumnStatus::from(status))
})
}
#[pyo3(signature = (column, timeout=None))]
pub fn wait_column_fresh<'py>(
self_: PyRef<'py, Self>,
column: String,
timeout: Option<Bound<'py, PyAny>>,
) -> PyResult<Bound<'py, PyAny>> {
let timeout = timeout
.map(|timeout| timeout.extract::<std::time::Duration>())
.transpose()?;
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let status = inner
.wait_for_function_column(&column, timeout)
.await
.infer_error()?;
Ok(FunctionColumnStatus::from(status))
})
}
#[pyo3(signature = (source_version=None, *, full=false, cpu_millis=None, memory_bytes=None, gpu_count=None, gpu_class=None))]
#[allow(clippy::too_many_arguments)]
pub fn refresh_materialized_view(
self_: PyRef<'_, Self>,
source_version: Option<u64>,
full: bool,
cpu_millis: Option<u64>,
memory_bytes: Option<u64>,
gpu_count: Option<u32>,
gpu_class: Option<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let view = lancedb::MaterializedView::from_table(inner)
.await
.infer_error()?;
let mut builder = view.refresh().full(full);
if let Some(value) = cpu_millis {
builder = builder.cpu_millis(value);
}
if let Some(value) = memory_bytes {
builder = builder.memory_bytes(value);
}
if let Some(value) = gpu_count {
builder = builder.gpu_count(value);
}
if let Some(value) = gpu_class {
builder = builder.gpu_class(value);
}
if let Some(version) = source_version {
builder = builder.source_version(version);
}
let result = builder.execute().await.infer_error()?;
Ok(RefreshMaterializedViewResult::from(result))
})
}
#[pyo3(signature = (source_version=None, *, full=false, cpu_millis=None, memory_bytes=None, gpu_count=None, gpu_class=None))]
#[allow(clippy::too_many_arguments)]
pub fn refresh_materialized_view_async(
self_: PyRef<'_, Self>,
source_version: Option<u64>,
full: bool,
cpu_millis: Option<u64>,
memory_bytes: Option<u64>,
gpu_count: Option<u32>,
gpu_class: Option<String>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let view = lancedb::MaterializedView::from_table(inner)
.await
.infer_error()?;
let mut builder = view.refresh().full(full);
if let Some(value) = cpu_millis {
builder = builder.cpu_millis(value);
}
if let Some(value) = memory_bytes {
builder = builder.memory_bytes(value);
}
if let Some(value) = gpu_count {
builder = builder.gpu_count(value);
}
if let Some(value) = gpu_class {
builder = builder.gpu_class(value);
}
if let Some(version) = source_version {
builder = builder.source_version(version);
}
let job = builder.execute_async().await.infer_error()?;
Ok(crate::job::Job::new_typed(job))
})
}
pub fn materialized_view_definition(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let view = lancedb::MaterializedView::from_table(inner)
.await
.infer_error()?;
Ok(lancedb::materialized_view::definition_metadata_from_sql(
view.definition_sql(),
))
})
}
pub fn add_columns_with_schema(
self_: PyRef<'_, Self>,
schema: PyArrowType<Schema>,
) -> PyResult<Bound<'_, PyAny>> {
let arrow_schema = &schema.0;
let transform = NewColumnTransform::AllNulls(Arc::new(arrow_schema.clone()));
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let result = inner
.add_columns()
.transform(transform)
.execute()
.await
.infer_error()?;
Ok(AddColumnsResult::from(result))
})
}
pub fn alter_columns<'a>(
self_: PyRef<'a, Self>,
alterations: Vec<Bound<PyDict>>,
) -> PyResult<Bound<'a, PyAny>> {
let alterations = alterations
.iter()
.map(|alteration| {
for key in alteration.keys().iter() {
let key = key.extract::<String>()?;
if !matches!(
key.as_str(),
"path" | "rename" | "name" | "nullable" | "data_type"
) {
return Err(PyValueError::new_err(format!(
"Unknown column alteration key '{key}'"
)));
}
}
let path = alteration
.get_item("path")?
.ok_or_else(|| PyValueError::new_err("Missing path"))?
.extract()?;
let rename = {
// We prefer rename, but support name for backwards compatibility
let rename = if let Ok(Some(rename)) = alteration.get_item("rename") {
Some(rename)
} else {
alteration.get_item("name")?
};
rename.map(|name| name.extract()).transpose()?
};
let nullable = alteration
.get_item("nullable")?
.map(|val| val.extract())
.transpose()?;
let data_type = alteration
.get_item("data_type")?
.map(|val| DataType::from_pyarrow_bound(&val))
.transpose()?;
Ok(ColumnAlteration {
path,
rename,
nullable,
data_type,
})
})
.collect::<PyResult<Vec<_>>>()?;
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let result = inner.alter_columns(&alterations).await.infer_error()?;
Ok(AlterColumnsResult::from(result))
})
}
pub fn drop_columns(self_: PyRef<Self>, columns: Vec<String>) -> PyResult<Bound<PyAny>> {
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let column_refs = columns.iter().map(String::as_str).collect::<Vec<&str>>();
let result = inner.drop_columns(&column_refs).await.infer_error()?;
Ok(DropColumnsResult::from(result))
})
}
pub fn replace_field_metadata<'a>(
self_: PyRef<'a, Self>,
field_name: String,
metadata: &Bound<'_, PyDict>,
) -> PyResult<Bound<'a, PyAny>> {
// Deprecated: forwards to the update_field_metadata path (replace mode).
let mut update = FieldMetadataUpdate::new(field_name).replace();
for (key, value) in metadata.into_iter() {
update = update.set(key.extract::<String>()?, value.extract::<String>()?);
}
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
inner.update_field_metadata(&[update]).await.infer_error()?;
Ok(())
})
}
pub fn update_field_metadata<'a>(
self_: PyRef<'a, Self>,
updates: Vec<Bound<PyDict>>,
) -> PyResult<Bound<'a, PyAny>> {
let updates = updates
.iter()
.map(|update| {
let path: String = update
.get_item("path")?
.ok_or_else(|| PyValueError::new_err("Missing path"))?
.extract()?;
let mut field_update = FieldMetadataUpdate::new(path);
if let Some(metadata) = update.get_item("metadata")? {
let metadata_dict = metadata.cast::<PyDict>()?;
for (key, value) in metadata_dict.iter() {
let key: String = key.extract()?;
if value.is_none() {
field_update = field_update.remove(key);
} else {
field_update = field_update.set(key, value.extract::<String>()?);
}
}
}
if let Some(replace) = update.get_item("replace")?
&& replace.extract::<bool>()?
{
field_update = field_update.replace();
}
Ok(field_update)
})
.collect::<PyResult<Vec<_>>>()?;
let inner = self_.inner_ref()?.clone();
future_into_py(self_.py(), async move {
let result = inner.update_field_metadata(&updates).await.infer_error()?;
Ok(UpdateFieldMetadataResult::from(result))
})
}
}
#[derive(FromPyObject)]
pub enum LanceVersion {
Version(u64),
Tag(String),
}
#[derive(FromPyObject)]
#[pyo3(from_item_all)]
pub struct MergeInsertParams {
on: Vec<String>,
when_matched_update_all: bool,
when_matched_update_all_condition: Option<String>,
when_not_matched_insert_all: bool,
when_not_matched_by_source_delete: bool,
when_not_matched_by_source_condition: Option<String>,
when_not_matched_by_source_condition_expr: Option<PyExpr>,
timeout: Option<std::time::Duration>,
use_index: Option<bool>,
use_rte: Option<bool>,
validate_single_shard: Option<bool>,
}
#[pyclass]
pub struct Tags {
inner: LanceDbTable,
}
impl Tags {
pub fn new(table: LanceDbTable) -> Self {
Self { inner: table }
}
}
#[pymethods]
impl Tags {
pub fn list(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let tags = inner.tags().await.infer_error()?;
let res = tags.list().await.infer_error()?;
Python::attach(|py| {
let py_dict = PyDict::new(py);
for (key, contents) in res {
let value_dict = PyDict::new(py);
value_dict.set_item("version", contents.version)?;
value_dict.set_item("manifest_size", contents.manifest_size)?;
py_dict.set_item(key, value_dict)?;
}
Ok(py_dict.unbind())
})
})
}
pub fn get_version(self_: PyRef<'_, Self>, tag: String) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let tags = inner.tags().await.infer_error()?;
let res = tags.get_version(tag.as_str()).await.infer_error()?;
Ok(res)
})
}
pub fn create(self_: PyRef<Self>, tag: String, version: u64) -> PyResult<Bound<PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let mut tags = inner.tags().await.infer_error()?;
tags.create(tag.as_str(), version).await.infer_error()?;
Ok(())
})
}
pub fn delete(self_: PyRef<Self>, tag: String) -> PyResult<Bound<PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let mut tags = inner.tags().await.infer_error()?;
tags.delete(tag.as_str()).await.infer_error()?;
Ok(())
})
}
pub fn update(self_: PyRef<Self>, tag: String, version: u64) -> PyResult<Bound<PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let mut tags = inner.tags().await.infer_error()?;
tags.update(tag.as_str(), version).await.infer_error()?;
Ok(())
})
}
}
#[pyclass]
pub struct Branches {
inner: LanceDbTable,
}
impl Branches {
pub fn new(table: LanceDbTable) -> Self {
Self { inner: table }
}
}
#[pymethods]
impl Branches {
pub fn list(self_: PyRef<'_, Self>) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let res = inner.list_branches().await.infer_error()?;
Python::attach(|py| {
let py_dict = PyDict::new(py);
for (name, contents) in res {
let value = PyDict::new(py);
value.set_item("parent_branch", contents.parent_branch)?;
value.set_item("parent_version", contents.parent_version)?;
value.set_item("manifest_size", contents.manifest_size)?;
py_dict.set_item(name, value)?;
}
Ok(py_dict.unbind())
})
})
}
#[pyo3(signature = (name, from_ref=None, from_version=None))]
pub fn create(
self_: PyRef<'_, Self>,
name: String,
from_ref: Option<String>,
from_version: Option<u64>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let from = Ref::Version(from_ref, from_version);
let table = inner.create_branch(&name, from).await.infer_error()?;
Ok(Table::new(table))
})
}
#[pyo3(signature = (name, version=None))]
pub fn checkout(
self_: PyRef<'_, Self>,
name: String,
version: Option<u64>,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let table = inner.checkout_branch(&name, version).await.infer_error()?;
Ok(Table::new(table))
})
}
pub fn delete(self_: PyRef<'_, Self>, name: String) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
inner.delete_branch(&name).await.infer_error()?;
Ok(())
})
}
pub fn diff(self_: PyRef<'_, Self>, from_branch: String) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let diff = inner.diff_branch(&from_branch).await.infer_error()?;
Python::attach(|py| struct_to_wire_py(py, &diff))
})
}
#[pyo3(signature = (from_branch, dry_run=false))]
pub fn cherry_pick(
self_: PyRef<'_, Self>,
from_branch: String,
dry_run: bool,
) -> PyResult<Bound<'_, PyAny>> {
let inner = self_.inner.clone();
future_into_py(self_.py(), async move {
let result = inner
.cherry_pick(&from_branch, dry_run)
.await
.infer_error()?;
Python::attach(|py| struct_to_wire_py(py, &result))
})
}
}
/// Decode a serde value as the wire JSON object (camelCase keys).
fn struct_to_wire_py(py: Python<'_>, value: &impl serde::Serialize) -> PyResult<Py<PyAny>> {
let json = py.import("json")?;
Ok(json
.call_method1(
"loads",
(serde_json::to_string(value)
.map_err(|e| PyRuntimeError::new_err(format!("failed to serialize json: {e}")))?,),
)?
.unbind())
}