Compare commits

...

2 Commits

Author SHA1 Message Date
ChilePiquin b5afa0bead fix: tolerate a transient TableNotFound in wait_for_index's list_indices poll
A read can race ahead of manifest/cache propagation and 404 on
list_indices() immediately after create_index(), even though the table was
just confirmed to exist by the caller. index_stats() already treats a 404 as
"not indexed yet" (Ok(None)); list_indices() had no equivalent tolerance, so
a single unlucky poll aborted the whole wait_timeout with a misleading
TableNotFound instead of continuing to poll until the real timeout.
2026-07-27 11:38:05 -07:00
lancedb automation 159cebcb90 chore: update lance dependency to v9.1.0-beta.8 2026-07-22 02:21:44 +00:00
6 changed files with 133 additions and 69 deletions
Generated
+43 -42
View File
@@ -3421,8 +3421,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4777,8 +4777,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arc-swap",
"arrow",
@@ -4852,8 +4852,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4875,7 +4875,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4889,7 +4889,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -4898,8 +4898,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrayref",
"crunchy",
@@ -4909,8 +4909,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4948,8 +4948,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow",
"arrow-array",
@@ -4979,8 +4979,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow",
"arrow-array",
@@ -4997,8 +4997,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"proc-macro2",
"quote",
@@ -5007,8 +5007,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5043,8 +5043,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5074,13 +5074,14 @@ dependencies = [
[[package]]
name = "lance-index"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arc-swap",
"arrow",
"arrow-arith",
"arrow-array",
"arrow-ipc",
"arrow-ord",
"arrow-schema",
"arrow-select",
@@ -5141,8 +5142,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5164,8 +5165,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow",
"arrow-arith",
@@ -5208,8 +5209,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5225,8 +5226,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow",
"async-trait",
@@ -5238,8 +5239,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5293,8 +5294,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5309,8 +5310,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow",
"arrow-array",
@@ -5349,8 +5350,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5363,8 +5364,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "9.1.0-beta.7"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.7#4b6a69a1279c62600bd00c858d2002f53c801bd4"
version = "9.1.0-beta.8"
source = "git+https://github.com/lance-format/lance.git?tag=v9.1.0-beta.8#5107a99e3f3912851c8cbb3822bd5b4cbf7fda2f"
dependencies = [
"icu_segmenter",
"jieba-rs",
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=9.1.0-beta.7", default-features = false, "tag" = "v9.1.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=9.1.0-beta.7", "tag" = "v9.1.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=9.1.0-beta.7", "tag" = "v9.1.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=9.1.0-beta.7", "tag" = "v9.1.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=9.1.0-beta.7", default-features = false, "tag" = "v9.1.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=9.1.0-beta.7", "tag" = "v9.1.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=9.1.0-beta.7", "tag" = "v9.1.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=9.1.0-beta.7", "tag" = "v9.1.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=9.1.0-beta.7", default-features = false, "tag" = "v9.1.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=9.1.0-beta.7", "tag" = "v9.1.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=9.1.0-beta.7", "tag" = "v9.1.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=9.1.0-beta.7", "tag" = "v9.1.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=9.1.0-beta.7", "tag" = "v9.1.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=9.1.0-beta.7", "tag" = "v9.1.0-beta.7", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=9.1.0-beta.8", default-features = false, "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=9.1.0-beta.8", default-features = false, "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=9.1.0-beta.8", default-features = false, "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=9.1.0-beta.8", "tag" = "v9.1.0-beta.8", "git" = "https://github.com/lance-format/lance.git" }
ahash = "0.8"
# Note that this one does not include pyarrow
arrow = { version = "58.0.0", optional = false }
+1 -1
View File
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>9.1.0-beta.7</lance-core.version>
<lance-core.version>9.1.0-beta.8</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+14 -1
View File
@@ -29,7 +29,20 @@ pub async fn wait_for_index(
// poll via list_indices() and index_stats() until all indices are created and fully indexed
while start.elapsed() < timeout {
let mut completed = vec![];
let indices = table.list_indices().await?;
// A transient TableNotFound here (e.g. a replica read racing ahead of
// manifest propagation) looks identical to a real missing table, but
// the table was just confirmed to exist by the caller. Treat it the
// same as index_stats()'s "not found yet" case and keep polling
// instead of failing the whole wait on a single bad read.
let indices = match table.list_indices().await {
Ok(indices) => indices,
Err(Error::TableNotFound { .. }) => {
debug!("list_indices returned not found; still waiting for index");
sleep(Duration::from_millis(DEFAULT_SLEEP_MS)).await;
continue;
}
Err(e) => return Err(e),
};
for &idx in &remaining {
if !indices.iter().any(|i| i.name == *idx) {
+61
View File
@@ -5791,6 +5791,67 @@ mod tests {
);
}
#[tokio::test]
async fn test_wait_for_index_tolerates_transient_list_indices_not_found() {
// A replica read can race ahead of manifest propagation and 404 on
// list_indices() even though the table (just confirmed by the caller)
// and its index both exist. wait_for_index() should treat that the
// same as index_stats()'s "not found yet" case and keep polling
// instead of failing the whole wait on a single bad read.
let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let table = Table::new_with_handler("my_table", move |request| {
let response_body = match request.url().path() {
"/v1/table/my_table/describe/" => {
let schema = Schema::new(vec![Field::new(
"vector",
DataType::FixedSizeList(
Arc::new(Field::new("item", DataType::Float32, true)),
8,
),
false,
)]);
serde_json::from_str::<serde_json::Value>(&describe_response(&schema)).unwrap()
}
"/v1/table/my_table/index/list/" => {
let call = calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
if call == 0 {
return http::Response::builder()
.status(404)
.body("{}".to_string())
.unwrap();
}
serde_json::json!({
"indexes": [
{
"index_name": "vector_idx",
"index_uuid": "3fa85f64-5717-4562-b3fc-2c963f66afa6",
"columns": ["vector"],
"index_status": "done",
},
]
})
}
"/v1/table/my_table/index/vector_idx/stats/" => {
serde_json::json!({
"num_indexed_rows": 100000,
"num_unindexed_rows": 0,
"index_type": "IVF_PQ",
"distance_type": "l2"
})
}
_path => serde_json::json!(None::<String>),
};
let body = serde_json::to_string(&response_body).unwrap();
let status = if body == "null" { 404 } else { 200 };
http::Response::builder().status(status).body(body).unwrap()
});
table
.wait_for_index(&["vector_idx"], Duration::from_secs(5))
.await
.unwrap();
}
fn _make_table_with_indices(unindexed_rows: usize) -> Table {
Table::new_with_handler("my_table", move |request| {
assert_eq!(request.method(), "POST");
-11
View File
@@ -774,9 +774,6 @@ fn shard_writer_config_from_defaults(defaults: &HashMap<String, String>) -> Shar
if let Some(v) = bool_of("durable_write") {
config = config.with_durable_write(v);
}
if let Some(v) = bool_of("sync_indexed_write") {
config = config.with_sync_indexed_write(v);
}
if let Some(v) = usize_of("max_wal_buffer_size") {
config = config.with_max_wal_buffer_size(v);
}
@@ -798,12 +795,6 @@ fn shard_writer_config_from_defaults(defaults: &HashMap<String, String>) -> Shar
if let Some(v) = millis_of("backpressure_log_interval_ms") {
config = config.with_backpressure_log_interval(v);
}
if let Some(v) = usize_of("async_index_buffer_rows") {
config = config.with_async_index_buffer_rows(v);
}
if let Some(v) = millis_of("async_index_interval_ms") {
config = config.with_async_index_interval(v);
}
if let Some(v) = bool_of("enable_memtable") {
config = config.with_enable_memtable(v);
}
@@ -1008,13 +999,11 @@ mod tests {
let defaults = HashMap::from([
("durable_write".to_string(), "false".to_string()),
("max_memtable_rows".to_string(), "4096".to_string()),
("async_index_interval_ms".to_string(), "250".to_string()),
("unknown_key".to_string(), "ignored".to_string()),
]);
let config = shard_writer_config_from_defaults(&defaults);
assert!(!config.durable_write);
assert_eq!(config.max_memtable_rows, 4096);
assert_eq!(config.async_index_interval, Duration::from_millis(250));
assert_eq!(config.shard_spec_id, SHARDING_SPEC_ID);
}