diff --git a/Cargo.lock b/Cargo.lock index c97a1fc5f..a4e95d624 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3765,8 +3765,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "fsst" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrow-array", "rand 0.9.5", @@ -5243,8 +5243,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a" [[package]] name = "lance" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arc-swap", "arrow", @@ -5318,8 +5318,8 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrow-array", "arrow-buffer", @@ -5341,7 +5341,7 @@ dependencies = [ [[package]] name = "lance-arrow-scalar" version = "59.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrow-array", "arrow-buffer", @@ -5355,7 +5355,7 @@ dependencies = [ [[package]] name = "lance-arrow-stats" version = "59.0.0" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrow-array", "arrow-schema", @@ -5364,8 +5364,8 @@ dependencies = [ [[package]] name = "lance-bitpacking" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrayref", "crunchy", @@ -5375,8 +5375,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrow-array", "arrow-buffer", @@ -5414,8 +5414,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrow", "arrow-array", @@ -5446,8 +5446,8 @@ dependencies = [ [[package]] name = "lance-datafusion-exec" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrow", "arrow-array", @@ -5472,8 +5472,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrow", "arrow-array", @@ -5490,8 +5490,8 @@ dependencies = [ [[package]] name = "lance-derive" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "proc-macro2", "quote", @@ -5500,8 +5500,8 @@ dependencies = [ [[package]] name = "lance-encoding" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrow-arith", "arrow-array", @@ -5534,8 +5534,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrow-arith", "arrow-array", @@ -5566,8 +5566,8 @@ dependencies = [ [[package]] name = "lance-geo" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "geo-traits", "geo-types", @@ -5579,8 +5579,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arc-swap", "arrow", @@ -5655,8 +5655,8 @@ dependencies = [ [[package]] name = "lance-index-core" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrow-array", "arrow-schema", @@ -5679,8 +5679,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrow", "arrow-array", @@ -5725,8 +5725,8 @@ dependencies = [ [[package]] name = "lance-jsonb" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "byteorder", "ethnum", @@ -5743,8 +5743,8 @@ dependencies = [ [[package]] name = "lance-linalg" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "argminmax 0.7.1", "arrow-array", @@ -5759,8 +5759,8 @@ dependencies = [ [[package]] name = "lance-namespace" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrow", "async-trait", @@ -5774,8 +5774,8 @@ dependencies = [ [[package]] name = "lance-namespace-impls" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arc-swap", "arrow", @@ -5829,8 +5829,8 @@ dependencies = [ [[package]] name = "lance-select" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrow-array", "arrow-buffer", @@ -5844,8 +5844,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrow", "arrow-array", @@ -5886,8 +5886,8 @@ dependencies = [ [[package]] name = "lance-testing" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "arrow-array", "arrow-schema", @@ -5900,8 +5900,8 @@ dependencies = [ [[package]] name = "lance-tokenizer" -version = "15.0.0-beta.10" -source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.10#f56a08d40b5630ce1124535c4c3b1406d5ceedf1" +version = "15.0.0-beta.11" +source = "git+https://github.com/lance-format/lance.git?tag=v15.0.0-beta.11#01052c56a308ed915a046ed42b0d39ce8c09c2d8" dependencies = [ "frostem", "icu_segmenter", diff --git a/Cargo.toml b/Cargo.toml index 8cd8d7292..3dfcf3ef3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,21 +19,21 @@ categories = ["database-implementations"] rust-version = "1.95.0" [workspace.dependencies] -lance = { "version" = "=15.0.0-beta.10", default-features = false, "tag" = "v15.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-core = { "version" = "=15.0.0-beta.10", "tag" = "v15.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-datagen = { "version" = "=15.0.0-beta.10", "tag" = "v15.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-file = { "version" = "=15.0.0-beta.10", "tag" = "v15.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-io = { "version" = "=15.0.0-beta.10", default-features = false, "tag" = "v15.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-index = { "version" = "=15.0.0-beta.10", "tag" = "v15.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-jsonb = { "version" = "=15.0.0-beta.10", "tag" = "v15.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-linalg = { "version" = "=15.0.0-beta.10", "tag" = "v15.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace = { "version" = "=15.0.0-beta.10", "tag" = "v15.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-namespace-impls = { "version" = "=15.0.0-beta.10", default-features = false, "tag" = "v15.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-table = { "version" = "=15.0.0-beta.10", "tag" = "v15.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-testing = { "version" = "=15.0.0-beta.10", "tag" = "v15.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-datafusion = { "version" = "=15.0.0-beta.10", "tag" = "v15.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-encoding = { "version" = "=15.0.0-beta.10", "tag" = "v15.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } -lance-arrow = { "version" = "=15.0.0-beta.10", "tag" = "v15.0.0-beta.10", "git" = "https://github.com/lance-format/lance.git" } +lance = { "version" = "=15.0.0-beta.11", default-features = false, "tag" = "v15.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-core = { "version" = "=15.0.0-beta.11", "tag" = "v15.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-datagen = { "version" = "=15.0.0-beta.11", "tag" = "v15.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-file = { "version" = "=15.0.0-beta.11", "tag" = "v15.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-io = { "version" = "=15.0.0-beta.11", default-features = false, "tag" = "v15.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-index = { "version" = "=15.0.0-beta.11", "tag" = "v15.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-jsonb = { "version" = "=15.0.0-beta.11", "tag" = "v15.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-linalg = { "version" = "=15.0.0-beta.11", "tag" = "v15.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace = { "version" = "=15.0.0-beta.11", "tag" = "v15.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-namespace-impls = { "version" = "=15.0.0-beta.11", default-features = false, "tag" = "v15.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-table = { "version" = "=15.0.0-beta.11", "tag" = "v15.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-testing = { "version" = "=15.0.0-beta.11", "tag" = "v15.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-datafusion = { "version" = "=15.0.0-beta.11", "tag" = "v15.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-encoding = { "version" = "=15.0.0-beta.11", "tag" = "v15.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } +lance-arrow = { "version" = "=15.0.0-beta.11", "tag" = "v15.0.0-beta.11", "git" = "https://github.com/lance-format/lance.git" } lancedb = { path = "rust/lancedb", default-features = false } ahash = "0.8" anyhow = "1" diff --git a/nodejs/__test__/query.test.ts b/nodejs/__test__/query.test.ts index 068e55c2f..c0b30ac63 100644 --- a/nodejs/__test__/query.test.ts +++ b/nodejs/__test__/query.test.ts @@ -564,7 +564,8 @@ describe("IVF search effort", () => { it.each([ [null, 6], [0, 1], - [0.25, 3], + [0.1, 2], + [0.25, 4], [0.5, 6], [0.75, 10], [1, 16], diff --git a/python/python/tests/test_query.py b/python/python/tests/test_query.py index d39f01a39..95148dcf9 100644 --- a/python/python/tests/test_query.py +++ b/python/python/tests/test_query.py @@ -2658,17 +2658,24 @@ def test_blob_v2_to_batches_row_id(tmp_db): assert [blobs[i].as_py() for i in range(len(blobs))] == [b"one", b"two"] -@pytest.mark.parametrize("metric", ["l2", "cosine", "dot"]) @pytest.mark.parametrize( - "effort,probes", [(None, 4), (0, 1), (0.25, 2), (0.5, 4), (0.75, 8), (1, 16)] + "metric,probes80,probes90", [("l2", 2, 3), ("cosine", 1, 2), ("dot", 1, 2)] ) -def test_search_effort_probe_budget(tmp_path, monkeypatch, metric, effort, probes): +@pytest.mark.parametrize("effort", [None, 0, 0.1, 0.25, 0.5, 0.75, 1]) +def test_search_effort_probe_budget( + tmp_path, monkeypatch, metric, effort, probes80, probes90 +): import re from lancedb.index import IvfFlat monkeypatch.setenv("LANCE_AUTO_PROBE_MARGIN", "0") monkeypatch.setenv("LANCE_AUTO_MIN_INITIAL_NPROBES", "4") monkeypatch.setenv("LANCE_AUTO_MAX_INITIAL_NPROBES", "4") + # The lower tiers use metric/k calibration; Auto and the upper tiers retain + # their existing budgets. Keep exact execution counts and precedence checks. + probes = {None: 4, 0: 1, 0.1: probes80, 0.25: probes90, 0.5: 4, 0.75: 8, 1: 16}[ + effort + ] rng = np.random.default_rng(42) vectors = np.repeat(np.eye(16, dtype=np.float32), 32, axis=0) vectors += rng.normal(0, 0.001, vectors.shape).astype(np.float32) @@ -2752,7 +2759,7 @@ def test_search_effort_invalid_sync(table, effort): @pytest.mark.asyncio @pytest.mark.parametrize("hybrid", [False, True]) -@pytest.mark.parametrize("effort", [None, 0, 0.25, 0.5, 0.75, 1]) +@pytest.mark.parametrize("effort", [None, 0, 0.1, 0.25, 0.5, 0.75, 1]) async def test_search_effort_async_roundtrip(table_async, hybrid, effort): query = table_async.vector_search([0, 0]) if hybrid: