Compare commits

..
Author SHA1 Message Date
Lance Release d3077b7641 Bump version: 0.39.0-beta.5 → 0.39.0-beta.6 2026-09-08 12:14:12 +00:00
43 changed files with 186 additions and 530 deletions
-20
View File
@@ -1,20 +0,0 @@
name: Typo checker
on:
push:
branches:
- main
pull_request:
permissions:
contents: read
jobs:
run:
name: Spell Check with Typos
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v6
- name: Check spelling of the entire repository
uses: crate-ci/typos@6802cc60d4e7f78b9d5454f6cf3935c042d5e1e3 # v1.26.0
-4
View File
@@ -10,10 +10,6 @@ repos:
rev: v0.9.9
hooks:
- id: ruff
- repo: https://github.com/crate-ci/typos
rev: v1.26.0
hooks:
- id: typos
# - repo: https://github.com/RobertCraigie/pyright-python
# rev: v1.1.395
# hooks:
-19
View File
@@ -1,19 +0,0 @@
[default]
extend-ignore-re = ["(?Rm)^.*(#|//)\\s*spellchecker:disable-line$"]
[default.extend-words]
# Azure Kubernetes Service, mentioned in rust/lancedb/src/remote/oauth.rs.
AKS = "AKS"
# RabitQ is the name of a vector quantization algorithm, not a typo of "Rabbit".
Rabit = "Rabit"
# `VarBuilder::from_mmaped_safetensors` is the real (if oddly-spelled) name of
# the candle-core API we call in rust/lancedb/src/embeddings/sentence_transformers.rs.
mmaped = "mmaped"
# `WriteableBuffer` is the real name of a type from Python's `_typeshed` stubs,
# used in python/python/lancedb/_blob.py.
Writeable = "Writeable"
[files]
extend-exclude = [
"*_THIRD_PARTY_LICENSES.*",
]
Generated
+45 -45
View File
@@ -3526,8 +3526,8 @@ checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
[[package]]
name = "fsst"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow-array",
"rand 0.9.5",
@@ -4886,8 +4886,8 @@ checksum = "e037a2e1d8d5fdbd49b16a4ea09d5d6401c1f29eca5ff29d03d3824dba16256a"
[[package]]
name = "lance"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arc-swap",
"arrow",
@@ -4959,8 +4959,8 @@ dependencies = [
[[package]]
name = "lance-arrow"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4982,7 +4982,7 @@ dependencies = [
[[package]]
name = "lance-arrow-scalar"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -4996,7 +4996,7 @@ dependencies = [
[[package]]
name = "lance-arrow-stats"
version = "58.0.0"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5005,8 +5005,8 @@ dependencies = [
[[package]]
name = "lance-bitpacking"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrayref",
"crunchy",
@@ -5016,8 +5016,8 @@ dependencies = [
[[package]]
name = "lance-core"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5054,8 +5054,8 @@ dependencies = [
[[package]]
name = "lance-datafusion"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow",
"arrow-array",
@@ -5085,8 +5085,8 @@ dependencies = [
[[package]]
name = "lance-datagen"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow",
"arrow-array",
@@ -5103,8 +5103,8 @@ dependencies = [
[[package]]
name = "lance-derive"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"proc-macro2",
"quote",
@@ -5113,8 +5113,8 @@ dependencies = [
[[package]]
name = "lance-encoding"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5147,8 +5147,8 @@ dependencies = [
[[package]]
name = "lance-file"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -5179,8 +5179,8 @@ dependencies = [
[[package]]
name = "lance-index"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arc-swap",
"arrow",
@@ -5244,8 +5244,8 @@ dependencies = [
[[package]]
name = "lance-index-core"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5267,8 +5267,8 @@ dependencies = [
[[package]]
name = "lance-io"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow",
"arrow-array",
@@ -5308,8 +5308,8 @@ dependencies = [
[[package]]
name = "lance-linalg"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5323,8 +5323,8 @@ dependencies = [
[[package]]
name = "lance-namespace"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow",
"async-trait",
@@ -5338,8 +5338,8 @@ dependencies = [
[[package]]
name = "lance-namespace-impls"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow",
"arrow-ipc",
@@ -5392,8 +5392,8 @@ dependencies = [
[[package]]
name = "lance-select"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5407,8 +5407,8 @@ dependencies = [
[[package]]
name = "lance-table"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow",
"arrow-array",
@@ -5448,8 +5448,8 @@ dependencies = [
[[package]]
name = "lance-testing"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"arrow-array",
"arrow-schema",
@@ -5462,8 +5462,8 @@ dependencies = [
[[package]]
name = "lance-tokenizer"
version = "12.0.0-beta.16"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.16#f7df098f5860cf9b3ac5339e7a641430cfbc6351"
version = "12.0.0-beta.15"
source = "git+https://github.com/lance-format/lance.git?tag=v12.0.0-beta.15#271a155ba505fd8e1094c095d4ce356707e93866"
dependencies = [
"frostem",
"icu_segmenter",
@@ -5476,7 +5476,7 @@ dependencies = [
[[package]]
name = "lancedb"
version = "0.39.0-beta.6"
version = "0.39.0-beta.5"
dependencies = [
"ahash",
"anyhow",
@@ -5567,7 +5567,7 @@ dependencies = [
[[package]]
name = "lancedb-nodejs"
version = "0.39.0-beta.6"
version = "0.39.0-beta.5"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -5592,7 +5592,7 @@ dependencies = [
[[package]]
name = "lancedb-python"
version = "0.39.0-beta.6"
version = "0.39.0-beta.5"
dependencies = [
"arrow",
"async-trait",
+14 -14
View File
@@ -13,20 +13,20 @@ categories = ["database-implementations"]
rust-version = "1.91.0"
[workspace.dependencies]
lance = { "version" = "=12.0.0-beta.16", default-features = false, "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=12.0.0-beta.16", default-features = false, "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=12.0.0-beta.16", default-features = false, "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=12.0.0-beta.16", "tag" = "v12.0.0-beta.16", "git" = "https://github.com/lance-format/lance.git" }
lance = { "version" = "=12.0.0-beta.15", default-features = false, "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-core = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-datagen = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-file = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-io = { "version" = "=12.0.0-beta.15", default-features = false, "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-index = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-linalg = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-namespace-impls = { "version" = "=12.0.0-beta.15", default-features = false, "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-table = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-testing = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-datafusion = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-encoding = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lance-arrow = { "version" = "=12.0.0-beta.15", "tag" = "v12.0.0-beta.15", "git" = "https://github.com/lance-format/lance.git" }
lancedb = { path = "rust/lancedb", default-features = false }
ahash = "0.8"
# Note that this one does not include pyarrow
+1 -1
View File
@@ -155,7 +155,7 @@ paths:
vector:
type: FixedSizeList
description: |
The targeted vector to search for. Required.
The targetted vector to search for. Required.
vector_column:
type: string
description: |
+1 -1
View File
@@ -141,7 +141,7 @@ Currently this causes multiple copies of the row to be created
but that behavior is subject to change.
An optional condition may be specified. If it is, then only
matched rows that satisfy the condition will be updated. Any
matched rows that satisfy the condtion will be updated. Any
rows that do not satisfy the condition will be left as they
are. Failing to satisfy the condition does not cause a
"matched row" to become a "not matched" row.
+1 -1
View File
@@ -1266,7 +1266,7 @@ value is 0")
Note: if your condition is something like "some_id_column == 7" and
you are updating many rows (with different ids) then you will get
better performance with a single [`merge_insert`] call instead of
repeatedly calling this method.
repeatedly calilng this method.
##### Parameters
+1 -1
View File
@@ -118,7 +118,7 @@ Number of sub-vectors of PQ.
This value controls how much the vector is compressed during the quantization step.
The more sub vectors there are the less the vector is compressed. The default is
the dimension of the vector divided by 16. If the dimension is not evenly divisible
by 16 we use the dimension divided by 8.
by 16 we use the dimension divded by 8.
The above two cases are highly preferred. Having 8 or 16 values per subvector allows
us to use efficient SIMD instructions.
+1 -1
View File
@@ -16,7 +16,7 @@ optional config: Index;
Advanced index configuration
This option allows you to specify a specific index to create and also
This option allows you to specify a specfic index to create and also
allows you to pass in configuration for training the index.
See the static methods on Index for details on the various index types.
+1 -1
View File
@@ -112,7 +112,7 @@ Number of sub-vectors of PQ.
This value controls how much the vector is compressed during the quantization step.
The more sub vectors there are the less the vector is compressed. The default is
the dimension of the vector divided by 16. If the dimension is not evenly divisible
by 16 we use the dimension divided by 8.
by 16 we use the dimension divded by 8.
The above two cases are highly preferred. Having 8 or 16 values per subvector allows
us to use efficient SIMD instructions.
+1 -1
View File
@@ -28,7 +28,7 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<arrow.version>15.0.0</arrow.version>
<lance-core.version>12.0.0-beta.16</lance-core.version>
<lance-core.version>12.0.0-beta.15</lance-core.version>
<spotless.skip>false</spotless.skip>
<spotless.version>2.30.0</spotless.version>
<spotless.java.googlejavaformat.version>1.7</spotless.java.googlejavaformat.version>
+2 -2
View File
@@ -3252,7 +3252,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
const db = await connect(tmpDir.name);
const data = [
{ text: "fa", vector: [0.1, 0.2, 0.3] },
{ text: "fo", vector: [0.4, 0.5, 0.6] }, // spellchecker:disable-line
{ text: "fo", vector: [0.4, 0.5, 0.6] },
{ text: "fob", vector: [0.4, 0.5, 0.6] },
{ text: "focus", vector: [0.4, 0.5, 0.6] },
{ text: "foo", vector: [0.4, 0.5, 0.6] },
@@ -3277,7 +3277,7 @@ describe.each([arrow15, arrow16, arrow17, arrow18])(
const resultSet = new Set(fuzzyResults.map((r) => r.text));
expect(resultSet.has("foo")).toBe(true);
expect(resultSet.has("fob")).toBe(true);
expect(resultSet.has("fo")).toBe(true); // spellchecker:disable-line
expect(resultSet.has("fo")).toBe(true);
expect(resultSet.has("food")).toBe(true);
const prefixResults = await table
+2 -2
View File
@@ -600,7 +600,7 @@ function makeVector(
}
if (values.length === 0) {
throw Error(
"makeVector requires at least one value or the type must be specified",
"makeVector requires at least one value or the type must be specfied",
);
}
const sampleValue = values.find((val) => val !== null && val !== undefined);
@@ -858,7 +858,7 @@ async function applyEmbeddings<T>(
* customized by the `embeddingDataType` property of the embedding function.
*
* If a schema is provided in `makeTableOptions` then it should include the
* embedding columns. If no schema is provided then embedding columns will
* embedding columns. If no schema is provded then embedding columns will
* be placed at the end of the table, after all of the input columns.
*/
export async function convertToTable(
+3 -3
View File
@@ -26,7 +26,7 @@ export interface IvfPqOptions {
* This value controls how much the vector is compressed during the quantization step.
* The more sub vectors there are the less the vector is compressed. The default is
* the dimension of the vector divided by 16. If the dimension is not evenly divisible
* by 16 we use the dimension divided by 8.
* by 16 we use the dimension divded by 8.
*
* The above two cases are highly preferred. Having 8 or 16 values per subvector allows
* us to use efficient SIMD instructions.
@@ -228,7 +228,7 @@ export interface HnswPqOptions {
* This value controls how much the vector is compressed during the quantization step.
* The more sub vectors there are the less the vector is compressed. The default is
* the dimension of the vector divided by 16. If the dimension is not evenly divisible
* by 16 we use the dimension divided by 8.
* by 16 we use the dimension divded by 8.
*
* The above two cases are highly preferred. Having 8 or 16 values per subvector allows
* us to use efficient SIMD instructions.
@@ -825,7 +825,7 @@ export interface IndexOptions {
/**
* Advanced index configuration
*
* This option allows you to specify a specific index to create and also
* This option allows you to specify a specfic index to create and also
* allows you to pass in configuration for training the index.
*
* See the static methods on Index for details on the various index types.
+1 -1
View File
@@ -27,7 +27,7 @@ export class MergeInsertBuilder {
* but that behavior is subject to change.
*
* An optional condition may be specified. If it is, then only
* matched rows that satisfy the condition will be updated. Any
* matched rows that satisfy the condtion will be updated. Any
* rows that do not satisfy the condition will be left as they
* are. Failing to satisfy the condition does not cause a
* "matched row" to become a "not matched" row.
+1 -1
View File
@@ -3,7 +3,7 @@
// The utilities in this file help sanitize data from the user's arrow
// library into the types expected by vectordb's arrow library. Node
// generally allows for multiple versions of the same library (and sometimes
// generally allows for mulitple versions of the same library (and sometimes
// even multiple copies of the same version) to be installed at the same
// time. However, arrow-js uses instanceof which expected that the input
// comes from the exact same library instance. This is not always the case
+1 -1
View File
@@ -313,7 +313,7 @@ export abstract class Table {
* Note: if your condition is something like "some_id_column == 7" and
* you are updating many rows (with different ids) then you will get
* better performance with a single [`merge_insert`] call instead of
* repeatedly calling this method.
* repeatedly calilng this method.
* @param {Map<string, string> | Record<string, string>} updates - the
* columns to update
* @returns {Promise<UpdateResult>} A promise that resolves to an object
+1 -1
View File
@@ -21,7 +21,7 @@ class GteEmbeddings(TextEmbeddingFunction):
An embedding function that uses GTE-LARGE MLX format(for Apple silicon devices only)
as well as the standard cpu/gpu version from: https://huggingface.co/thenlper/gte-large.
For Apple users, you will need the mlx package installed, which can be done with:
For Apple users, you will need the mlx package insalled, which can be done with:
pip install mlx
Parameters
@@ -60,7 +60,7 @@ class InstructorEmbeddingFunction(TextEmbeddingFunction):
import lancedb
from lancedb.pydantic import LanceModel, Vector
from lancedb.embeddings import get_registry, InstructorEmbeddingFunction
from lancedb.embeddings import get_registry, InstuctorEmbeddingFunction
instructor = get_registry().get("instructor").create(
source_instruction="represent the document for retrieval",
+1 -1
View File
@@ -751,7 +751,7 @@ class IvfPq:
This value controls how much the vector is compressed during the
quantization step. The more sub vectors there are the less the vector is
compressed. The default is the dimension of the vector divided by 16. If
the dimension is not evenly divisible by 16 we use the dimension divided by
the dimension is not evenly divisible by 16 we use the dimension divded by
8.
The above two cases are highly preferred. Having 8 or 16 values per
+17 -58
View File
@@ -78,10 +78,6 @@ if TYPE_CHECKING:
T = TypeVar("T", bound="LanceModel")
AnalyzePlanDistributedMetrics = Literal["aggregate", "per_worker", "full"]
# Number of rows a hybrid query returns when no limit was set on it. This
# mirrors the default the Rust query builder applies to its sub-queries.
DEFAULT_HYBRID_LIMIT = 10
@runtime_checkable
class _LanceScanner(Protocol):
@@ -863,7 +859,7 @@ class Query(pydantic.BaseModel):
return query
# This tells pydantic to allow custom types (needed for the `vector` query since
# pa.Array wouldn't be allowed otherwise)
# pa.Array wouln't be allowed otherwise)
model_config = pydantic.ConfigDict(arbitrary_types_allowed=True)
@@ -3897,54 +3893,14 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
return self
def _create_child_queries(
self,
) -> Tuple["AsyncFTSQuery", "AsyncVectorQuery", int, int]:
"""Build the sub-queries that make up this hybrid query.
Execution, `explain_plan` and `analyze_plan` all go through here so that
the plans that are reported are the plans that actually run.
Returns the two sub-queries along with the effective limit and offset of
the hybrid query itself.
"""
fts_query = AsyncFTSQuery(self._inner.to_fts_query(), self._table)
vec_query = AsyncVectorQuery(self._inner.to_vector_query(), self._table)
fts_req = fts_query._inner.to_query_request()
vec_req = vec_query._inner.to_query_request()
# Only one of the two sub-queries carries the limit when it was never
# set explicitly: nearest_to()/nearest_to_text() build the sibling query
# from scratch, and that is where the default gets filled in. Which one
# that is depends on the order the hybrid query was built in, so look at
# both rather than at a single side.
limit = fts_req.limit if fts_req.limit is not None else vec_req.limit
if limit is None:
limit = DEFAULT_HYBRID_LIMIT
offset = fts_req.offset or vec_req.offset or 0
fts_query.with_row_id()
vec_query.with_row_id()
# offset() pushes the offset down into both sub-queries, which would make
# each of them skip its own first `offset` rows. The window has to be
# taken out of the combined, reranked results instead, so fetch the
# skipped prefix here too and slice it off afterwards.
fts_query.limit(limit + offset)
vec_query.limit(limit + offset)
fts_query.offset(0)
vec_query.offset(0)
return fts_query, vec_query, limit, offset
async def to_batches(
self,
*,
max_batch_length: Optional[int] = None,
timeout: Optional[timedelta] = None,
) -> AsyncRecordBatchReader:
fts_query, vec_query, limit, offset = self._create_child_queries()
fts_query = AsyncFTSQuery(self._inner.to_fts_query(), self._table)
vec_query = AsyncVectorQuery(self._inner.to_vector_query(), self._table)
req = fts_query._inner.to_query_request()
blob_auto_row_id = False
@@ -3964,6 +3920,9 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
self._blob_auto_row_id = blob_auto_row_id
self._blob_paths = blob_paths
fts_query.with_row_id()
vec_query.with_row_id()
fts_results, vector_results = await asyncio.gather(
fts_query.to_arrow(timeout=timeout),
vec_query.to_arrow(timeout=timeout),
@@ -3975,9 +3934,8 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
norm=self._norm,
fts_query=fts_query.get_query(),
reranker=self._reranker,
limit=limit,
limit=self._inner.get_limit(),
with_row_ids=True,
offset=offset,
)
if (
not self._user_requested_row_id()
@@ -4006,14 +3964,14 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
... print(plan)
>>> asyncio.run(doctest_example()) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE
RRFReranker(K=60)
ProjectionExec: expr=[vector@0 as vector, text@3 as text, _distance@2 as _distance, _rowid@1 as _rowid]
ProjectionExec: expr=[vector@0 as vector, text@3 as text, _distance@2 as _distance]
LanceRead: uri=..., projection=[text], source=stream(_rowid)
GlobalLimitExec: skip=0, fetch=10
FilterExec: _distance@2 IS NOT NULL
SortExec: TopK(fetch=10), expr=[_distance@2 ASC NULLS LAST, _rowid@1 ASC NULLS LAST], preserve_partitioning=[false]
KNNVectorDistance: metric=l2
LanceRead: uri=..., projection=[vector], ...
ProjectionExec: expr=[vector@2 as vector, text@3 as text, _score@1 as _score, _rowid@0 as _rowid]
ProjectionExec: expr=[vector@2 as vector, text@3 as text, _score@1 as _score]
LanceRead: uri=..., projection=[vector, text], source=stream(_rowid)
GlobalLimitExec: skip=0, fetch=10
MatchQuery: column=text, query=[hello]
@@ -4028,9 +3986,8 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
plan : str
""" # noqa: E501
fts_query, vec_query, _, _ = self._create_child_queries()
vector_plan = await vec_query.explain_plan(verbose)
fts_plan = await fts_query.explain_plan(verbose)
vector_plan = await self._inner.to_vector_query().explain_plan(verbose)
fts_plan = await self._inner.to_fts_query().explain_plan(verbose)
# Indent sub-plans under the reranker
indented_vector = "\n".join(" " + line for line in vector_plan.splitlines())
indented_fts = "\n".join(" " + line for line in fts_plan.splitlines())
@@ -4057,12 +4014,14 @@ class AsyncHybridQuery(AsyncStandardQuery, AsyncVectorQueryBase):
-------
plan : str
"""
fts_query, vec_query, _, _ = self._create_child_queries()
results = ["Vector Search Query:"]
results.append(await vec_query.analyze_plan(distributed_metrics))
results.append(
await self._inner.to_vector_query().analyze_plan(distributed_metrics)
)
results.append("FTS Search Query:")
results.append(await fts_query.analyze_plan(distributed_metrics))
results.append(
await self._inner.to_fts_query().analyze_plan(distributed_metrics)
)
return "\n".join(results)
+2 -2
View File
@@ -720,7 +720,7 @@ class RemoteTable(Table):
Parameters
----------
query: list/np.ndarray/str/PIL.Image.Image, default None
The targeted vector to search for.
The targetted vector to search for.
- *default None*.
Acceptable types are: list, np.ndarray, PIL.Image.Image
@@ -1008,7 +1008,7 @@ class RemoteTable(Table):
return LOOP.run(self._table.drop_columns(columns))
def set_unenforced_primary_key(self, columns: Union[str, Iterable[str]]) -> None:
"""Set the unenforced primary key for this table to a single column."""
"""Not supported on LanceDB Cloud."""
return LOOP.run(self._table.set_unenforced_primary_key(columns))
def set_lsm_write_spec(self, spec: "LsmWriteSpec") -> None:
+1 -1
View File
@@ -175,7 +175,7 @@ class Reranker(ABC):
if the results haven't been executed yet or the results in arrow format.
query : str or None,
The input query. Some rerankers might not need the query to rerank.
In that case, it can be set to None explicitly. This is intended to
In that case, it can be set to None explicitly. This is inteded to
be handled by the reranker implementations.
deduplicate : bool, optional
Whether to deduplicate the results based on the `_rowid` column,
+4 -4
View File
@@ -1619,7 +1619,7 @@ class Table(ABC):
Parameters
----------
query: list/np.ndarray/str/PIL.Image.Image, default None
The targeted vector to search for.
The targetted vector to search for.
- *default None*.
Acceptable types are: list, np.ndarray, PIL.Image.Image
@@ -3841,7 +3841,7 @@ class LanceTable(Table):
Parameters
----------
query: list/np.ndarray/str/PIL.Image.Image, default None
The targeted vector to search for.
The targetted vector to search for.
- *default None*.
Acceptable types are: list, np.ndarray, PIL.Image.Image
@@ -5638,7 +5638,7 @@ class AsyncTable:
if fill_value is None:
fill_value = 0.0
# _sanitize_data is an old code path, but we will use it until the
# _santitize_data is an old code path, but we will use it until the
# new code path is ready.
if mode == "overwrite":
# For overwrite, apply the same preprocessing as create_table
@@ -5814,7 +5814,7 @@ class AsyncTable:
Parameters
----------
query: list/np.ndarray/str/PIL.Image.Image, default None
The targeted vector to search for.
The targetted vector to search for.
- *default None*.
Acceptable types are: list, np.ndarray, PIL.Image.Image
+5 -5
View File
@@ -327,8 +327,8 @@ def test_embedding_function_with_pandas(tmp_path):
) -> List[np.array]:
return [np.random.randn(self.ndims()).tolist() for _ in range(len(texts))]
registry = get_registry()
func = registry.get("mock-embedding").create()
registery = get_registry()
func = registery.get("mock-embedding").create()
class TestSchema(LanceModel):
text: str = func.SourceField()
@@ -394,9 +394,9 @@ def test_multiple_embeddings_for_pandas(tmp_path):
) -> List[np.array]:
return [np.random.randn(self.ndims()).tolist() for _ in range(len(texts))]
registry = get_registry()
func1 = registry.get("mock-embedding").create()
func2 = registry.get("mock-embedding2").create()
registery = get_registry()
func1 = registery.get("mock-embedding").create()
func2 = registery.get("mock-embedding2").create()
class TestSchema(LanceModel):
text: str = func1.SourceField()
+4 -14
View File
@@ -1011,13 +1011,8 @@ def test_fts_ngram(mem_db: DBConnection):
assert set(r["text"] for r in results) == {"lance database", "lance is cool"}
results = (
table.search(
"nce", # spellchecker:disable-line
query_type="fts",
)
.limit(10)
.to_list()
)
table.search("nce", query_type="fts").limit(10).to_list()
) # spellchecker:disable-line
assert len(results) == 2
assert set(r["text"] for r in results) == {"lance database", "lance is cool"}
@@ -1039,13 +1034,8 @@ def test_fts_ngram(mem_db: DBConnection):
assert set(r["text"] for r in results) == {"lance database", "lance is cool"}
results = (
table.search(
"nce", # spellchecker:disable-line
query_type="fts",
)
.limit(10)
.to_list()
)
table.search("nce", query_type="fts").limit(10).to_list()
) # spellchecker:disable-line
assert len(results) == 0
results = table.search("la", query_type="fts").limit(10).to_list()
-87
View File
@@ -203,93 +203,6 @@ async def test_async_hybrid_query_default_limit(table: AsyncTable):
assert texts.count("a") == 1
@pytest.mark.asyncio
async def test_async_hybrid_query_offset(table: AsyncTable):
# The offset window of a hybrid query must be a suffix of the same query
# run without an offset. Skipping the first rows of each sub-query instead
# of the first rows of the fused result silently changes which rows land in
# the window.
full = await (
table.query()
.nearest_to([0.0, 0.4])
.nearest_to_text("dog")
.limit(4)
.with_row_id()
.to_arrow()
)
assert len(full) == 4
second_page = await (
table.query()
.nearest_to([0.0, 0.4])
.nearest_to_text("dog")
.offset(2)
.limit(2)
.with_row_id()
.to_arrow()
)
assert second_page["_rowid"].to_pylist() == full["_rowid"].to_pylist()[2:]
first_page = await (
table.query()
.nearest_to([0.0, 0.4])
.nearest_to_text("dog")
.limit(2)
.with_row_id()
.to_arrow()
)
# Paging through the result must visit every row exactly once: no row
# repeated from the previous page and none dropped between the two.
paged = first_page["_rowid"].to_pylist() + second_page["_rowid"].to_pylist()
assert sorted(paged) == sorted(full["_rowid"].to_pylist())
@pytest.mark.asyncio
async def test_async_hybrid_query_fts_first_default_limit(table: AsyncTable):
# nearest_to() and nearest_to_text() build their new sibling sub-query from
# scratch, and that is the sub-query the default limit ends up on. So the
# side that carries the limit depends on the order the hybrid query was
# built in, and looking at only one side loses the limit for half the ways
# a hybrid query can be written. Without a limit the combined results are
# not truncated at all and the whole union of both candidate lists is
# returned.
await table.add([{"text": "dog", "vector": [50.0 + i, 50.0]} for i in range(10)])
result = await (
table.query().nearest_to_text("dog").nearest_to([0.1, 0.1]).to_arrow()
)
assert len(result) == 10
offset_result = await (
table.query().nearest_to_text("dog").nearest_to([0.1, 0.1]).offset(2).to_arrow()
)
assert len(offset_result) == 10
@pytest.mark.asyncio
async def test_async_hybrid_query_explain_plan_matches_execution(table: AsyncTable):
# Paging rewrites the sub-queries: each one fetches limit + offset rows with
# no offset of its own, and the window is sliced out after fusion. The plans
# have to be built from those rewritten sub-queries, otherwise explain_plan
# and analyze_plan describe a query that is never run.
query = (
table.query().nearest_to([0.0, 0.4]).nearest_to_text("dog").offset(2).limit(2)
)
await query.to_arrow()
plan = await query.explain_plan()
assert [
line.strip() for line in plan.splitlines() if "GlobalLimitExec" in line
] == [
"GlobalLimitExec: skip=0, fetch=4",
"GlobalLimitExec: skip=0, fetch=4",
]
analyzed = await query.analyze_plan()
assert analyzed.count("skip=0, fetch=4") == 2
assert "skip=2" not in analyzed
def test_hybrid_query_offset(sync_table: Table):
# The offset window of a hybrid query must be a suffix of the same query
# run without an offset -- it must not be silently ignored.
+1 -1
View File
@@ -81,7 +81,7 @@ def get_test_table(tmp_path):
"but his son was mortal",
"there hasn't been a good battlefield game since 2142",
"I wish they would make another one",
"campaigns are not as good as they used to be",
"campains are not as good as they used to be",
"Multiplayer and open world games have destroyed the single player experience",
"Maybe the future is console games",
"I don't know",
+1 -1
View File
@@ -3354,7 +3354,7 @@ def test_empty_query(mem_db: DBConnection):
# None is the same as default
df = table.search().select(["id"]).limit(None).to_arrow()
assert df.num_rows == 100
# invalid limist is the same as None, which is the same as default
# invalid limist is the same as None, wihch is the same as default
df = table.search().select(["id"]).limit(-1).to_arrow()
assert df.num_rows == 100
# valid limit should work
+1 -1
View File
@@ -334,7 +334,7 @@ pub struct PyQueryRequest {
pub column: Option<String>,
pub query_vector: Option<PyQueryVectors>,
pub minimum_nprobes: Option<usize>,
// None means user did not set it and default should be used (currently 20)
// None means user did not set it and default shoud be used (currenty 20)
// Some(0) means user set it to None and there is no limit
pub maximum_nprobes: Option<usize>,
pub lower_bound: Option<f32>,
+1 -1
View File
@@ -163,7 +163,7 @@ pub struct PolarsDataFrameRecordBatchReader {
impl PolarsDataFrameRecordBatchReader {
/// Creates a new `PolarsDataFrameRecordBatchReader` from a given Polars DataFrame.
/// If the input dataframe does not have aligned chunks, this function undergoes
/// the costly operation of reallocating each series as a single contiguous chunk.
/// the costly operation of reallocating each series as a single contigous chunk.
pub fn new(mut df: DataFrame) -> Result<Self> {
df.align_chunks();
let arrow_schema =
+1 -1
View File
@@ -827,7 +827,7 @@ impl Connection {
pub struct ConnectRequest {
/// Database URI
///
/// ### Accepted URI formats
/// ### Accpeted URI formats
///
/// - `/path/to/database` - local database on file system.
/// - `s3://bucket/path/to/database` or `gs://bucket/path/to/database` - database on cloud object store
+9 -9
View File
@@ -512,7 +512,7 @@ impl ListingDatabase {
// iter thru the query params and extract the commit store param
let mut engine = None;
let mut mirrored_store = None;
let mut filtered_queries = vec![];
let mut filtered_querys = vec![];
// WARNING: specifying engine is NOT a publicly supported feature in lancedb yet
// THE API WILL CHANGE
@@ -528,13 +528,13 @@ impl ListingDatabase {
mirrored_store = Some(value.to_string());
} else {
// to owned so we can modify the url
filtered_queries.push((key.to_string(), value.to_string()));
filtered_querys.push((key.to_string(), value.to_string()));
}
}
// Filter out the commit store query param -- it's a lancedb param
url.query_pairs_mut().clear();
url.query_pairs_mut().extend_pairs(filtered_queries);
url.query_pairs_mut().extend_pairs(filtered_querys);
// Take a copy of the query string so we can propagate it to lance.
// `query_pairs_mut()` leaves the URL with `Some("")` even when no
// pairs survive (or none existed in the first place), so an empty
@@ -896,11 +896,11 @@ impl Database for ListingDatabase {
}
async fn read_consistency(&self) -> Result<ReadConsistency> {
if let Some(interval) = self.read_consistency_interval {
if interval.is_zero() {
if let Some(read_consistency_inverval) = self.read_consistency_interval {
if read_consistency_inverval.is_zero() {
Ok(ReadConsistency::Strong)
} else {
Ok(ReadConsistency::Eventual(interval))
Ok(ReadConsistency::Eventual(read_consistency_inverval))
}
} else {
Ok(ReadConsistency::Manual)
@@ -3043,15 +3043,15 @@ mod tests {
/// across platforms — see the `file://` test below).
fn capture_query_like_connect(input_uri: &str) -> Option<String> {
let mut url = url::Url::parse(input_uri).unwrap();
let mut filtered_queries = Vec::new();
let mut filtered_querys = Vec::new();
for (key, value) in url.query_pairs() {
if key == ENGINE || key == MIRRORED_STORE {
continue;
}
filtered_queries.push((key.to_string(), value.to_string()));
filtered_querys.push((key.to_string(), value.to_string()));
}
url.query_pairs_mut().clear();
url.query_pairs_mut().extend_pairs(filtered_queries);
url.query_pairs_mut().extend_pairs(filtered_querys);
url.query().filter(|q| !q.is_empty()).map(|s| s.to_string())
}
+3 -3
View File
@@ -251,11 +251,11 @@ impl Database for LanceNamespaceDatabase {
}
async fn read_consistency(&self) -> Result<ReadConsistency> {
if let Some(interval) = self.read_consistency_interval {
if interval.is_zero() {
if let Some(read_consistency_inverval) = self.read_consistency_interval {
if read_consistency_inverval.is_zero() {
Ok(ReadConsistency::Strong)
} else {
Ok(ReadConsistency::Eventual(interval))
Ok(ReadConsistency::Eventual(read_consistency_inverval))
}
} else {
Ok(ReadConsistency::Manual)
+1 -1
View File
@@ -125,7 +125,7 @@ macro_rules! impl_pq_params_setter {
/// This value controls how much the vector is compressed during the quantization step.
/// The more sub vectors there are the less the vector is compressed. The default is
/// the dimension of the vector divided by 16. If the dimension is not evenly divisible
/// by 16 we use the dimension divided by 8.
/// by 16 we use the dimension divded by 8.
///
/// The above two cases are highly preferred. Having 8 or 16 values per subvector allows
/// us to use efficient SIMD instructions.
+1 -1
View File
@@ -1299,7 +1299,7 @@ impl VectorQuery {
/// This can be useful when there is a narrow filter to allow these queries to
/// spend more time searching and avoid potential false negatives.
///
/// Set to None to search all partitions, if needed, to satisfy the limit
/// Set to None to search all partitions, if needed, to satsify the limit
pub fn maximum_nprobes(mut self, maximum_nprobes: Option<usize>) -> Result<Self> {
if let Some(maximum_nprobes) = maximum_nprobes {
if maximum_nprobes == 0 {
+4 -137
View File
@@ -32,7 +32,6 @@ use crate::table::Tags;
use crate::table::UpdateResult;
use crate::table::lsm_stats::GetLsmStatsResponse;
use crate::table::merge::MergeFilter;
use crate::table::primary_key;
use crate::table::query::create_multi_vector_plan;
use crate::table::write_progress::FinishOnDrop;
use crate::table::{
@@ -69,7 +68,6 @@ use lance::arrow::json::{JsonDataType, JsonSchema};
use lance::dataset::refs::TagContents;
use lance::dataset::scanner::DatasetRecordBatchStream;
use lance::dataset::{ColumnAlteration, NewColumnTransform, Version};
use lance_core::datatypes::Schema as LanceSchema;
use lance_datafusion::exec::{OneShotExec, execute_plan};
use reqwest::{RequestBuilder, Response};
use serde::{Deserialize, Serialize};
@@ -2994,27 +2992,10 @@ impl<S: HttpSend> BaseTable for RemoteTable<S> {
}
}
/// The unenforced primary key is Lance schema field metadata, so this
/// installs it through the `update_field_metadata` endpoint. The commit
/// layer behind that endpoint is what actually installs and validates the
/// key, exactly as on a native table; the checks here only fail fast with
/// the same messages a native table gives.
async fn set_unenforced_primary_key(&self, columns: &[&str]) -> Result<()> {
self.check_mutable().await?;
let arrow_schema = self.schema().await?;
let schema = LanceSchema::try_from(arrow_schema.as_ref()).map_err(|e| Error::Schema {
message: format!("Invalid schema: {}", e),
})?;
primary_key::validate(&schema, columns)?;
self.update_field_metadata(&[FieldMetadataUpdate {
path: columns[0].to_string(),
metadata: primary_key::install_edit(),
replace: false,
}])
.await?;
Ok(())
async fn set_unenforced_primary_key(&self, _columns: &[&str]) -> Result<()> {
Err(Error::NotSupported {
message: "set_unenforced_primary_key is not supported on LanceDB cloud.".into(),
})
}
async fn flush_lsm(&self) -> Result<()> {
@@ -11851,120 +11832,6 @@ mod tests {
assert_eq!(result.version, 7);
}
/// The unenforced primary key is field metadata, so the remote table
/// installs it through the `update_field_metadata` endpoint.
#[tokio::test]
async fn test_set_unenforced_primary_key() {
let table = Table::new_with_handler("my_table", |request| {
assert_eq!(request.method(), "POST");
match request.url().path() {
"/v1/table/my_table/describe/" => {
let schema = Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("name", DataType::Utf8, true),
]);
http::Response::builder()
.status(200)
.body(describe_response(&schema))
.unwrap()
}
"/v1/table/my_table/update_field_metadata/" => {
let body = request_body_json(&request);
assert_eq!(body["updates"].as_array().unwrap().len(), 1);
let update = &body["updates"][0];
assert_eq!(update["path"], "id");
assert_eq!(update["replace"], json!(false));
assert_eq!(
update["metadata"]["lance-schema:unenforced-primary-key:position"],
"1"
);
assert_eq!(
update["metadata"]["lance-schema:unenforced-primary-key"],
json!(null)
);
http::Response::builder()
.status(200)
.body(r#"{"version": 3, "fields": {}}"#.to_string())
.unwrap()
}
path => panic!("Unexpected path: {}", path),
}
});
table.set_unenforced_primary_key(["id"]).await.unwrap();
}
/// Requests the native table rejects are rejected here too, before any
/// write reaches the server.
#[tokio::test]
async fn test_set_unenforced_primary_key_rejects_invalid_requests() {
let table = Table::new_with_handler("my_table", |request| match request.url().path() {
"/v1/table/my_table/describe/" => {
let schema = Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("score", DataType::Float32, true),
]);
http::Response::builder()
.status(200)
.body(describe_response(&schema))
.unwrap()
}
path => panic!("Unexpected path: {}", path),
});
for columns in [
vec![],
vec!["id", "score"],
vec!["nonexistent"],
vec!["score"],
] {
let err = table
.set_unenforced_primary_key(columns.clone())
.await
.unwrap_err();
assert!(
matches!(err, Error::InvalidInput { .. }),
"unexpected error for {:?}: {:?}",
columns,
err
);
}
}
/// The key is immutable once set, and the schema the server already
/// reports is enough to say so.
#[tokio::test]
async fn test_set_unenforced_primary_key_already_set() {
let table = Table::new_with_handler("my_table", |request| match request.url().path() {
"/v1/table/my_table/describe/" => {
let schema = Schema::new(vec![
Field::new("id", DataType::Int64, false).with_metadata(HashMap::from([(
"lance-schema:unenforced-primary-key:position".to_string(),
"1".to_string(),
)])),
Field::new("name", DataType::Utf8, false),
]);
http::Response::builder()
.status(200)
.body(describe_response(&schema))
.unwrap()
}
path => panic!("Unexpected path: {}", path),
});
for column in ["name", "id"] {
let err = table
.set_unenforced_primary_key([column])
.await
.unwrap_err();
assert!(
err.to_string().contains("already set"),
"unexpected error: {:?}",
err
);
}
}
// ----- Branch support -----
/// Parse a request's in-memory JSON body. Only valid for JSON-body ops
+3 -3
View File
@@ -78,7 +78,7 @@ pub mod delete;
pub mod lsm_stats;
pub mod merge;
pub mod optimize;
pub(crate) mod primary_key;
mod primary_key;
pub mod query;
pub mod refresh;
pub mod schema_evolution;
@@ -240,7 +240,7 @@ enum BadVectorHandling {
/// An error is returned
#[default]
Error,
/// The offending row is dropped
/// The offending row is droppped
Drop,
/// The invalid/missing items are replaced by fill_value
Fill(f32),
@@ -1326,7 +1326,7 @@ impl Table {
/// Note: if your condition is something like "some_id_column == 7" and
/// you are updating many rows (with different ids) then you will get
/// better performance with a single [`merge_insert`] call instead of
/// repeatedly calling this method.
/// repeatedly calilng this method.
pub fn update(&self) -> UpdateBuilder {
UpdateBuilder::new(self.inner.clone())
}
+1 -1
View File
@@ -52,7 +52,7 @@ enum ConsistencyMode {
/// refresh_window = min(3s, TTL/4)
///
/// | t < TTL - refresh_window | t < TTL | t >= TTL |
/// | Return value | Background refresh & return value | synchronous refresh |
/// | Return value | Background refresh & return value | syncronous refresh |
Eventual(BackgroundCache<Arc<Dataset>, Error>),
}
+1 -1
View File
@@ -103,7 +103,7 @@ impl MergeInsertBuilder {
/// but that behavior is subject to change.
///
/// An optional condition may be specified. If it is, then only
/// matched rows that satisfy the condition will be updated. Any
/// matched rows that satisfy the condtion will be updated. Any
/// rows that do not satisfy the condition will be left as they
/// are. Failing to satisfy the condition does not cause a
/// "matched row" to become a "not matched" row.
+1 -1
View File
@@ -904,7 +904,7 @@ fn unsharded_shard_id() -> Uuid {
/// Build a [`ShardWriterConfig`] from the persisted `writer_config_defaults`.
///
/// Unknown or unparsable keys are ignored; absent keys keep the
/// Unknown or unparseable keys are ignored; absent keys keep the
/// [`ShardWriterConfig`] default. The shard id is set by `mem_wal_writer`.
fn shard_writer_config_from_defaults(defaults: &HashMap<String, String>) -> ShardWriterConfig {
let mut config = ShardWriterConfig::default().with_shard_spec_id(SHARDING_SPEC_ID);
+45 -75
View File
@@ -11,26 +11,24 @@
//! Only a single-column primary key is supported, and the key cannot be
//! changed once set.
use std::collections::HashMap;
use arrow_schema::DataType;
use lance_core::datatypes::{
Field as LanceField, LANCE_UNENFORCED_PRIMARY_KEY, LANCE_UNENFORCED_PRIMARY_KEY_POSITION,
Schema as LanceSchema,
};
use lance_core::datatypes::{LANCE_UNENFORCED_PRIMARY_KEY, LANCE_UNENFORCED_PRIMARY_KEY_POSITION};
use crate::error::{Error, Result};
use crate::table::NativeTable;
/// Validate a `set_unenforced_primary_key` request against `schema`, returning
/// the field the key would be installed on.
/// Set the unenforced primary key on `table` to the single column in `columns`.
///
/// Shared by [`NativeTable`] and the remote table so both reject the same
/// requests with the same messages. Fails if `columns` is not exactly one
/// column (compound primary keys are not supported), if the column does not
/// exist or has an unsupported dtype, or if the table already has an
/// unenforced primary key (changing the primary key is not supported).
pub fn validate<'a>(schema: &'a LanceSchema, columns: &[&str]) -> Result<&'a LanceField> {
/// Fails if `columns` is not exactly one column (compound primary keys are not
/// supported), if the column does not exist or has an unsupported dtype, or if
/// the table already has an unenforced primary key (changing the primary key
/// is not supported).
pub(super) async fn set_unenforced_primary_key(
table: &NativeTable,
columns: &[&str],
) -> Result<()> {
table.dataset.ensure_mutable()?;
if columns.is_empty() {
return Err(Error::InvalidInput {
message: "set_unenforced_primary_key: a column is required".into(),
@@ -46,71 +44,43 @@ pub fn validate<'a>(schema: &'a LanceSchema, columns: &[&str]) -> Result<&'a Lan
}
let column = columns[0];
// The primary key is immutable once set. The Lance commit layer is the
// source of truth for this (it also covers the concurrent-writer race);
// this check just fails fast with a clear message.
if !schema.unenforced_primary_key().is_empty() {
return Err(Error::InvalidInput {
message: "set_unenforced_primary_key: an unenforced primary key is already set on this table; changing it is not supported".into(),
});
}
let field = schema.field(column).ok_or_else(|| Error::InvalidInput {
message: format!(
"set_unenforced_primary_key: column '{}' not found on table",
column
),
})?;
if !is_supported_pk_dtype(&field.data_type()) {
return Err(Error::InvalidInput {
message: format!(
"set_unenforced_primary_key: column '{}' has dtype {:?} which is not supported as a primary key. Supported: Int32, Int64, Utf8, LargeUtf8, Binary, LargeBinary, FixedSizeBinary",
column,
field.data_type()
),
});
}
Ok(field)
}
/// The field metadata edit that installs the primary key on a field: keys to
/// set (`Some`) or delete (`None`).
///
/// Position metadata is 1-indexed; `Schema::unenforced_primary_key` treats
/// position 0 as a legacy "no specific position" fallback, so the legacy
/// boolean key is cleared and only the position governs.
pub fn install_edit() -> HashMap<String, Option<String>> {
HashMap::from([
(LANCE_UNENFORCED_PRIMARY_KEY.to_string(), None),
(
LANCE_UNENFORCED_PRIMARY_KEY_POSITION.to_string(),
Some("1".to_string()),
),
])
}
/// Set the unenforced primary key on `table` to the single column in `columns`.
pub(super) async fn set_unenforced_primary_key(
table: &NativeTable,
columns: &[&str],
) -> Result<()> {
table.dataset.ensure_mutable()?;
let updates = {
let dataset = table.dataset.get().await?;
let field = validate(dataset.schema(), columns)?;
let schema = dataset.schema();
let mut metadata = field.metadata.clone();
for (key, value) in install_edit() {
match value {
Some(value) => {
metadata.insert(key, value);
}
None => {
metadata.remove(&key);
}
}
// The primary key is immutable once set. The Lance commit layer is the
// source of truth for this (it also covers the concurrent-writer race);
// this check just fails fast with a clear message.
if !schema.unenforced_primary_key().is_empty() {
return Err(Error::InvalidInput {
message: "set_unenforced_primary_key: an unenforced primary key is already set on this table; changing it is not supported".into(),
});
}
let field = schema.field(column).ok_or_else(|| Error::InvalidInput {
message: format!(
"set_unenforced_primary_key: column '{}' not found on table",
column
),
})?;
if !is_supported_pk_dtype(&field.data_type()) {
return Err(Error::InvalidInput {
message: format!(
"set_unenforced_primary_key: column '{}' has dtype {:?} which is not supported as a primary key. Supported: Int32, Int64, Utf8, LargeUtf8, Binary, LargeBinary, FixedSizeBinary",
column,
field.data_type()
),
});
}
// Position metadata is 1-indexed; `Schema::unenforced_primary_key`
// treats position 0 as a legacy "no specific position" fallback.
let mut metadata = field.metadata.clone();
metadata.remove(LANCE_UNENFORCED_PRIMARY_KEY);
metadata.insert(
LANCE_UNENFORCED_PRIMARY_KEY_POSITION.to_string(),
"1".to_string(),
);
vec![(field_id_to_u32(field.id, &field.name)?, metadata)]
};